├── .github └── workflows │ └── codeql-analysis.yml ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── Redis_block_store_details.md ├── TODO ├── redisvfs.c ├── redisvfs.h ├── sqlitedis.cc └── test.sh /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '31 17 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'cpp' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | - name: Install builddeps 45 | run: | 46 | sudo apt-get install libhiredis-dev 47 | 48 | # Initializes the CodeQL tools for scanning. 49 | - name: Initialize CodeQL 50 | uses: github/codeql-action/init@v1 51 | with: 52 | languages: ${{ matrix.language }} 53 | # If you wish to specify custom queries, you can do so here or in a config file. 54 | # By default, queries listed here will override any specified in a config file. 55 | # Prefix the list here with "+" to use these queries and those in the config file. 56 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@v1 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 https://git.io/JvXDl 65 | 66 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 67 | # and modify them (or add more) to build your code if your project 68 | # uses a compiled language 69 | 70 | #- run: | 71 | # make bootstrap 72 | # make release 73 | 74 | - name: Perform CodeQL Analysis 75 | uses: github/codeql-action/analyze@v1 76 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | project(sqlitedis VERSION 0.9 LANGUAGES C CXX) 4 | if(NOT CMAKE_BUILD_TYPE) 5 | set(CMAKE_BUILD_TYPE Release) 6 | endif() 7 | 8 | add_executable(sqlitedis sqlitedis.cc) 9 | add_executable(static-sqlitedis redisvfs.c sqlitedis.cc) 10 | add_library(redisvfs SHARED redisvfs.c) 11 | 12 | find_library(SQLITE3 sqlite3 REQUIRED) 13 | find_library(HIREDIS hiredis REQUIRED) 14 | 15 | # sqlite extension module 16 | # sqlite3 wants us to drop the "lib" prefix for the .so 17 | set_property(TARGET redisvfs PROPERTY POSITION_INDEPENDENT_CODE 1) 18 | set_property(TARGET redisvfs PROPERTY PREFIX "") 19 | target_link_libraries(redisvfs sqlite3 hiredis) 20 | 21 | # sqlitedis without redisvfs (probably should rename as it's a misnomer) 22 | target_link_libraries(sqlitedis sqlite3 hiredis) 23 | 24 | # sqlitedis with the redisvfs extension statically linked in rather than 25 | # needing sqlite to dynload it at runtime 26 | target_compile_definitions(static-sqlitedis PUBLIC STATIC_REDISVFS) 27 | target_link_libraries(static-sqlitedis sqlite3 hiredis) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, David Basden 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the copyright holder nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CPP=g++ 3 | SHELL=bash 4 | 5 | SQLITE_BUILD=../sqlite/build 6 | INCLUDES=-I${SQLITE_BUILD} -I/usr/include/hiredis 7 | LDFLAGS=-L${SQLITE_BUILD}/.libs 8 | LDLIBS=-l:libsqlite3.a -ldl -lpthread -lhiredis 9 | 10 | #CPPFLAGS=-Wall -O2 -ggdb ${INCLUDES} 11 | CPPFLAGS=-Wall -ggdb ${INCLUDES} 12 | 13 | default: sqlitedis redisvfs.so static-sqlitedis static-redisvfs.o 14 | 15 | clean: 16 | rm -f redisvfs.so sqlitedis static-sqlitedis static-sqldis.o 17 | 18 | # sqlite extension module 19 | redisvfs.so: redisvfs.c redisvfs.h 20 | gcc ${CPPFLAGS} ${INCLUDES} -fPIC -shared redisvfs.c -l hiredis -o redisvfs.so 21 | 22 | # test tool with statically linked redisvfs 23 | 24 | static-sqlitedis: CFLAGS=-Wall -ggdb -DSTATIC_REDISVFS 25 | static-sqlitedis: CPPFLAGS=${CFLAGS} 26 | static-sqlitedis: sqlitedis.cc redisvfs.c redisvfs.h 27 | gcc ${CFLAGS} ${INCLUDES} -o static-redisvfs.o -c redisvfs.c 28 | g++ $(CPPFLAGS) $(INCLUDES) $(LDFLAGS) -o static-sqlitedis static-redisvfs.o sqlitedis.cc $(LDLIBS) 29 | 30 | # Link vfsstat module 31 | # (of limited use as it uses the VFS it's shadowing to write it's log) 32 | vfsstat.so: SQLITE_SRC=../sqlite 33 | vfsstat.so: 34 | gcc ${CPPFLAGS} ${INCLUDES} -fPIC -shared ${SQLITE_SRC}/ext/misc/vfsstat.c -o vfsstat.so 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqliteredis 2 | 3 | SQLite extension to use redis as storage via an emulated VFS 4 | 5 | I've only written this as a quick proof of concept at the expense of code quality. Don't ever be put into production or with data you care about. 6 | 7 | This isn't something you should use without realising the horrible, horrible implications of meshing these things together. The sqlite3 docs go into some detail about why it's a bad idea to back the VFS layer onto a network filesystem, and this has even more edge cases to take into account. 8 | 9 | ### Example with sqlite3 cli tool 10 | 11 | ```sh 12 | $ sqlite3 13 | SQLite version 3.27.2 2019-02-25 16:06:06 14 | Enter ".help" for usage hints. 15 | Connected to a transient in-memory database. 16 | Use ".open FILENAME" to reopen on a persistent database. 17 | sqlite> .load ./redisvfs 18 | sqlite> .open "example.sqlite" 19 | sqlite> CREATE TABLE fish(a,b,c); 20 | sqlite> INSERT INTO fish VALUES (1,2,3); 21 | sqlite> INSERT INTO fish VALUES (4,5,6); 22 | sqlite> .exit 23 | $ sqlite3 24 | SQLite version 3.27.2 2019-02-25 16:06:06 25 | Enter ".help" for usage hints. 26 | Connected to a transient in-memory database. 27 | Use ".open FILENAME" to reopen on a persistent database. 28 | sqlite> .load ./redisvfs 29 | sqlite> .open "example.sqlite" 30 | sqlite> SELECT * FROM fish; 31 | 1|2|3 32 | 4|5|6 33 | sqlite> .exit 34 | $ redis-cli 'KEYS' '*' 35 | 1) "example.sqlite-journal:3" 36 | 2) "example.sqlite-journal:5" 37 | 3) "example.sqlite-journal:0" 38 | 4) "example.sqlite:2" 39 | 5) "example.sqlite-journal:6" 40 | 6) "example.sqlite:3" 41 | 7) "example.sqlite-journal:8" 42 | 8) "example.sqlite:1" 43 | 9) "example.sqlite-journal:4" 44 | 10) "example.sqlite:4" 45 | 11) "example.sqlite-journal:1" 46 | 12) "example.sqlite-journal:2" 47 | 13) "example.sqlite:5" 48 | 14) "example.sqlite:0" 49 | 15) "example.sqlite:6" 50 | 16) "example.sqlite-journal:filelen" 51 | 17) "example.sqlite:filelen" 52 | 18) "example.sqlite:7" 53 | 19) "example.sqlite-journal:7" 54 | $ 55 | ``` 56 | 57 | ### Features / Warnings / Implementation 58 | 59 | * Uses sqlite3 VFS interface to emulate pseudo-posix file IO (to the bare minimum needed) 60 | * No need to change SQL at all to suit the backend 61 | * Uses redis keys to emulate raw block storage 62 | * Each file is split up into 1024 byte blocks (too small and there is too much network bandwidth/latency overhead. Too large and the single threaded redis server may start blocking for more than microseconds, starving other clients) 63 | * Sparse block implementation (reading *only* a sparse area may or may not work but sqlite does not seem to do this) 64 | * Partial block reads/writes are done with GETRANGE/SETRANGE avoid read/modify/write races and to cut down on network overhead 65 | * Relies on redis ordering and consistency guarantees to have a consistent view from multiple sqlite3 clients on the same database (entirely untested) 66 | * Uses different redis keys to emulate a "file" on top of the block store 67 | * Tracks file lengths on write. 68 | * Allows truncation (current lazy implementation: only filesize metadata is changed) 69 | * Multiple sqlite databases supported on the same redis server (current "filename" used as a prefix in redis keyspace) 70 | * Can be dynamically loaded as an sqlite3 extension (.so) or built statically 71 | * Sets itself as the default VFS on load, so if you can get your app to load sqlite3 extensions, you shouldn't need to change anything else 72 | * Redis server connection defaults to locahost:6379, or (TODO) set in database connection URI as option 73 | 74 | ### Build requirements 75 | 76 | * hiredis (redis client library for C/C++) https://github.com/redis/hiredis 77 | * A recent sqlite3 (https://sqlite.org/) 78 | * C / C++ compiler 79 | * cmake + GNU Make 80 | 81 | ### Building the extension and cli test tool 82 | 83 | ```sh 84 | mkdir build 85 | cd build 86 | cmake .. 87 | make 88 | # Test 89 | ../test.sh 90 | ``` 91 | 92 | (See `./test.sh` for examples of the test tooling. `sqlitedis` needs to be told to load the `redisvfs` extension to talk to redis. `static-sqlitedis` has the redis VFS compiled in, and uses it by default. ) 93 | 94 | 95 | ### Author 96 | 97 | David Basden 98 | 99 | ### License 100 | 101 | BSD 3-clause. See "LICENSE" for more detail 102 | -------------------------------------------------------------------------------- /Redis_block_store_details.md: -------------------------------------------------------------------------------- 1 | # Redis block storage implementation 2 | 3 | Straight off: It's not a good fit for sqlite to use a network backed datastore 4 | at all, even for a network filesystem that provides harder guarantees than the 5 | block store described here does. SQLite's documentation gives a good overview 6 | of why this is (and some better options to consider) in their article 7 | [Appropriate Uses For SQLite](https://sqlite.org/whentouse.html) 8 | 9 | That out of the way, let's get into how we do this anyway. 10 | 11 | # Design goals 12 | 13 | * Simplicity 14 | * Minimal guarantees 15 | * Block storage independent from file emulation 16 | * Fixed block size 17 | * Partial block reads + writes 18 | * Sparse block implementation 19 | * Namespaced 20 | 21 | ## Simplicity / minimal guarantees 22 | 23 | The implementation of the block storage for the sqlite3 redisvfs is 24 | deliberately simple with minimal guarantees. 25 | 26 | Although this makes the sqlite3 redisvfs extension code a little uglier 27 | and more complex, having a simpler virtual block storage layer 28 | might make it easier in the future to replace redis with a different 29 | distributed block storage layer entirely. 30 | 31 | ### Block storage independent from file emulation 32 | 33 | Although redisvfs emulates files, the file emulation is built on top of 34 | the block storage (or alongside) the simple block storage here. 35 | 36 | ### Minimal guarantees 37 | 38 | The less guarantees the block storage implementation has to provide, 39 | the more flexibility there can be in how to implement the block storage 40 | backend (e.g. using redis-cluster, or a different scalable distributed 41 | datastore) 42 | 43 | ## Block store requirements 44 | 45 | Any guarantees not explicitly made MUST NOT be relied upon, even if they seem 46 | to work. 47 | 48 | ### Fixed block size 49 | 50 | The blocks should be a fixed size. This is so we hopefully get the benefits of 51 | 52 | * Simple implementation 53 | * Upper bounds on read/write impact to other clients on the same redis server 54 | 55 | There is deliberately no requirement for any single read/write operation to 56 | affect multiple blocks and this SHOULD NOT be implemented 57 | 58 | ### Partial block reads + writes if possible 59 | 60 | * To save on network and IO overhead and to avoid adding fetch/modify/store races, 61 | reads and writes of less than a whole block should be possible. 62 | 63 | * There is deliberately no requirement that a partial block read/write cross a block 64 | boundary (i.e. a single read/write touch multiple blocks) and this SHOULD NOT 65 | be implemented. (This would significantly increase the guarantees the backend would have to provide.) 66 | 67 | * A partial write to a block MUST NOT fill the rest of the block with anything 68 | other than \0s 69 | * Any read (partial or whole) to a block that has been partially filed MUST 70 | treat parts of the block that have not been written to as be filled with \0s 71 | 72 | ### Sparse blocks 73 | 74 | * Blocks are independent and should be stored sparsely where possible. e.g. 75 | 76 | * Assuming a fixed block size of 1024 bytes, a write to the block at offset 77 | 819200 will take at most 1024 bytes of storage 78 | * Assuming a fixed block size of 1024 bytes, a write to the block at offset 79 | 819200, followed by a write to the block at 4096 will take at most 2048 bytes of storage 80 | * Assuming a fixed block size of 1024 bytes, a write to the block at offset 81 | 8192, followed by a write to the block at 40960 will take at most 2048 bytes of storage 82 | 83 | * Any reads from a block that does not exist MUST either be interpreted as the block being zero filled OR fail entirely 84 | * That said, there is no hard requirement that a read succeed where no block has been written before 85 | 86 | ### Multiple block stores on same backend 87 | 88 | The implementation shouldn't get in the way of running multiple different 89 | blockstores in the same backend server. e.g. in redis, any by adding a prefix to any 90 | keys should be enough for complete independence between prefixes 91 | 92 | ### IO should be pipelined if possible 93 | 94 | * If possible, reads to multiple blocks should be possible without blocking on response for each block. This is not a hard requirement 95 | * There is deliberately no requirement for pipelining of mixed reads and writes. and this SHOULD NOT be implemented 96 | e.g. Although the ops (READ BLOCK 2, READ BLOCK 3, READ BLOCK 4) should be pipelined, but there is no requirement for (READ BLOCK 3, WRITE BLOCK 2, READ BLOCK 4) to be pipelined 97 | 98 | 99 | ### Write barriers 100 | 101 | * There must be a way to guarantee all previous writes are available for read. 102 | * It is acceptable for this barrier to be implicit for every write 103 | 104 | # Implementation 105 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * Check the linker flags for redisvfs.so - It's showing as an ELF pie executable, not a ELF shared object. 2 | (might just be `file` output. It's not like there is a lot of difference between an executable and a shared object) 3 | * Pull redis hostname and port from database URI vars 4 | * Clean up makefile 5 | -------------------------------------------------------------------------------- /redisvfs.c: -------------------------------------------------------------------------------- 1 | #ifdef STATIC_REDISVFS 2 | #include "sqlite3.h" 3 | #else 4 | #include 5 | SQLITE_EXTENSION_INIT1 6 | #endif // STATIC_REDISVFS 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "redisvfs.h" 17 | 18 | // Debugging 19 | // 20 | // Reference the parent VFS that we reference in pAppData 21 | #define PARENT_VFS(vfs) ((sqlite3_vfs *)(vfs->pAppData)) 22 | 23 | /* keyspace helpers */ 24 | 25 | /* pre: outkeyname is exactly REDISVFS_MAX_KEYLEN+1 bytes */ 26 | static int get_blockkey(RedisFile *rf, int64_t offset, char *outkeyname) { 27 | // (REDISVFS_MAX_KEYLEN - REDISVFS_MAX_PREFIXLEN) = 32 characters 28 | // to encode the block number. 1 character for a delimiter, plus 29 | // 14 bytes for hex encoding any block number for a 64 bit offset 30 | // (given 1024 byte blocks) meeans we still have 17 bytes free 31 | // if we need to encode something else in the keyname later on. 32 | int blocknum = offset / REDISVFS_BLOCKSIZE; 33 | int written = snprintf(outkeyname, REDISVFS_KEYBUFLEN, "%s:%x", rf->keyprefix, blocknum); 34 | 35 | assert(written < REDISVFS_KEYBUFLEN); 36 | return written; 37 | } 38 | 39 | /* emulate file size tracking by storing the max value stored 40 | * pre: outkeyname is exactly REDISVFS_MAX_KEYLEN+1 bytes */ 41 | static int get_filesizekey(RedisFile *rf, char *outkeyname) { 42 | int written = snprintf(outkeyname, REDISVFS_KEYBUFLEN, "%s:filelen", rf->keyprefix); 43 | assert(written < REDISVFS_KEYBUFLEN); 44 | return written; 45 | } 46 | 47 | static inline int64_t _start_of_block(int64_t offset) { 48 | return offset - (offset % REDISVFS_BLOCKSIZE); 49 | } 50 | static inline int64_t _start_of_next_block(int64_t offset) { 51 | return _start_of_block(offset) + REDISVFS_BLOCKSIZE; 52 | } 53 | 54 | // Only used if we nest too much evil macro expansion of the debugreply macros 55 | static inline void redis_debugreplyarray (const redisReply *reply) { 56 | for (int i=0; ielements; ++i) redis_debugreply(reply->element[i]); 57 | } 58 | 59 | /* Make it easier to play fast and loose with redis pipelining */ 60 | static int redis_discard_replies(RedisFile *rf, int ndiscards) { 61 | for (int i=0; iredisctx, (void **)&reply) != REDIS_OK) 64 | return REDIS_ERR; 65 | #if 0 66 | DLOG("DISCARDING REPLY:"); 67 | redis_debugreply(reply); 68 | #endif 69 | freeReplyObject(reply); 70 | } 71 | return REDIS_OK; 72 | } 73 | 74 | 75 | /* redis blockio */ 76 | 77 | static int redis_queuecmd_whole_block_read(RedisFile *rf, const sqlite3_int64 offset) { 78 | assert((offset % REDISVFS_BLOCKSIZE) == 0); 79 | 80 | char key[REDISVFS_KEYBUFLEN]; 81 | int keylen = get_blockkey(rf, offset, key); 82 | 83 | return redisAppendCommandArgv(rf->redisctx, 2, 84 | (const char *[]){ "GET", key }, 85 | (const size_t[]){ 3, keylen }); 86 | } 87 | 88 | /* caller is saying filesize is at least 'minfilesize' 89 | * appends 2 commands */ 90 | static int redis_queue_increase_filesize_to(RedisFile *rf, int64_t minfilesize) { 91 | char key[REDISVFS_KEYBUFLEN]; 92 | get_filesizekey(rf, key); 93 | 94 | // We store filesize in an ordered set 95 | // 96 | // To get the filesize we peek the item with the highest cardinality 97 | // 98 | // If we have a lower bound on filesize, we just add the bound to the 99 | // ordered set, and then remove (trim) all but the highest cardinality item. 100 | // 101 | // As the trim is both optional, and safe to rerun, there is no race with 102 | // multiple interleved calls 103 | int ret = redisAppendCommand(rf->redisctx, "ZADD %s %d %d", key, minfilesize, minfilesize); 104 | if (ret == REDIS_ERR) 105 | return ret; 106 | 107 | // Trim to only largest entry 108 | return redisAppendCommand(rf->redisctx, "ZREMRANGEBYRANK %s %d %d", key, 0, -2); 109 | 110 | } 111 | static int redis_consume_increase_filesize_to(RedisFile *rf) { 112 | return redis_discard_replies(rf, 2); 113 | } 114 | 115 | // WARNING: Don't use in pipeline 116 | // Returns 0 if the file doesnt exist 117 | static int64_t redis_get_filesize(RedisFile *rf) { 118 | char key[REDISVFS_KEYBUFLEN]; 119 | get_filesizekey(rf, key); 120 | 121 | redisReply *reply; 122 | if ((reply = redisCommand(rf->redisctx, "ZREVRANGE %s 0 0", key)) == NULL) { 123 | return REDIS_ERR; 124 | } 125 | redis_debugreply(reply); 126 | 127 | int64_t filesize; 128 | if (reply->type != REDIS_REPLY_ARRAY) { 129 | filesize = -1; 130 | } else if (reply->elements == 0) { 131 | filesize = 0; 132 | } else if (reply->element[0]->type != REDIS_REPLY_STRING) { 133 | filesize = -1; 134 | } else { 135 | filesize = atoll(reply->element[0]->str); 136 | } 137 | freeReplyObject(reply); 138 | return filesize; 139 | } 140 | 141 | static int64_t redis_force_set_filesize(RedisFile *rf, int64_t filesize) { 142 | assert(filesize >= 0); 143 | char key[REDISVFS_KEYBUFLEN]; 144 | get_filesizekey(rf, key); 145 | 146 | if (redisAppendCommand(rf->redisctx, "MULTI") == REDIS_ERR) 147 | return REDIS_ERR; 148 | if (redisAppendCommand(rf->redisctx, "DEL %s", key) == REDIS_ERR) 149 | return REDIS_ERR; 150 | if (redis_queue_increase_filesize_to(rf, filesize) == REDIS_ERR) 151 | return REDIS_ERR; 152 | if (redisAppendCommand(rf->redisctx, "EXEC") == REDIS_ERR) 153 | return REDIS_ERR; 154 | 155 | // TODO: Actually check return codes. Ignore if DEL fails. 156 | if (redis_discard_replies(rf, 2) == REDIS_ERR) // MULTI,DEL 157 | return REDIS_ERR; 158 | if (redis_consume_increase_filesize_to(rf)) 159 | return REDIS_ERR; 160 | if (redis_discard_replies(rf, 1) == REDIS_ERR) // EXEC 161 | return REDIS_ERR; 162 | 163 | return REDIS_OK; 164 | } 165 | 166 | 167 | // NO PIPELINING HANDLED. Don't call it unless you are 168 | // sure there are no other commands sent before or after 169 | // without appropriate compartmentalisation 170 | // 171 | // FIXME: if we're not going to pipeline, just use redisCommand 172 | static bool redis_does_block_exist(RedisFile *rf, int64_t offset) { 173 | assert((offset % REDISVFS_BLOCKSIZE) == 0); 174 | 175 | char key[REDISVFS_KEYBUFLEN]; 176 | int keylen = get_blockkey(rf, offset, key); 177 | 178 | if (redisAppendCommandArgv(rf->redisctx, 2, 179 | (const char *[]){ "EXISTS", key }, 180 | (const size_t[]){ 6, keylen }) != REDIS_OK) { 181 | return false; 182 | }; 183 | bool exists = false; 184 | redisReply *reply; 185 | if (redisGetReply(rf->redisctx, (void**)&reply) == REDIS_OK) { 186 | if (reply->type == REDIS_REPLY_INTEGER) { 187 | DLOG("Redis INT: %lld", reply->integer); 188 | exists = reply->integer == 1; 189 | } else { DLOG("Redis something else"); } 190 | } 191 | return exists; 192 | } 193 | 194 | /* pre: buf is >= REDISVFS_BLOCKSIZE */ 195 | static int redis_queuecmd_whole_block_write(RedisFile *rf, int64_t offset, const char *buf) { 196 | assert((offset % REDISVFS_BLOCKSIZE) == 0); 197 | 198 | char key[REDISVFS_KEYBUFLEN]; 199 | int keylen = get_blockkey(rf, offset, key); 200 | 201 | return redisAppendCommandArgv(rf->redisctx, 3, 202 | (const char *[]){ "SET", key, buf }, 203 | (const size_t[]){ 3, keylen, REDISVFS_BLOCKSIZE }); 204 | } 205 | 206 | static int redis_queuecmd_partial_block_read(RedisFile *rf, int64_t offset, int64_t len) { 207 | // GETRANGE range is inclusive of first and last indices 208 | int64_t block_first = offset % REDISVFS_BLOCKSIZE; 209 | int64_t block_last = block_first + len - 1; 210 | 211 | assert(len > 0); 212 | assert(block_last < REDISVFS_BLOCKSIZE); 213 | 214 | 215 | char key[REDISVFS_KEYBUFLEN]; 216 | get_blockkey(rf, offset, key); 217 | 218 | return redisAppendCommand(rf->redisctx, "GETRANGE %s %d %d", key, block_first, block_last); 219 | } 220 | 221 | static int redis_queuecmd_partial_block_write(RedisFile *rf, int64_t offset, const char *buf, int64_t len) { 222 | assert(len > 0); 223 | int64_t block_first = offset % REDISVFS_BLOCKSIZE; 224 | assert((block_first + len) <= REDISVFS_BLOCKSIZE); 225 | 226 | char key[REDISVFS_KEYBUFLEN]; 227 | int keylen = get_blockkey(rf, offset, key); 228 | char block_offsetstr[32]; 229 | snprintf(block_offsetstr, 32, "%ld", block_first); 230 | 231 | DLOG("SETRANGE %s %s ...(len %ld)",key, block_offsetstr, len); 232 | return redisAppendCommandArgv(rf->redisctx, 4, 233 | (const char *[]){ "SETRANGE", key, block_offsetstr, buf }, 234 | (const size_t[]){ 8, keylen, strlen(block_offsetstr), len }); 235 | } 236 | 237 | 238 | static int redis_queuecmd_delete_block(RedisFile *rf, sqlite3_int64 offset) { 239 | assert((offset % REDISVFS_BLOCKSIZE) == 0); 240 | 241 | char key[REDISVFS_KEYBUFLEN]; 242 | int keylen = get_blockkey(rf, offset, key); 243 | 244 | return redisAppendCommandArgv(rf->redisctx, 2, 245 | (const char *[]){ "DEL", key }, 246 | (const size_t[]){ 3, keylen }); 247 | } 248 | 249 | /* 250 | * File API implementation 251 | * 252 | * These all work on a specific open file. pointers to these 253 | * are rolled up in an sqlite3_io_methods struct, which is referrenced 254 | * by each sqlite3_file * generated by redisvfs_open 255 | */ 256 | 257 | int redisvfs_close(sqlite3_file *fp) { 258 | DLOG("disconnecting from redis"); 259 | RedisFile *rf = (RedisFile *)fp; 260 | if (rf->redisctx) { 261 | redisFree(rf->redisctx); 262 | rf->redisctx = 0; 263 | } 264 | return SQLITE_OK; 265 | } 266 | int redisvfs_write(sqlite3_file *fp, const void *buf, int iAmt, sqlite3_int64 iOfst) { 267 | RedisFile *rf = (RedisFile *)fp; 268 | DLOG("(fp=%p prefix='%s' offset=%lld len=%d)", rf, rf->keyprefix, iOfst, iAmt); 269 | 270 | int64_t write_startp = iOfst; 271 | int64_t write_endp = iOfst+iAmt; 272 | 273 | // Queue writes 274 | for (int64_t leftp=write_startp; leftp blknext) ? blknext : write_endp; 278 | 279 | const char *bufleft = (const char *)buf + (leftp - write_startp); 280 | 281 | if ((leftp == blkstart) && (rightp == blknext)) { 282 | assert((rightp-leftp) == REDISVFS_BLOCKSIZE); 283 | DLOG("%s full block write @ %ld", rf->keyprefix, leftp); 284 | if( redis_queuecmd_whole_block_write(rf, leftp, bufleft) == REDIS_ERR) { 285 | return SQLITE_IOERR; 286 | // TODO: use redis check error and bubble up to sql last error 287 | } 288 | } else { 289 | DLOG("%s Partial block write [%ld..%ld)", rf->keyprefix, leftp,rightp); 290 | if( redis_queuecmd_partial_block_write(rf, leftp, bufleft, rightp-leftp) == REDIS_ERR) { 291 | return SQLITE_IOERR; 292 | } 293 | } 294 | } 295 | 296 | // Execute write and check responses 297 | int64_t successfully_written = 0; 298 | int return_status = SQLITE_OK; 299 | 300 | for (int64_t leftp=write_startp; leftp blknext) ? blknext : write_endp; 303 | 304 | redisReply *reply; 305 | 306 | DLOG("checking reply for [%ld..%ld)", leftp,rightp); 307 | if (redisGetReply(rf->redisctx, (void **)&reply) == REDIS_ERR) { 308 | DLOG("ERROR: redisGetReply: %s", rf->redisctx->errstr); 309 | return SQLITE_IOERR_WRITE; 310 | } 311 | 312 | redis_debugreply(reply); 313 | // FIXME: check reply before incrementing written 314 | if (return_status == SQLITE_OK) { 315 | successfully_written += rightp-leftp; 316 | } 317 | freeReplyObject(reply); 318 | } 319 | 320 | // write barrier (guaranteed for single server) then update filesize 321 | // TODO: Make write barrier optional and remove SAFE_APPEND guarantee 322 | // if we need to increase write performance 323 | // FIXME : Add write barrier cross-cluster 324 | if (successfully_written > 0) { 325 | int64_t minfilesize = iOfst + successfully_written; 326 | // I think we could do this earlier on there without the extra RTT, but 327 | // that assumes all the writes succeeded. 328 | if (redis_queue_increase_filesize_to(rf, minfilesize) != REDIS_OK) 329 | return_status = SQLITE_IOERR_WRITE; 330 | if (redis_consume_increase_filesize_to(rf) != REDIS_OK) 331 | return_status = SQLITE_IOERR_WRITE; 332 | } 333 | DLOG("written %ld/%d. Returning %s\n", successfully_written, iAmt, 334 | return_status == SQLITE_OK ? "SQLITE_OK" : "NOT OK"); 335 | return return_status; 336 | } 337 | 338 | int redisvfs_read(sqlite3_file *fp, void *buf, int iAmt, sqlite3_int64 iOfst) { 339 | RedisFile *rf = (RedisFile *)fp; 340 | DLOG("(fp=%p prefix='%s' offset=%lld len=%d)", rf, rf->keyprefix, iOfst, iAmt); 341 | 342 | int64_t read_startp = iOfst; 343 | int64_t read_endp = iOfst+iAmt; 344 | 345 | // Queue reads 346 | for (int64_t leftp=read_startp; leftp blknext) ? blknext : read_endp; 350 | 351 | if ((leftp == blkstart) && (rightp == blknext)) { 352 | DLOG("full block read"); 353 | if( redis_queuecmd_whole_block_read(rf, blkstart) == REDIS_ERR) { 354 | return SQLITE_IOERR; 355 | // TODO: use redis check error and bubble up to sql last error 356 | } 357 | } else { 358 | DLOG("Partial block read [%ld..%ld)", leftp,rightp); 359 | if( redis_queuecmd_partial_block_read(rf, leftp, rightp-leftp) == REDIS_ERR) { 360 | return SQLITE_IOERR; 361 | } 362 | } 363 | } 364 | 365 | // sqlite3 requires short reads be zero-filled for the rest of the buffer, 366 | // and says database corruption will otherwise occur 367 | memset(buf, 0, iAmt); /* This will cover the requirement but only required in the case of a short read */ 368 | 369 | int64_t successfully_read = 0; 370 | 371 | // We track this becausei we need to continue draining the 372 | // connection of command responses regardless of if the commands 373 | // were successful. 374 | int returnStatus = SQLITE_OK; 375 | 376 | // Execute and read responses 377 | for (int64_t leftp=read_startp; leftp blknext) ? blknext : read_endp; 380 | 381 | redisReply *reply; 382 | 383 | DLOG("fetching next (sub)block from redis stream"); 384 | if (redisGetReply(rf->redisctx, (void **)&reply) == REDIS_ERR) { 385 | DLOG("ERROR: redisGetReply: %s", rf->redisctx->errstr); 386 | return SQLITE_IOERR_READ; 387 | } 388 | if (reply->type == REDIS_REPLY_STRING) { 389 | DLOG("Redis STRING: %lu bytes", reply->len); 390 | // The read counter can only increment if any previous 391 | // reads were successful and not short 392 | if (returnStatus == SQLITE_OK) { 393 | if (reply->len > rightp-leftp) { 394 | DLOG("read reply overflow"); 395 | return SQLITE_IOERR_READ; 396 | } 397 | if (reply->len < rightp-leftp) { 398 | DLOG("short read"); 399 | returnStatus = SQLITE_IOERR_SHORT_READ; 400 | } 401 | if (reply->len > 0) { 402 | memcpy(buf+(leftp-read_startp), reply->str, reply->len); 403 | } 404 | successfully_read += rightp-leftp; 405 | } 406 | else { 407 | DLOG("Dropping because lack of continuity"); 408 | } 409 | } 410 | else if (reply->type == REDIS_REPLY_NIL) { 411 | DLOG("Block not found"); 412 | if (returnStatus == SQLITE_OK) 413 | returnStatus = SQLITE_IOERR_SHORT_READ; 414 | } 415 | else { 416 | DLOG("wrong reply type. Bailing straight away"); 417 | return SQLITE_IOERR_READ; 418 | } 419 | 420 | freeReplyObject(reply); 421 | } 422 | if ((returnStatus == SQLITE_IOERR_SHORT_READ) && (successfully_read == 0)) { 423 | returnStatus = SQLITE_IOERR_READ; 424 | } 425 | assert(!SQLITE_OK || (successfully_read == iAmt)); 426 | return returnStatus; 427 | } 428 | int redisvfs_truncate(sqlite3_file *fp, sqlite3_int64 size) { 429 | sqlite3_int64 existing_size; 430 | if (redisvfs_fileSize(fp, &existing_size) == REDIS_ERR) 431 | return SQLITE_ERROR; 432 | if (existing_size < size) 433 | return SQLITE_ERROR; 434 | if (redis_force_set_filesize((RedisFile *)fp, size) == REDIS_ERR) 435 | return SQLITE_ERROR; 436 | return SQLITE_OK; 437 | } 438 | int redisvfs_sync(sqlite3_file *fp, int flags) { 439 | DLOG("stub"); 440 | // Noop. All our writes are synchronous. 441 | // TODO: We can put a hard barrier in here to redis and block if we really want 442 | return SQLITE_OK; 443 | } 444 | int redisvfs_fileSize(sqlite3_file *fp, sqlite3_int64 *pSize) { 445 | RedisFile *rf = (RedisFile *)fp; 446 | DLOG("get_filesize(%s)", rf->keyprefix); 447 | *pSize = redis_get_filesize(rf); 448 | DLOG("... get_filesize(%s) = %lld", rf->keyprefix, *pSize); 449 | return (*pSize >= 0) ? SQLITE_OK : SQLITE_ERROR; 450 | } 451 | int redisvfs_lock(sqlite3_file *fp, int eLock) { 452 | DLOG("stub flock(%s,%d)",((RedisFile *)fp)->keyprefix,eLock); 453 | return SQLITE_OK; // FIXME: Implement 454 | } 455 | int redisvfs_unlock(sqlite3_file *fp, int eLock) { 456 | DLOG("stub funlock(%s,%d)",((RedisFile *)fp)->keyprefix,eLock); 457 | return SQLITE_OK; // FIXME: Implement 458 | } 459 | int redisvfs_checkReservedLock(sqlite3_file *fp, int *pResOut) { 460 | DLOG("stub"); 461 | return !SQLITE_OK; // FIXME: Implement 462 | } 463 | int redisvfs_fileControl(sqlite3_file *fp, int op, void *pArg) { 464 | if ( op == SQLITE_FCNTL_VFSNAME ) { 465 | DLOG("SQLITE_FCNTL_VFSNAME"); 466 | char **out = (char **)pArg; 467 | *out = sqlite3_mprintf("redisvfs"); 468 | return SQLITE_OK; 469 | } 470 | DLOG("No idea what %d is", op); 471 | return SQLITE_NOTFOUND; 472 | } 473 | int redisvfs_sectorSize(sqlite3_file *fp) { 474 | DLOG("stub"); 475 | return REDISVFS_BLOCKSIZE; 476 | } 477 | int redisvfs_deviceCharacteristics(sqlite3_file *fp) { 478 | DLOG("entry"); 479 | // Describe ordering and consistency guarantees that we 480 | // can provide. See sqlite3.h 481 | // TODO implement SQLITE_IOCAP_BATCH_ATOMIC 482 | // TODO: If we remove SQLITE_IOCAP_ATOMIC and replace with caveated 483 | // atomic op flags, we can remove transactions with redis entirely 484 | return ( SQLITE_IOCAP_ATOMIC | SQLITE_IOCAP_SAFE_APPEND | 485 | SQLITE_IOCAP_SEQUENTIAL | SQLITE_IOCAP_POWERSAFE_OVERWRITE | 486 | SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN ); 487 | } 488 | 489 | #if 0 490 | /* Methods above are valid for version 1 */ 491 | int redisvfs_shmMap(sqlite3_file *fp, int iPg, int pgsz, int, void volatile **pp); 492 | int redisvfs_shmLock(sqlite3_file *fp, int offset, int n, int flags); 493 | void redisvfs_shmBarrier(sqlite3_file *fp); 494 | int redisvfs_shmUnmap(sqlite3_file *fp, int deleteFlag); 495 | /* Methods above are valid for version 2 */ 496 | int redisvfs_fetch(sqlite3_file *fp, sqlite3_int64 iOfst, int iAmt, void **pp); 497 | int redisvfs_unfetch(sqlite3_file *fp, sqlite3_int64 iOfst, void *p); 498 | /* Methods above are valid for version 3 */ 499 | #endif 500 | 501 | /* references to file API implementation. Added to each RedisFile * 502 | */ 503 | const sqlite3_io_methods redisvfs_io_methods = { 504 | 1, 505 | redisvfs_close, 506 | redisvfs_read, 507 | redisvfs_write, 508 | redisvfs_truncate, 509 | redisvfs_sync, 510 | redisvfs_fileSize, 511 | redisvfs_lock, 512 | redisvfs_unlock, 513 | redisvfs_checkReservedLock, 514 | redisvfs_fileControl, 515 | redisvfs_sectorSize, 516 | redisvfs_deviceCharacteristics, 517 | }; 518 | 519 | 520 | /* 521 | * VFS API implementation 522 | * 523 | * Stuff that isn't just for a specific already-open file. 524 | * This all gets referenced by our sqlite3_vfs * 525 | */ 526 | 527 | int redisvfs_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *f, int flags, int *pOutFlags) { 528 | DLOG("(zName='%s',flags=%d)", zName,flags); 529 | #if 0 530 | if (!(flags & SQLITE_OPEN_MAIN_DB)) { 531 | return SQLITE_CANTOPEN; 532 | } 533 | #endif 534 | 535 | //hardcode hostname and port. for now. grab from database URI later 536 | const char *hostname = REDISVFS_DEFAULT_HOST; 537 | int port = REDISVFS_DEFAULT_PORT; 538 | 539 | 540 | RedisFile *rf = (RedisFile *)f; 541 | memset(rf, 0, sizeof(RedisFile)); 542 | // pMethods must be set even if redisvfs_open fails! 543 | rf->base.pMethods = &redisvfs_io_methods; 544 | 545 | rf->keyprefixlen = strnlen(zName, REDISVFS_MAX_PREFIXLEN+1); 546 | if (rf->keyprefixlen > REDISVFS_MAX_PREFIXLEN) { 547 | DLOG("key prefix ('filename') length too long"); 548 | return SQLITE_CANTOPEN; 549 | } 550 | rf->keyprefix = zName; // Guaranteed to be unchanged until after xClose(*rf) 551 | DLOG("key prefix: '%s'", rf->keyprefix); 552 | 553 | rf->redisctx = redisConnect(hostname,port); 554 | if (!(rf->redisctx) || rf->redisctx->err) { 555 | if (rf->redisctx) 556 | fprintf(stderr, "%s: Error: %s\n", __func__, rf->redisctx->errstr); 557 | return SQLITE_CANTOPEN; 558 | } 559 | 560 | // FIXME: Check if OCREATE 561 | #if 0 562 | if (!redis_does_block_exist(rf, 0)) { 563 | DLOG("does not exists. Told."); 564 | return SQLITE_IOERR; 565 | } 566 | #endif 567 | 568 | return SQLITE_OK; 569 | } 570 | 571 | int redisvfs_delete(sqlite3_vfs *vfs, const char *zName, int syncDir) { 572 | DLOG("(zName='%s',syncDir=%d)", zName, syncDir); 573 | // TODO: Better implementation that actually deletes things 574 | RedisFile rf; 575 | int openflags; 576 | 577 | if (redisvfs_open(vfs, zName, (sqlite3_file *)(&rf), 0, &openflags) != SQLITE_OK) 578 | return SQLITE_IOERR_DELETE; 579 | 580 | if (redis_force_set_filesize(&rf, 0) == REDIS_ERR) 581 | return SQLITE_IOERR_DELETE; 582 | 583 | redisvfs_close((sqlite3_file *)(&rf)); 584 | return SQLITE_OK; 585 | } 586 | int redisvfs_access(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut) { 587 | DLOG("(zName='%s', flags=%d (%s%s%s))", zName, flags, 588 | (flags & SQLITE_ACCESS_EXISTS) == SQLITE_ACCESS_EXISTS ? "SQLITE_ACCESS_EXISTS" : "", 589 | (flags & SQLITE_ACCESS_READWRITE) == SQLITE_ACCESS_READWRITE ? "SQLITE_ACCESS_READWRITE" : "", 590 | (flags & SQLITE_ACCESS_READ) == SQLITE_ACCESS_READ ? "SQLITE_ACCESS_READ" : ""); 591 | 592 | // FIXME: Can only check redis from a file created with redisvfs_open 593 | //static bool redis_does_block_exist(RedisFile *rf, int64_t offset) { 594 | *pResOut = 0; 595 | return SQLITE_OK; 596 | } 597 | int redisvfs_fullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut) { 598 | DLOG("(zName='%s',nOut=%d)", zName,nOut); 599 | sqlite3_snprintf(nOut, zOut, "%s", zName); // effectively strcpy with sqlite3 mm 600 | return SQLITE_OK; 601 | } 602 | 603 | // 604 | // These we don't implement but just pass through to the existing default VFS 605 | // As they are more OS abstraction than FS abstraction it doesn't affect us 606 | // 607 | // Note: Turns out that you really need to pass the parent VFS struct referennce into it's own 608 | // calls not our VFS struct ref. This is obvious in retrospect. 609 | // 610 | #ifdef DEBUG_REDISVFS 611 | #define VFS_SHIM_CALL(_callname,_vfs,...) \ 612 | DLOG("%s->" #_callname,PARENT_VFS(_vfs)->zName), \ 613 | PARENT_VFS(_vfs)->_callname(PARENT_VFS(_vfs), __VA_ARGS__) 614 | #else 615 | #define VFS_SHIM_CALL(_callname,_vfs,...) \ 616 | PARENT_VFS(_vfs)->_callname(PARENT_VFS(_vfs), __VA_ARGS__) 617 | #endif 618 | 619 | void * redisvfs_dlOpen(sqlite3_vfs *vfs, const char *zFilename) { 620 | return VFS_SHIM_CALL(xDlOpen, vfs, zFilename); 621 | } 622 | void redisvfs_dlError(sqlite3_vfs *vfs, int nByte, char *zErrMsg) { 623 | VFS_SHIM_CALL(xDlError, vfs, nByte, zErrMsg); 624 | } 625 | void (* redisvfs_dlSym(sqlite3_vfs *vfs, void *pHandle, const char *zSymbol))(void) { 626 | return VFS_SHIM_CALL(xDlSym, vfs, pHandle, zSymbol); 627 | } 628 | void redisvfs_dlClose(sqlite3_vfs *vfs, void *pHandle) { 629 | VFS_SHIM_CALL(xDlClose, vfs, pHandle); 630 | } 631 | int redisvfs_randomness(sqlite3_vfs *vfs, int nByte, char *zOut) { 632 | return VFS_SHIM_CALL(xRandomness, vfs, nByte, zOut); 633 | } 634 | int redisvfs_sleep(sqlite3_vfs *vfs, int microseconds) { 635 | return VFS_SHIM_CALL(xSleep, vfs, microseconds); 636 | } 637 | int redisvfs_currentTime(sqlite3_vfs *vfs, double *prNow) { 638 | return VFS_SHIM_CALL(xCurrentTime, vfs, prNow); 639 | } 640 | int redisvfs_getLastError(sqlite3_vfs *vfs, int nBuf, char *zBuf) { 641 | // FIXME: Implement. Called after another call fails. 642 | return VFS_SHIM_CALL(xGetLastError, vfs, nBuf, zBuf); 643 | } 644 | int redisvfs_currentTimeInt64(sqlite3_vfs *vfs, sqlite3_int64 *piNow) { 645 | return VFS_SHIM_CALL(xCurrentTimeInt64, vfs, piNow); 646 | } 647 | 648 | /* VFS object for sqlite3 */ 649 | sqlite3_vfs redis_vfs = { 650 | 2, 0, REDISVFS_MAX_PREFIXLEN, 0, /* iVersion, szOzFile, mxPathname, pNext */ 651 | "redisvfs", 0, /* zName, pAppData */ 652 | redisvfs_open, 653 | redisvfs_delete, 654 | redisvfs_access, 655 | redisvfs_fullPathname, 656 | redisvfs_dlOpen, 657 | redisvfs_dlError, 658 | redisvfs_dlSym, 659 | redisvfs_dlClose, 660 | redisvfs_randomness, 661 | redisvfs_sleep, 662 | redisvfs_currentTime, 663 | redisvfs_getLastError, 664 | redisvfs_currentTimeInt64 665 | }; 666 | 667 | 668 | /* Setup VFS structures and initialise */ 669 | int redisvfs_register() { 670 | int ret; 671 | 672 | //FIXME Move out of here. Can be evaluated ompiletime 673 | redis_vfs.szOsFile = sizeof(RedisFile); 674 | 675 | // Get the existing default vfs and pilfer it's OS abstrations 676 | // This will normally work as long as the (previously) default 677 | // vfs is not unloaded. 678 | // 679 | // If the existing default vfs is trying to do the same thing 680 | // things may get weird, but sqlite3 concrete implementations 681 | // out of the box will not do that (only vfs shim layers like vfslog) 682 | // 683 | // 684 | sqlite3_vfs *defaultVFS = sqlite3_vfs_find(0); 685 | if (defaultVFS == 0) 686 | return SQLITE_NOLFS; 687 | 688 | // Use our pAppData opaque pointer to store a reference to the 689 | // underlying VFS. 690 | redis_vfs.pAppData = (void *)defaultVFS; 691 | 692 | // Register outselves as the new default 693 | ret = sqlite3_vfs_register(&redis_vfs, 1); 694 | if (ret != SQLITE_OK) { 695 | fprintf(stderr, "redisvfsinit could not register itself\n"); 696 | return ret; 697 | } 698 | 699 | return SQLITE_OK; 700 | } 701 | 702 | 703 | #ifndef STATIC_REDISVFS 704 | 705 | #ifdef _WIN32 706 | __declspec(dllexport) 707 | #endif 708 | 709 | /* If we are compiling as an sqlite3 extension make a module load entrypoint 710 | * 711 | * sqlite3_SONAME_init is a well-known symbol 712 | */ 713 | int sqlite3_redisvfs_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { 714 | int ret; 715 | 716 | SQLITE_EXTENSION_INIT2(pApi); 717 | ret = redisvfs_register(); 718 | return (ret == SQLITE_OK) ? SQLITE_OK_LOAD_PERMANENTLY : ret; 719 | } 720 | 721 | #endif 722 | -------------------------------------------------------------------------------- /redisvfs.h: -------------------------------------------------------------------------------- 1 | #ifndef __redisvfs_h 2 | #define __redisvfs_h 3 | 4 | #include 5 | 6 | typedef struct sqlite3_vfs RedisVFS; 7 | typedef struct RedisFile RedisFile; 8 | 9 | #define REDISVFS_DEFAULT_HOST "127.0.0.1" 10 | #define REDISVFS_DEFAULT_PORT 6379 11 | 12 | #define REDISVFS_BLOCKSIZE 1024 13 | 14 | // These are mostly arbitrary, but both MAX_PREFIXLEN and MAX_KEYLEN 15 | // must be increased/decreased by the same amount. Given every file 16 | // operation sends the key over the wire, there is an impact of a larger 17 | // key size 18 | #define REDISVFS_MAX_PREFIXLEN 96 19 | #define REDISVFS_MAX_KEYLEN 128 20 | 21 | #define REDISVFS_KEYBUFLEN ( REDISVFS_MAX_KEYLEN + 1 ) 22 | 23 | /* virtual file that we can use to keep per "file" state */ 24 | struct RedisFile { 25 | // mandatory base class 26 | sqlite3_file base; 27 | 28 | // Just have a file be the same as a redis connection for now 29 | redisContext *redisctx; 30 | 31 | const char *keyprefix; 32 | size_t keyprefixlen; 33 | }; 34 | 35 | /* Prototypes of all sqlite3 file op functions that can be implemented 36 | */ 37 | int redisvfs_close(sqlite3_file *fp); 38 | int redisvfs_read(sqlite3_file *fp, void *buf, int iAmt, sqlite3_int64 iOfst); 39 | int redisvfs_write(sqlite3_file *fp, const void *buf, int iAmt, sqlite3_int64 iOfst); 40 | int redisvfs_truncate(sqlite3_file *fp, sqlite3_int64 size); 41 | int redisvfs_sync(sqlite3_file *fp, int flags); 42 | int redisvfs_fileSize(sqlite3_file *fp, sqlite3_int64 *pSize); 43 | int redisvfs_lock(sqlite3_file *fp, int eLock); 44 | int redisvfs_unlock(sqlite3_file *fp, int eLock); 45 | int redisvfs_checkReservedLock(sqlite3_file *fp, int *pResOut); 46 | int redisvfs_fileControl(sqlite3_file *fp, int op, void *pArg); 47 | int redisvfs_sectorSize(sqlite3_file *fp); 48 | int redisvfs_deviceCharacteristics(sqlite3_file *fp); 49 | /* Methods above are valid for version 1 */ 50 | int redisvfs_shmMap(sqlite3_file *fp, int iPg, int pgsz, int, void volatile **pp); 51 | int redisvfs_shmLock(sqlite3_file *fp, int offset, int n, int flags); 52 | void redisvfs_shmBarrier(sqlite3_file *fp); 53 | int redisvfs_shmUnmap(sqlite3_file *fp, int deleteFlag); 54 | /* Methods above are valid for version 2 */ 55 | int redisvfs_fetch(sqlite3_file *fp, sqlite3_int64 iOfst, int iAmt, void **pp); 56 | int redisvfs_unfetch(sqlite3_file *fp, sqlite3_int64 iOfst, void *p); 57 | /* Methods above are valid for version 3 */ 58 | 59 | /* Prototypes of all sqlite3 vfs functions that can be implemented 60 | */ 61 | int redisvfs_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *f, int flags, int *pOutFlags); 62 | int redisvfs_delete(sqlite3_vfs *vfs, const char *zName, int syncDir); 63 | int redisvfs_access(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut); 64 | int redisvfs_fullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut); 65 | void * redisvfs_dlOpen(sqlite3_vfs*, const char *zFilename); 66 | void redisvfs_dlError(sqlite3_vfs*, int nByte, char *zErrMsg); 67 | void (* redisvfs_dlSym(sqlite3_vfs*,void*, const char *zSymbol))(void); 68 | void redisvfs_dlClose(sqlite3_vfs*, void*); 69 | int redisvfs_randomness(sqlite3_vfs*, int nByte, char *zOut); 70 | int redisvfs_sleep(sqlite3_vfs*, int microseconds); 71 | int redisvfs_currentTime(sqlite3_vfs*, double*); 72 | int redisvfs_getLastError(sqlite3_vfs*, int, char *); 73 | int redisvfs_currentTimeInt64(sqlite3_vfs*, sqlite3_int64*); 74 | 75 | int redisvfs_register(); 76 | #ifndef STATIC_REDISVFS 77 | int sqlite3_redisvfs_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); 78 | #endif 79 | 80 | /* Really bad debugging macros that go on forever */ 81 | #ifdef DEBUG_REDISVFS 82 | #define DLOG(fmt,...) fprintf(stderr, "%s[%d]: %s: " fmt "\n", __FILE__, __LINE__, __func__, ##__VA_ARGS__),fflush(stderr) 83 | #else 84 | #define DLOG(fmt,...) do{}while(0) 85 | #endif 86 | 87 | #define _redis_debugreply_val(reply) do {\ 88 | if (reply->type == REDIS_REPLY_INTEGER) \ 89 | DLOG("Redis INT: %lld", reply->integer); \ 90 | else if (reply->type == REDIS_REPLY_STRING) \ 91 | DLOG("Redis STRING: %lu bytes: %s", reply->len, reply->str); \ 92 | else if (reply->type == REDIS_REPLY_STATUS) \ 93 | DLOG("Redis STATUS: %s", reply->str); \ 94 | else if (reply->type == REDIS_REPLY_ERROR) \ 95 | DLOG("Redis ERROR: %s", reply->str); \ 96 | else if (reply->type == REDIS_REPLY_NIL) \ 97 | DLOG("Redis NIL"); \ 98 | else if (reply->type == REDIS_REPLY_ARRAY) { \ 99 | DLOG("REDIS ARRAY {"); \ 100 | redis_debugreplyarray(reply); \ 101 | DLOG("} END REDIS ARRAY"); \ 102 | } \ 103 | else DLOG("Unknown redis reply type %d", reply->type); \ 104 | } while(0) 105 | 106 | #define redis_debugreply(reply) \ 107 | if (reply->type == REDIS_REPLY_ARRAY) { \ 108 | DLOG("REDIS ARRAY {"); \ 109 | for (int i=0; ielements; ++i) _redis_debugreply_val(reply->element[i]); \ 110 | DLOG("} END REDIS ARRAY"); \ 111 | } else { \ 112 | _redis_debugreply_val(reply); \ 113 | } 114 | #endif // __redisvfs_h 115 | -------------------------------------------------------------------------------- /sqlitedis.cc: -------------------------------------------------------------------------------- 1 | /* A simple C++ wrapper for sqlite3 to allow for easy testing of extensions 2 | * 3 | * David Basden 4 | */ 5 | 6 | #include 7 | #include 8 | 9 | #include "sqlite3.h" 10 | 11 | #ifdef STATIC_REDISVFS 12 | extern "C" { 13 | #include "redisvfs.h" 14 | } 15 | #endif 16 | 17 | 18 | class SQLengine { 19 | sqlite3 *db; 20 | 21 | static int _row_print_callback(void *arg, int ncols, char **cols, char **colnames) { 22 | for (int i=0; izName); 69 | } 70 | 71 | static void loadPersistentExtension(const char *sharedLib) { 72 | // If the know the extension is persistent, we just create a 73 | // memory backed sqlite db load the extension into it, and then 74 | // throw away the db 75 | SQLengine tmpeng(":memory:"); 76 | tmpeng.loadExtension(sharedLib); 77 | } 78 | 79 | static void dumpvfslist() { 80 | std::cerr << "vfs available:"; 81 | for(sqlite3_vfs *vfs=sqlite3_vfs_find(0); vfs; vfs=vfs->pNext){ 82 | std::cerr << " " << vfs->zName; 83 | } 84 | std::cerr << std::endl; 85 | std::cerr << "default vfs is " << SQLengine::defaultVFS() << std::endl; 86 | } 87 | static std::string defaultVFS() { 88 | sqlite3_vfs *vfs = sqlite3_vfs_find(0); 89 | return std::string(vfs->zName); 90 | } 91 | 92 | 93 | }; 94 | 95 | int main(int argc, const char **argv) { 96 | #ifdef STATIC_REDISVFS 97 | if (redisvfs_register() != SQLITE_OK) { 98 | return 1; 99 | } 100 | #endif 101 | 102 | const char* extname = getenv("SQLITE_LOADEXT"); 103 | if (extname != NULL) { 104 | //std::cerr << "Loading extension "<" << std::endl << 111 | std::endl << "optional environment variables: SQLITE_DB SQLITE_LOADEXT" < sql; 117 | 118 | char *envdbname = getenv("SQLITE_DB"); 119 | if (envdbname) { 120 | //std::cerr << "(using database '"<(envdbname); 122 | } 123 | else { 124 | sql = std::make_shared(); 125 | } 126 | 127 | //std::cerr << "(using vfs " << sql->currentVFSname() << ")" << std::endl; 128 | sql->exec(argv[1]); 129 | 130 | return 0; 131 | } 132 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -o pipefail 4 | 5 | unset SQLITE_DB 6 | unset SQLITE_LOADEXT 7 | 8 | echo --- no sqliteredis 9 | 10 | (./sqlitedis 2>&1 || true )| fgrep vfs # Dump list of VFSs 11 | 12 | ( 13 | set -x 14 | export SQLITE_DB=":memory:" 15 | ./sqlitedis 'select 1+2;' 16 | ./sqlitedis ' 17 | DROP TABLE IF EXISTS fish; 18 | CREATE TABLE fish (a,b,c); 19 | INSERT INTO fish VALUES (1,2,3); 20 | INSERT INTO fish VALUES (4,5,6); 21 | SELECT * FROM fish; 22 | DROP TABLE fish;' 23 | ) 24 | 25 | echo 26 | echo --- static sqliteredis 27 | ( 28 | (./static-sqlitedis 2>&1 || true )| fgrep vfs # Dump list of VFSs 29 | 30 | export SQLITE_DB='file:database?vfs=redisvfs' 31 | set -x 32 | ./static-sqlitedis 'select 1+2' 33 | ./static-sqlitedis 'DROP TABLE IF EXISTS fish' 34 | ./static-sqlitedis 'CREATE TABLE fish (a,b,c)' 35 | ./static-sqlitedis 'INSERT INTO fish VALUES (1,2,3)' 36 | ./static-sqlitedis 'INSERT INTO fish VALUES (4,5,6)' 37 | ./static-sqlitedis 'SELECT * FROM fish' 38 | ./static-sqlitedis 'DROP TABLE fish' 39 | ) 40 | 41 | echo 42 | echo --- dynload sqliteredis 43 | ( 44 | export SQLITE_LOADEXT=./redisvfs 45 | (./sqlitedis 2>&1 || true )| fgrep vfs # Dump list of VFSs 46 | 47 | export SQLITE_DB='file:database?vfs=redisvfs' 48 | set -x 49 | ./sqlitedis 'select 1+2' 50 | ./sqlitedis 'DROP TABLE IF EXISTS fish' 51 | ./sqlitedis 'CREATE TABLE fish (a,b,c)' 52 | ./sqlitedis 'INSERT INTO fish VALUES (1,2,3)' 53 | ./sqlitedis 'INSERT INTO fish VALUES (4,5,6)' 54 | ./sqlitedis 'SELECT * FROM fish' 55 | ./sqlitedis 'DROP TABLE fish' 56 | ) 57 | --------------------------------------------------------------------------------