├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── bitallpos └── bitallpos.c ├── geo ├── bench-test.sh ├── geo.c ├── geo.h ├── geo.tcl ├── geohash.c ├── geohash.h ├── geohash_helper.c ├── geohash_helper.h ├── geojson.c ├── geojson.h ├── module.c ├── test.c ├── to-redis.sh ├── zset.c └── zset.h ├── hmsetif └── hmsetif.c ├── hmsetnx └── hmsetnx.c ├── json ├── i_yajl.c ├── i_yajl.h ├── json.c ├── json.h ├── jsonobj.c ├── jsonobj.h ├── jsonobj.tcl ├── jsonobj_box.c ├── jsonobj_box.h ├── jsonobj_delete.c ├── jsonobj_delete.h ├── jsonobj_field_set.c ├── jsonobj_field_set.h ├── jsonobj_get.c ├── jsonobj_get.h ├── jsonobj_set.c └── jsonobj_set.h ├── poong └── poong.c └── scriptname └── scriptname.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.so 2 | *.dSYM 3 | tags 4 | geo-test 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Redistribution and use in source and binary forms, with or without 2 | modification, are permitted provided that the following conditions are met: 3 | 4 | 1. Redistributions of source code must retain the above copyright notice, this 5 | list of conditions and the following disclaimer. 6 | 2. Redistributions in binary form must reproduce the above copyright notice, 7 | this list of conditions and the following disclaimer in the documentation 8 | and/or other materials provided with the distribution. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 11 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 12 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 13 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 14 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 15 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 16 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 17 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 18 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 19 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Redis Module Toolkit (krmt) Makefile 2 | # 3 | # The more colorful parts of this Makefile were 4 | # taken from the primary Redis Makefile. 5 | # 6 | # Place a checkout of the Dynamic Redis source tree 7 | # at ../redis or alter REDIS_BASE to point 8 | # to your redis source tree checkout. 9 | 10 | REDIS_BASE=../redis 11 | LUA_SRC=$(REDIS_BASE)/deps/lua/src 12 | REDIS_SRC=$(REDIS_BASE)/src 13 | 14 | # We include these files directly in our modules so our compiler 15 | # can inline their function bodies when appropriate. 16 | # Including sds.c directly gives us a 5% to 10% throughput boost 17 | # due to clever inlining at compile time. 18 | SDS_C:=$(REDIS_SRC)/sds.c 19 | ZMALLOC_C:=$(REDIS_SRC)/zmalloc.c 20 | NETWORKING_C=$(REDIS_SRC)/networking.c 21 | IMPORT=$(SDS_C) $(ZMALLOC_C) $(NETWORKING_C) 22 | 23 | # YAJL is used by json currently, and maybe geo soon. Let's make 24 | # it a hard-stop requirement for now. 25 | # Note: we include YAJL directly in our modules instead of linking to a 26 | # the yajl library so the compiler can attempt to inline where appropriate. 27 | YAJL_BASE=../yajl 28 | YAJL_S=$(YAJL_BASE)/src 29 | # Hopefully you only have one built yajl version: 30 | YAJL_I=$(dir $(wildcard $(YAJL_BASE)/build/yajl-*/include/)) 31 | 32 | ifeq ($(wildcard $(YAJL_BASE)/build),) 33 | $(error You must build YAJL (cd ..;git clone https://github.com/lloyd/yajl;cd yajl;./configure;make) before using krmt) 34 | endif 35 | 36 | # For debugging, you may want to back this down to -O0 and remove -DPRODUCTION to 37 | # generate verbose logging in some modules. 38 | CFLAGS:=-O2 -DPRODUCTION -g -Wall -pedantic -std=c99 -Wno-unused-function -Wno-format $(CFLAGS) 39 | 40 | # Flags for .so compiling; OS X be crazy 41 | ifeq ($(shell uname -s),Darwin) 42 | SHARED_FLAGS=-fPIC -dynamiclib -Wl,-undefined,dynamic_lookup 43 | # Clang supports -flto on OS X 10.9 44 | CFLAGS:=-flto $(CFLAGS) 45 | # On OS X, GCC doesn't support march=native, so try to use sse4.2 instead 46 | ifneq (,$(findstring gcc,$(CC))) 47 | CFLAGS:=-msse4.2 $(CFLAGS) 48 | else 49 | CFLAGS:=-march=native $(CFLAGS) 50 | endif 51 | else 52 | SHARED_FLAGS=-fPIC -shared 53 | # _DEFAULT_SOURCE provides built-in M_ macros for math constants under Linux 54 | # You can add -flto if you're using GCC >= 4.9 for potential improvements when bulk linking 55 | CFLAGS:=-march=native -D_DEFAULT_SOURCE $(CFLAGS) 56 | endif 57 | 58 | FINAL_CFLAGS:=$(CFLAGS) -I$(LUA_SRC) -I$(REDIS_SRC) 59 | 60 | CCCOLOR="\033[34m" 61 | SRCCOLOR="\033[33m" 62 | BINCOLOR="\033[37;1m" 63 | CLEANCOLOR="\033[32;1m" 64 | ENDCOLOR="\033[0m" 65 | 66 | ifndef_any_of = $(filter undefined,$(foreach v,$(1),$(origin $(v)))) 67 | ifdef_any_of = $(filter-out undefined,$(foreach v,$(1),$(origin $(v)))) 68 | # Run with "make VERBOSE=1" if you want full command output 69 | ifeq ($(call ifdef_any_of,VERBOSE V),) 70 | QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR) 1>&2; 71 | QUIET_CLEAN = @printf ' %b %b\n' $(CLEANCOLOR)CLEAN$(ENDCOLOR) $(BINCOLOR)$^$(ENDCOLOR) 1>&2; 72 | endif 73 | 74 | .PHONY: all clean 75 | .SUFFIXES: .so 76 | 77 | DIRS:=$(filter-out ./, $(dir $(wildcard *[^dSYM]/))) 78 | OBJ:=$(DIRS:/=.so) 79 | OBJ_D:=$(OBJ:%=%.dSYM) 80 | 81 | # Yeah, so, this Makefile is more of a "build" file. It doesn't respond to 82 | # changes in source files for rebuilds. 83 | # You should always "make clean; make -j" when building. 84 | all: $(OBJ) 85 | #$(filter-out /home/matt/repos/redis/deps/lua/src/lua.c, $(wildcard /home/matt/repos/redis/deps/lua/src/*.c)) 86 | %.so: DIRNAME=$* 87 | %.so: $(wildcard %/*.c) 88 | $(QUIET_CC) $(CC) $(SHARED_FLAGS) $(FINAL_CFLAGS) -I./$(DIRNAME)/ -o $@ $(wildcard $(DIRNAME)/*.c) $(IMPORT) 89 | 90 | geo.so: $(wildcard %/*.c) 91 | $(QUIET_CC) $(CC) $(SHARED_FLAGS) $(FINAL_CFLAGS) -I./$(DIRNAME)/ -o $@ $(filter-out $(DIRNAME)/test.c, $(wildcard $(DIRNAME)/*.c)) $(IMPORT) 92 | $(QUIET_CC) $(CC) $(FINAL_CFLAGS) -I./$(DIRNAME)/ -o geo-test $(DIRNAME)/geohash.c $(DIRNAME)/test.c 93 | 94 | json.so: $(wildcard %/*.c) 95 | @echo "Note: JSON module ONLY works on 2.8 branches (DO NOT use JSON module with 3.0 or unstable branches)." 96 | $(QUIET_CC) $(CC) $(SHARED_FLAGS) $(FINAL_CFLAGS) -I$(YAJL_I) -I./$(DIRNAME)/ -o $@ $(wildcard $(DIRNAME)/*.c) $(wildcard $(YAJL_S)/*.c) $(IMPORT) -lm 97 | 98 | clean: 99 | $(QUIET_CLEAN) rm -rf $(OBJ) $(OBJ_D) 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | krmt: Kit of Redis Module Tools 2 | =============================== 3 | 4 | About 5 | ----- 6 | Welcome to a kit of tools you can easily add to Redis. 7 | 8 | This repository gives you multiple ready-to-use Redis Add-on Modules. 9 | 10 | You can also use the modules here for examples and templates to create 11 | your own your own loadable Redis modules. 12 | 13 | **Note:** You must use a 14 | [Dynamic Redis](https://matt.sh/dynamic-redis) server if you 15 | want to use modules. Regular Redis doesn't know about modules. 16 | 17 | @skidder maintains a [deployable](https://registry.hub.docker.com/u/urlgrey/dynamic-redis/) 18 | [dockerfile](https://github.com/urlgrey/docker-dynamic-redis) based on the current [development 19 | branch](https://registry.hub.docker.com/u/urlgrey/dynamic-redis/dockerfile/) 20 | of [dynamic-redis-unstable](https://github.com/mattsta/redis/commits/dynamic-redis-unstable). 21 | 22 | Disclaimers: 23 | 24 | - Redis is single threaded for all command processing. 25 | - While Redis is inside your command function, nothing else runs. 26 | - Your command should operate and return as quickly as possible. See existing 27 | Redis commands in the main `redisCommandTable[]` at the top of redis.c for 28 | examples of how to write different Redis commands. 29 | - Make sure to release objects and free memory you allocate. You will 30 | introduce memory leaks if you forget to free things. 31 | - You don't have to write in C. You can use C, C++, Go (?), Guile, Chicken, or anything else 32 | generating a shared library with C symbols for your operating system. You 33 | can even link additional libraries (want numerical routines? embedded 34 | sqlite3? embedded snappy/zippy?) 35 | 36 | 37 | Writing Your Module 38 | ------------------- 39 | To write your module: start out with the simple module provided in this repository 40 | and modify commands, functions, fields, and structs as needed. 41 | 42 | You will need to study the existing Redis source to learn how existing Redis 43 | commands work. 44 | 45 | Also see the notes at [Dynamic Redis: Use Command Modules](https://matt.sh/dynamic-redis#_use-command-modules). 46 | 47 | Bundled Commands 48 | ---------------- 49 | Included in `krmt`: 50 | - `json.so` - an experiment in storing JSON documents in Redis 51 | - See [Redis JSON Document Store](https://matt.sh/redis-json) for usage details. 52 | - `geo.so` - provides efficient geographic coordinate range searches 53 | - See [Redis Geo Commands](https://matt.sh/redis-geo) for usage details. 54 | - `hmsetif.so` - provides `HMSETIF` (renamed from [HMSETCK](https://github.com/tsee/redis/commit/9fdc57ca481b05433f9c37d9f041861e008c595b)) 55 | - `hmsetnx.so` - provides `HMSETNX` (from a [pull request](https://github.com/antirez/redis/issues/542) 56 | and updated to integrate into more recent Redis versions). 57 | - `bitallpos.so` - provides `BITALLPOS` command returning the positions 58 | of all set bits in a string (from a [pull request](https://github.com/antirez/redis/pull/1295)). 59 | - `scriptname.so` - provides `SCRIPTNAME` and `EVALNAME` commands allowing you 60 | to bind user friendly names to loaded script SHA hashes, then you can call 61 | scripts by name (using `EVALNAME`) instead of by a 40 character long hash 62 | reference. 63 | - Right now, this module has the best comment structure and best design 64 | patterns for creating your own modules. 65 | - Use the `dynamic-redis-unstable` branch to build `scriptname.c` since 66 | `scriptname.c` depends on a header not exported on released versions yet. 67 | - `poong.so` - minimal module showing how the basic Dynamic Redis interface 68 | API works. 69 | 70 | Building 71 | -------- 72 | For building, you need a copy of the Dynamic Redis source tree. 73 | 74 | If you want to build against the most recent 2.8 commits, use: 75 | 76 | ```haskell 77 | mkdir -p ~/repos 78 | cd ~/repos 79 | # We need YAJL for JSON commands and the Makefile refuses to run without it. 80 | git clone https://github.com/lloyd/yajl 81 | cd yajl 82 | ./configure; make 83 | cd .. 84 | git clone https://github.com/mattsta/redis 85 | cd redis 86 | git checkout dynamic-redis-2.8 87 | make 88 | cd .. 89 | git clone https://github.com/mattsta/krmt 90 | cd krmt 91 | make clean; make -j 92 | ``` 93 | 94 | If you want to build against the current development branch, 95 | just change `dynamic-redis-2.8` to `dynamic-redis-unstable`. 96 | 97 | Usage 98 | ----- 99 | After building your module, you can load it into [Dynamic Redis](https://matt.sh/dynamic-redis). 100 | 101 | Compatability 102 | ------------- 103 | As Redis adds or removes features, sometimes modules may need to 104 | be updated to take into account different functions or interfaces 105 | available to them. 106 | 107 | Stable releases of Dynamic Redis (as of 0.5.2) define a few 108 | `DYN_FEATURE_[feature]` constants you can use for preprocessor selective 109 | including or excluding of code. 110 | 111 | The currently defined features are negative features allowing the absense 112 | of them to mean "use behavior from unstable branch." 113 | 114 | You can check all the current feature defines in `version.h` of your 115 | Dynamic Redis checkout. 116 | 117 | For the unstable branch, no features are defined. 118 | 119 | Automatic Module Reloading 120 | -------------------------- 121 | During development, you can automatically reload your 122 | modules using some simple loops. Just run these 123 | from the directory where you compile your modules 124 | and everything should work. 125 | 126 | Linux: 127 | ```bash 128 | while inotifywait -e modify module.so; do 129 | redis-cli config set module-add `pwd`/module.so 130 | done 131 | ``` 132 | 133 | OS X: 134 | ```bash 135 | brew install kqwait 136 | while kqwait module.so; do 137 | redis-cli config set module-add `pwd`/module.so 138 | done 139 | ``` 140 | 141 | Tips 142 | ---- 143 | If you are on OS X, you should monitor your redis-server for 144 | new memory leaks by running `leaks` in a persistent terminal window: 145 | ```bash 146 | watch "leaks redis-server" 147 | ``` 148 | 149 | Think of `leaks` as `valgrind`, except it can monitor leaks in any 150 | live process. We run `leaks` in a `watch` loop to refresh the 151 | check fairly often. 152 | 153 | Under some use cases, `leaks` can cause an infinite loop in OS X's 154 | `malloc` routine. If that happens, rest assured it's a problem 155 | with `leaks` and not your code. You can also try running under 156 | `jemalloc` on OS X (not the default) to clear up any strange OS X 157 | malloc issues. 158 | 159 | If you are on Linux, you should run your module tests under 160 | `valgrind` after a good day of development. Any changes should 161 | also be re-evaluated for memory leaks and invalid memory access. 162 | 163 | Perfecting a Redis command takes time and understanding. Using 164 | memory allocation debuggers will help you eventually internalize 165 | the complexities of how and when Redis uses reference counted objects 166 | and malloc'd strings for certain operations. 167 | 168 | Sample Output 169 | ------------- 170 | Sample output on startup on OS X (Linux will complain if you give it a module name without a path): 171 | ```c 172 | matt@ununoctium:~/repos/redis/src% ./redis-server --module-strict no --module-add ping.so 173 | [81021] 28 Mar 14:43:14.500 * Module [] loaded 145 commands. 174 | [81021] 28 Mar 14:43:14.501 * Loading new [ping.so] module. 175 | [81021] 28 Mar 14:43:14.501 # [ping.so] version mismatch. Module was compiled against 2.9.11, but we are 2.8.8-dynamic-0.5. 176 | [81021] 28 Mar 14:43:14.501 # Strict version enforcement is disabled. Loading [ping.so] but undefined behavior may occur. 177 | [81021] 28 Mar 14:43:14.501 * Added command poong [ping.so] 178 | [81021] 28 Mar 14:43:14.501 * Added command pooooong [ping.so] 179 | [81021] 28 Mar 14:43:14.501 * Added command pooong [ping.so] 180 | [81021] 28 Mar 14:43:14.501 * Added command pinger [ping.so] 181 | [81021] 28 Mar 14:43:14.501 * Module [ping.so] loaded 4 commands. 182 | [81021] 28 Mar 14:43:14.501 * Running load function of module [ping.so]. 183 | ``` 184 | 185 | Module reloaded with: `CONFIG SET module-add /Users/matt/repos/krmt/poong.so` 186 | 187 | Sample output on reloading a module (using a different filename, but the same module name): 188 | ```c 189 | [81021] 28 Mar 14:43:29.796 * Running cleanup function for [ping.so] module. 190 | [81021] 28 Mar 14:43:29.796 * Closed previous [ping.so] module. 191 | [81021] 28 Mar 14:43:29.796 * Loading new [/Users/matt/repos/krmt/poong.so] module. 192 | [81021] 28 Mar 14:43:29.796 * Replaced existing command poong [/Users/matt/repos/krmt/poong.so] 193 | [81021] 28 Mar 14:43:29.796 * Replaced existing command pooooong [/Users/matt/repos/krmt/poong.so] 194 | [81021] 28 Mar 14:43:29.796 * Replaced existing command pooong [/Users/matt/repos/krmt/poong.so] 195 | [81021] 28 Mar 14:43:29.796 * Replaced existing command pinger [/Users/matt/repos/krmt/poong.so] 196 | [81021] 28 Mar 14:43:29.796 * Module [/Users/matt/repos/krmt/poong.so] loaded 4 commands. 197 | [81021] 28 Mar 14:43:29.796 * Running load function of module [/Users/matt/repos/krmt/poong.so]. 198 | ``` 199 | 200 | Sample `INFO modules` output: 201 | ```c 202 | module_:filename=,compiled_against=2.8.8-dynamic-0.5, 203 | module_version=2.8.8-dynamic-0.5,first_loaded=0,last_loaded=0,commands=get,set, 204 | setnx,setex,psetex,append,strlen,del,exists,setbit,getbit,setrange,getrange, 205 | substr,incr,decr,mget,rpush,lpush,rpushx,lpushx,linsert,rpop,lpop,brpop, 206 | brpoplpush,blpop,llen,lindex,lset,lrange,ltrim,lrem,rpoplpush,sadd,srem,smove, 207 | sismember,scard,spop,srandmember,sinter,sinterstore,sunion,sunionstore,sdiff, 208 | sdiffstore,smembers,sscan,zadd,zincrby,zrem,zremrangebyscore,zremrangebyrank, 209 | zunionstore,zinterstore,zrange,zrangebyscore,zrevrangebyscore,zcount,zrevrange, 210 | zcard,zscore,zrank,zrevrank,zscan,hset,hsetnx,hget,hmset,hmget,hincrby, 211 | hincrbyfloat,hdel,hlen,hkeys,hvals,hgetall,hexists,hscan,incrby,decrby, 212 | incrbyfloat,getset,mset,msetnx,randomkey,select,move,rename,renamenx,expire, 213 | expireat,pexpire,pexpireat,keys,scan,dbsize,auth,ping,echo,save,bgsave, 214 | bgrewriteaof,shutdown,lastsave,type,multi,exec,discard,sync,psync,replconf, 215 | flushdb,flushall,sort,info,monitor,ttl,pttl,persist,slaveof,debug,config, 216 | subscribe,unsubscribe,psubscribe,punsubscribe,publish,pubsub,watch,unwatch, 217 | restore,migrate,dump,object,client,eval,evalsha,slowlog,script,time,bitop, 218 | bitcount,bitpos 219 | module_sh.matt.test.pong:filename=/Users/matt/repos/krmt/poong.so, 220 | compiled_against=2.8.8-dynamic-0.5,module_version=0.0001, 221 | first_loaded=0,last_loaded=1396032209,commands=poong,pooooong,pooong,pinger 222 | ``` 223 | 224 | Contributions 225 | ------------- 226 | Did you make a nice Redis Add-On Module? Does it have a beginning, 227 | a middle, and an end? Got a protagonist with some obstacles to overcome? 228 | 229 | Just open a pull request against this repository if you want 230 | to share your modules and get them added to `krmt`. 231 | 232 | Modules are more likely to be accepted if they have meaningful comments, 233 | don't leak memory, don't crash Redis, and show us something new and 234 | delightful while solving problems we've had forever (or solving problems 235 | we didn't even know we had). 236 | -------------------------------------------------------------------------------- /bitallpos/bitallpos.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | 3 | /* ==================================================================== 4 | * Redis Add-on Module: bitallpos 5 | * Provides commands: bitallpos 6 | * Behaviors: 7 | * - bitallpos - Return positions of all set bits 8 | * Origin: https://github.com/antirez/redis/pull/1295 9 | * https://github.com/jonahharris/redis/commit/fb813b3196e399ac49e1720c824d6cf37fa6af8f 10 | * ==================================================================== */ 11 | 12 | /* ==================================================================== 13 | * Command Implementations 14 | * ==================================================================== */ 15 | /* BITPOS key [start end limit] */ 16 | static void bitallposCommand(redisClient *c) { 17 | robj *o; 18 | long start, end, limit = -1, strlen; 19 | void *replylen = NULL; 20 | unsigned char *p, byte; 21 | char llbuf[32]; 22 | unsigned long bytecount = 0; 23 | unsigned long bitcount = 0; 24 | 25 | /* Lookup, check for type, and return 0 for non existing keys. */ 26 | if ((o = lookupKeyReadOrReply(c, c->argv[1], shared.emptymultibulk)) == 27 | NULL || 28 | checkType(c, o, REDIS_STRING)) 29 | return; 30 | 31 | /* Set the 'p' pointer to the string, that can be just a stack allocated 32 | * array if our string was integer encoded. */ 33 | if (o->encoding == REDIS_ENCODING_INT) { 34 | p = (unsigned char *)llbuf; 35 | strlen = ll2string(llbuf, sizeof(llbuf), (long)o->ptr); 36 | } else { 37 | p = (unsigned char *)o->ptr; 38 | strlen = sdslen(o->ptr); 39 | } 40 | 41 | /* Parse start/end range if any. */ 42 | if (c->argc == 5) { 43 | if (getLongFromObjectOrReply(c, c->argv[4], &limit, NULL) != REDIS_OK) 44 | return; 45 | } 46 | if (c->argc >= 4) { 47 | if (getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) 48 | return; 49 | if (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK) 50 | return; 51 | } else if (c->argc == 3) { 52 | if (getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) 53 | return; 54 | end = strlen - 1; 55 | } else if (c->argc == 2) { 56 | /* The whole string. */ 57 | start = 0; 58 | end = strlen - 1; 59 | } else { 60 | /* Syntax error. */ 61 | addReply(c, shared.syntaxerr); 62 | return; 63 | } 64 | 65 | /* Convert negative indexes */ 66 | if (start < 0) 67 | start = strlen + start; 68 | if (end < 0) 69 | end = strlen + end; 70 | if (start < 0) 71 | start = 0; 72 | if (end < 0) 73 | end = 0; 74 | if (end >= strlen) 75 | end = strlen - 1; 76 | 77 | /* Precondition: end >= 0 && end < strlen, so the only condition where 78 | * zero can be returned is: start > end. */ 79 | if (start > end) { 80 | addReply(c, shared.emptymultibulk); 81 | return; 82 | } 83 | 84 | p = (p + start); 85 | bytecount = (end - start + 1); 86 | replylen = addDeferredMultiBulkLength(c); 87 | 88 | /* iterate over bytes */ 89 | while (bytecount--) { 90 | unsigned int i = 128, pos = 0; 91 | byte = *p++; 92 | while (byte && limit) { 93 | if (byte & i) { 94 | addReplyBulkLongLong(c, (start * 8 + pos)); 95 | byte &= ~(1 << (7 - pos)); 96 | ++bitcount; 97 | --limit; 98 | limit = (((-1) > (limit)) ? (-1) : (limit)); 99 | } 100 | i >>= 1; 101 | pos++; 102 | } 103 | start++; 104 | } 105 | 106 | setDeferredMultiBulkLength(c, replylen, bitcount); 107 | } 108 | 109 | /* ==================================================================== 110 | * Bring up / Teardown 111 | * ==================================================================== */ 112 | void *load() { 113 | return NULL; 114 | } 115 | 116 | /* If you reload the module *without* freeing things you allocate in load(), 117 | * then you *will* introduce memory leaks. */ 118 | void cleanup(void *privdata) { 119 | return; 120 | } 121 | 122 | /* ==================================================================== 123 | * Dynamic Redis API Requirements 124 | * ==================================================================== */ 125 | struct redisModule redisModuleDetail = { 126 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 127 | REDIS_VERSION, /* Provided by redis.h */ 128 | "0.3", /* Version of this module (only for reporting) */ 129 | "com.github.antirez.redis.pullrequest.1295", /* Unique name of this module 130 | */ 131 | load, /* Load function pointer (optional) */ 132 | cleanup /* Cleanup function pointer (optional) */ 133 | }; 134 | 135 | struct redisCommand redisCommandTable[] = { 136 | {"bitallpos", bitallposCommand, -2, "r", 0, NULL, 1, 1, 1, 0, 0}, 137 | {0} /* Always end your command table with {0} 138 | * If you forget, you will be reminded with a segfault on load. */ 139 | }; 140 | -------------------------------------------------------------------------------- /geo/bench-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Purpose: Quickly get benchmark results for geo commands 4 | # Secondary Purpose: To stress test Redis Geo commands by running 5 | # them continuously under lldb, gdb, or valgrind. 6 | 7 | PATH=~/repos/redis/src:$PATH 8 | LOOPR=10000 9 | BOOST=10 10 | PLINE=32 11 | SOCK=/tmp/redis.sock 12 | 13 | ten() { 14 | CMD=$* 15 | 16 | echo $CMD 17 | (for i in {1..10}; do 18 | $CMD |grep "per second" 19 | done) |sort -n 20 | echo 21 | 22 | } 23 | 24 | # Compare Redis unix domain socket vs. Redis TCP socket speeds 25 | # (Spoiler: Unix sockets are 10% to 35% faster than TCP sockets due to less overhead) 26 | if [[ -f $SOCK ]]; then 27 | ten redis-benchmark -s $SOCK -n $LOOPR geoencode 0 0 28 | ten redis-benchmark -s $SOCK -n $LOOPR geoencode 0 0 withjson 29 | ten redis-benchmark -s $SOCK -n $LOOPR geodecode 3471579339700058 30 | ten redis-benchmark -s $SOCK -n $LOOPR geodecode 3471579339700058 withjson 31 | fi 32 | 33 | # With Pipelining 34 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) geoencode 0 0 35 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) geoencode 0 0 withjson 36 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) geodecode 3471579339700058 37 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) geodecode 3471579339700058 withjson 38 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) georadiusbymember nyc 4545 20 km withdistance 39 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) georadiusbymember nyc 4545 20 km withgeojson 40 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) georadiusbymember nyc 4545 20 km withgeojsoncollection 41 | ten redis-benchmark -P $PLINE -n $((LOOPR * BOOST)) georadiusbymember nyc 4545 20 km withgeojson withgeojsoncollection 42 | 43 | # Without Pipelining 44 | ten redis-benchmark -n $LOOPR geoencode 0 0 45 | ten redis-benchmark -n $LOOPR geoencode 0 0 withjson 46 | ten redis-benchmark -n $LOOPR geodecode 3471579339700058 47 | ten redis-benchmark -n $LOOPR geodecode 3471579339700058 withjson 48 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withdistance 49 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojson 50 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojsoncollection 51 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojson withgeojsoncollection 52 | 53 | #exit 3 54 | while true; do 55 | ten redis-benchmark -n $LOOPR geoencode 40.7480973 -73.9564142 56 | ten redis-benchmark -n $LOOPR geodecode 1791875796750882 57 | ten redis-benchmark -n $LOOPR geoadd nyc 40.7480973 -73.9564142 4545 58 | ten redis-benchmark -n $LOOPR geoadd nyc 40.7648057 -73.9733487 "central park n/q/r" 40.7362513 -73.9903085 "union square" 40.7126674 -74.0131604 "wtc one" 40.6428986 -73.7858139 "jfk" 40.7498929 -73.9375699 "q4" 40.7480973 -73.9564142 4545 59 | ten redis-benchmark -n $LOOPR georadius nyc 40.7480973 -73.9564142 5 mi 60 | ten redis-benchmark -n $LOOPR georadius nyc 40.7480973 -73.9564142 5 mi withgeojson 61 | ten redis-benchmark -n $LOOPR georadius nyc 40.7480973 -73.9564142 5 mi withgeojson withgeojsoncollection 62 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km 63 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withdistance 64 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojson 65 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojson withgeojsoncollection withdistance 66 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojsoncollection withdistance 67 | ten redis-benchmark -n $LOOPR georadiusbymember nyc 4545 20 km withgeojsoncollection 68 | done 69 | -------------------------------------------------------------------------------- /geo/geo.h: -------------------------------------------------------------------------------- 1 | #ifndef __GEO_H__ 2 | #define __GEO_H__ 3 | 4 | #include "redis.h" 5 | 6 | void geoEncodeCommand(redisClient *c); 7 | void geoDecodeCommand(redisClient *c); 8 | void geoRadiusByMemberCommand(redisClient *c); 9 | void geoRadiusCommand(redisClient *c); 10 | void geoAddCommand(redisClient *c); 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /geo/geo.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"geo"}} { 2 | test {GEOADD create} { 3 | r geoadd nyc 40.747533 -73.9454966 "lic market" 4 | } {1} 5 | 6 | test {GEOADD update} { 7 | r geoadd nyc 40.747533 -73.9454966 "lic market" 8 | } {0} 9 | 10 | test {GEOADD multi add} { 11 | r geoadd nyc 40.7648057 -73.9733487 "central park n/q/r" 40.7362513 -73.9903085 "union square" 40.7126674 -74.0131604 "wtc one" 40.6428986 -73.7858139 "jfk" 40.7498929 -73.9375699 "q4" 40.7480973 -73.9564142 4545 12 | } {6} 13 | 14 | test {Check geoset values} { 15 | r zrange nyc 0 -1 withscores 16 | } {{wtc one} 1791873972053020 {union square} 1791875485187452 {central park n/q/r} 1791875761332224 4545 1791875796750882 {lic market} 1791875804419201 q4 1791875830079666 jfk 1791895905559723} 17 | 18 | test {GEORADIUS simple (sorted)} { 19 | r georadius nyc 40.7598464 -73.9798091 3 km ascending 20 | } {{central park n/q/r} 4545 {union square}} 21 | 22 | test {GEORADIUS withdistance (sorted)} { 23 | r georadius nyc 40.7598464 -73.9798091 3 km withdistance ascending 24 | } {{{central park n/q/r} 0.78} {4545 2.37} {{union square} 2.77}} 25 | 26 | test {GEORADIUSBYMEMBER simple (sorted)} { 27 | r georadiusbymember nyc "wtc one" 7 km 28 | } {{wtc one} {union square} {central park n/q/r} 4545 {lic market}} 29 | 30 | test {GEORADIUSBYMEMBER simple (sorted, json)} { 31 | r georadiusbymember nyc "wtc one" 7 km withgeojson 32 | } {{{wtc one} {{"type":"Feature","geometry":{"type":"Point","coordinates":[-74.01316255331,40.712667181451]},"properties":{"distance":0,"member":"wtc one","units":"km","set":"nyc"}}}}\ 33 | {{union square} {{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.990310132504,40.736250227118]},"properties":{"distance":3.2543954573354,"member":"union square","units":"km","set":"nyc"}}}}\ 34 | {{central park n/q/r} {{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.973347842693,40.764806395699]},"properties":{"distance":6.7000029092796,"member":"central park n\/q\/r","units":"km","set":"nyc"}}}}\ 35 | {4545 {{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.956412374973,40.748097513816]},"properties":{"distance":6.1975173818008,"member":"4545","units":"km","set":"nyc"}}}}\ 36 | {{lic market} {{"type":"Feature","geometry":{"type":"Point","coordinates":[-73.945495784283,40.747532270998]},"properties":{"distance":6.8968709532081,"member":"lic market","units":"km","set":"nyc"}}}}} 37 | 38 | test {GEORADIUSBYMEMBER withdistance (sorted)} { 39 | r georadiusbymember nyc "wtc one" 7 km withdist 40 | } {{{wtc one} 0.00} {{union square} 3.25} {{central park n/q/r} 6.70} {4545 6.20} {{lic market} 6.90}} 41 | 42 | test {GEOENCODE simple} { 43 | r geoencode 41.2358883 1.8063239 44 | } {3471579339700058 {41.235888125243704 1.8063229322433472}\ 45 | {41.235890659964866 1.806328296661377}\ 46 | {41.235889392604285 1.8063256144523621}} 47 | 48 | test {GEODECODE simple} { 49 | r geodecode 3471579339700058 50 | } {{41.235888125243704 1.8063229322433472}\ 51 | {41.235890659964866 1.806328296661377}\ 52 | {41.235889392604285 1.8063256144523621}} 53 | } 54 | -------------------------------------------------------------------------------- /geo/geohash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2014, yinqiwen 3 | * Copyright (c) 2014, Matt Stancliff . 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 22 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 | * THE POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | #include "geohash.h" 31 | 32 | /** 33 | * Hashing works like this: 34 | * Divide the world into 4 buckets. Label each one as such: 35 | * ----------------- 36 | * | | | 37 | * | | | 38 | * | 0,1 | 1,1 | 39 | * ----------------- 40 | * | | | 41 | * | | | 42 | * | 0,0 | 1,0 | 43 | * ----------------- 44 | */ 45 | 46 | bool geohashGetCoordRange(uint8_t coord_type, GeoHashRange *lat_range, 47 | GeoHashRange *long_range) { 48 | switch (coord_type) { 49 | case GEO_WGS84_TYPE: { 50 | /* These are constraints from EPSG:900913 / EPSG:3785 / OSGEO:41001 */ 51 | /* We can't geocode at the north/south pole. */ 52 | lat_range->max = 85.05112878; 53 | lat_range->min = -85.05112878; 54 | long_range->max = 180.0; 55 | long_range->min = -180.0; 56 | break; 57 | } 58 | case GEO_MERCATOR_TYPE: { 59 | lat_range->max = 20037726.37; 60 | lat_range->min = -20037726.37; 61 | long_range->max = 20037726.37; 62 | long_range->min = -20037726.37; 63 | break; 64 | } 65 | default: { return false; } 66 | } 67 | return true; 68 | } 69 | 70 | /* Interleave lower bits of x and y, so the bits of x 71 | * are in the even positions and bits from y in the odd; 72 | * x and y must initially be less than 2**32 (65536). 73 | * From: https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN 74 | */ 75 | static inline uint64_t interleave64(uint32_t xlo, uint32_t ylo) { 76 | static const uint64_t B[] = {0x5555555555555555, 0x3333333333333333, 77 | 0x0F0F0F0F0F0F0F0F, 0x00FF00FF00FF00FF, 78 | 0x0000FFFF0000FFFF}; 79 | static const unsigned int S[] = {1, 2, 4, 8, 16}; 80 | 81 | uint64_t x = xlo; 82 | uint64_t y = ylo; 83 | 84 | x = (x | (x << S[4])) & B[4]; 85 | y = (y | (y << S[4])) & B[4]; 86 | 87 | x = (x | (x << S[3])) & B[3]; 88 | y = (y | (y << S[3])) & B[3]; 89 | 90 | x = (x | (x << S[2])) & B[2]; 91 | y = (y | (y << S[2])) & B[2]; 92 | 93 | x = (x | (x << S[1])) & B[1]; 94 | y = (y | (y << S[1])) & B[1]; 95 | 96 | x = (x | (x << S[0])) & B[0]; 97 | y = (y | (y << S[0])) & B[0]; 98 | 99 | return x | (y << 1); 100 | } 101 | 102 | /* reverse the interleave process 103 | * derived from http://stackoverflow.com/questions/4909263 104 | */ 105 | static inline uint64_t deinterleave64(uint64_t interleaved) { 106 | static const uint64_t B[] = {0x5555555555555555, 0x3333333333333333, 107 | 0x0F0F0F0F0F0F0F0F, 0x00FF00FF00FF00FF, 108 | 0x0000FFFF0000FFFF, 0x00000000FFFFFFFF}; 109 | static const unsigned int S[] = {0, 1, 2, 4, 8, 16}; 110 | 111 | uint64_t x = interleaved; 112 | uint64_t y = interleaved >> 1; 113 | 114 | x = (x | (x >> S[0])) & B[0]; 115 | y = (y | (y >> S[0])) & B[0]; 116 | 117 | x = (x | (x >> S[1])) & B[1]; 118 | y = (y | (y >> S[1])) & B[1]; 119 | 120 | x = (x | (x >> S[2])) & B[2]; 121 | y = (y | (y >> S[2])) & B[2]; 122 | 123 | x = (x | (x >> S[3])) & B[3]; 124 | y = (y | (y >> S[3])) & B[3]; 125 | 126 | x = (x | (x >> S[4])) & B[4]; 127 | y = (y | (y >> S[4])) & B[4]; 128 | 129 | x = (x | (x >> S[5])) & B[5]; 130 | y = (y | (y >> S[5])) & B[5]; 131 | 132 | return x | (y << 32); 133 | } 134 | 135 | bool geohashEncode(GeoHashRange lat_range, GeoHashRange long_range, 136 | double latitude, double longitude, uint8_t step, 137 | GeoHashBits *hash) { 138 | if (NULL == hash || step > 32 || step == 0 || RANGEISZERO(lat_range) || 139 | RANGEISZERO(long_range)) { 140 | return false; 141 | } 142 | 143 | hash->bits = 0; 144 | hash->step = step; 145 | 146 | if (latitude < lat_range.min || latitude > lat_range.max || 147 | longitude < long_range.min || longitude > long_range.max) { 148 | return false; 149 | } 150 | 151 | double lat_offset = 152 | (latitude - lat_range.min) / (lat_range.max - lat_range.min); 153 | double long_offset = 154 | (longitude - long_range.min) / (long_range.max - long_range.min); 155 | 156 | /* convert to fixed point based on the step size */ 157 | lat_offset *= (1 << step); 158 | long_offset *= (1 << step); 159 | 160 | uint32_t lat_offset_int = (uint32_t)lat_offset; 161 | uint32_t long_offset_int = (uint32_t)long_offset; 162 | 163 | hash->bits = interleave64(lat_offset_int, long_offset_int); 164 | return true; 165 | } 166 | 167 | bool geohashEncodeType(uint8_t coord_type, double latitude, double longitude, 168 | uint8_t step, GeoHashBits *hash) { 169 | GeoHashRange r[2] = {{0}}; 170 | geohashGetCoordRange(coord_type, &r[0], &r[1]); 171 | return geohashEncode(r[0], r[1], latitude, longitude, step, hash); 172 | } 173 | 174 | bool geohashEncodeWGS84(double latitude, double longitude, uint8_t step, 175 | GeoHashBits *hash) { 176 | return geohashEncodeType(GEO_WGS84_TYPE, latitude, longitude, step, hash); 177 | } 178 | 179 | bool geohashEncodeMercator(double latitude, double longitude, uint8_t step, 180 | GeoHashBits *hash) { 181 | return geohashEncodeType(GEO_MERCATOR_TYPE, latitude, longitude, step, 182 | hash); 183 | } 184 | 185 | bool geohashDecode(const GeoHashRange lat_range, const GeoHashRange long_range, 186 | const GeoHashBits hash, GeoHashArea *area) { 187 | if (HASHISZERO(hash) || NULL == area || RANGEISZERO(lat_range) || 188 | RANGEISZERO(long_range)) { 189 | return false; 190 | } 191 | 192 | area->hash = hash; 193 | uint8_t step = hash.step; 194 | uint64_t hash_sep = deinterleave64(hash.bits); /* hash = [LAT][LONG] */ 195 | 196 | double lat_scale = lat_range.max - lat_range.min; 197 | double long_scale = long_range.max - long_range.min; 198 | 199 | uint32_t ilato = hash_sep; /* get lat part of deinterleaved hash */ 200 | uint32_t ilono = hash_sep >> 32; /* shift over to get long part of hash */ 201 | 202 | /* divide by 2**step. 203 | * Then, for 0-1 coordinate, multiply times scale and add 204 | to the min to get the absolute coordinate. */ 205 | area->latitude.min = 206 | lat_range.min + (ilato * 1.0 / (1ull << step)) * lat_scale; 207 | area->latitude.max = 208 | lat_range.min + ((ilato + 1) * 1.0 / (1ull << step)) * lat_scale; 209 | area->longitude.min = 210 | long_range.min + (ilono * 1.0 / (1ull << step)) * long_scale; 211 | area->longitude.max = 212 | long_range.min + ((ilono + 1) * 1.0 / (1ull << step)) * long_scale; 213 | 214 | return true; 215 | } 216 | 217 | bool geohashDecodeType(uint8_t coord_type, const GeoHashBits hash, 218 | GeoHashArea *area) { 219 | GeoHashRange r[2] = {{0}}; 220 | geohashGetCoordRange(coord_type, &r[0], &r[1]); 221 | return geohashDecode(r[0], r[1], hash, area); 222 | } 223 | 224 | bool geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) { 225 | return geohashDecodeType(GEO_WGS84_TYPE, hash, area); 226 | } 227 | 228 | bool geohashDecodeMercator(const GeoHashBits hash, GeoHashArea *area) { 229 | return geohashDecodeType(GEO_MERCATOR_TYPE, hash, area); 230 | } 231 | 232 | bool geohashDecodeAreaToLatLong(const GeoHashArea *area, double *latlong) { 233 | double y, x; 234 | 235 | if (!latlong) 236 | return false; 237 | 238 | y = (area->latitude.min + area->latitude.max) / 2; 239 | x = (area->longitude.min + area->longitude.max) / 2; 240 | 241 | latlong[0] = y; 242 | latlong[1] = x; 243 | return true; 244 | } 245 | 246 | bool geohashDecodeToLatLongType(uint8_t coord_type, const GeoHashBits hash, 247 | double *latlong) { 248 | GeoHashArea area = {{0}}; 249 | if (!latlong || !geohashDecodeType(coord_type, hash, &area)) 250 | return false; 251 | return geohashDecodeAreaToLatLong(&area, latlong); 252 | } 253 | 254 | bool geohashDecodeToLatLongWGS84(const GeoHashBits hash, double *latlong) { 255 | return geohashDecodeToLatLongType(GEO_WGS84_TYPE, hash, latlong); 256 | } 257 | 258 | bool geohashDecodeToLatLongMercator(const GeoHashBits hash, double *latlong) { 259 | return geohashDecodeToLatLongType(GEO_MERCATOR_TYPE, hash, latlong); 260 | } 261 | 262 | static void geohash_move_x(GeoHashBits *hash, int8_t d) { 263 | if (d == 0) 264 | return; 265 | 266 | uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaLL; 267 | uint64_t y = hash->bits & 0x5555555555555555LL; 268 | 269 | uint64_t zz = 0x5555555555555555LL >> (64 - hash->step * 2); 270 | 271 | if (d > 0) { 272 | x = x + (zz + 1); 273 | } else { 274 | x = x | zz; 275 | x = x - (zz + 1); 276 | } 277 | 278 | x &= (0xaaaaaaaaaaaaaaaaLL >> (64 - hash->step * 2)); 279 | hash->bits = (x | y); 280 | } 281 | 282 | static void geohash_move_y(GeoHashBits *hash, int8_t d) { 283 | if (d == 0) 284 | return; 285 | 286 | uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaLL; 287 | uint64_t y = hash->bits & 0x5555555555555555LL; 288 | 289 | uint64_t zz = 0xaaaaaaaaaaaaaaaaLL >> (64 - hash->step * 2); 290 | if (d > 0) { 291 | y = y + (zz + 1); 292 | } else { 293 | y = y | zz; 294 | y = y - (zz + 1); 295 | } 296 | y &= (0x5555555555555555LL >> (64 - hash->step * 2)); 297 | hash->bits = (x | y); 298 | } 299 | 300 | void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors) { 301 | neighbors->east = *hash; 302 | neighbors->west = *hash; 303 | neighbors->north = *hash; 304 | neighbors->south = *hash; 305 | neighbors->south_east = *hash; 306 | neighbors->south_west = *hash; 307 | neighbors->north_east = *hash; 308 | neighbors->north_west = *hash; 309 | 310 | geohash_move_x(&neighbors->east, 1); 311 | geohash_move_y(&neighbors->east, 0); 312 | 313 | geohash_move_x(&neighbors->west, -1); 314 | geohash_move_y(&neighbors->west, 0); 315 | 316 | geohash_move_x(&neighbors->south, 0); 317 | geohash_move_y(&neighbors->south, -1); 318 | 319 | geohash_move_x(&neighbors->north, 0); 320 | geohash_move_y(&neighbors->north, 1); 321 | 322 | geohash_move_x(&neighbors->north_west, -1); 323 | geohash_move_y(&neighbors->north_west, 1); 324 | 325 | geohash_move_x(&neighbors->north_east, 1); 326 | geohash_move_y(&neighbors->north_east, 1); 327 | 328 | geohash_move_x(&neighbors->south_east, 1); 329 | geohash_move_y(&neighbors->south_east, -1); 330 | 331 | geohash_move_x(&neighbors->south_west, -1); 332 | geohash_move_y(&neighbors->south_west, -1); 333 | } 334 | -------------------------------------------------------------------------------- /geo/geohash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2014, yinqiwen 3 | * Copyright (c) 2014, Matt Stancliff . 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 22 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 | * THE POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef GEOHASH_H_ 32 | #define GEOHASH_H_ 33 | 34 | #include 35 | #include 36 | #include 37 | 38 | #if defined(__cplusplus) 39 | extern "C" { 40 | #endif 41 | 42 | #define HASHISZERO(r) (!(r).bits && !(r).step) 43 | #define RANGEISZERO(r) (!(r).max && !(r).min) 44 | 45 | #define GEO_WGS84_TYPE 1 46 | #define GEO_MERCATOR_TYPE 2 47 | 48 | #define GEO_STEP_MAX 26 49 | 50 | typedef enum { 51 | GEOHASH_NORTH = 0, 52 | GEOHASH_EAST, 53 | GEOHASH_WEST, 54 | GEOHASH_SOUTH, 55 | GEOHASH_SOUTH_WEST, 56 | GEOHASH_SOUTH_EAST, 57 | GEOHASH_NORT_WEST, 58 | GEOHASH_NORT_EAST 59 | } GeoDirection; 60 | 61 | typedef struct { 62 | uint64_t bits; 63 | uint8_t step; 64 | } GeoHashBits; 65 | 66 | typedef struct { 67 | double max; 68 | double min; 69 | } GeoHashRange; 70 | 71 | typedef struct { 72 | GeoHashBits hash; 73 | GeoHashRange latitude; 74 | GeoHashRange longitude; 75 | } GeoHashArea; 76 | 77 | typedef struct { 78 | GeoHashBits north; 79 | GeoHashBits east; 80 | GeoHashBits west; 81 | GeoHashBits south; 82 | GeoHashBits north_east; 83 | GeoHashBits south_east; 84 | GeoHashBits north_west; 85 | GeoHashBits south_west; 86 | } GeoHashNeighbors; 87 | 88 | /* 89 | * 0:success 90 | * -1:failed 91 | */ 92 | bool geohashGetCoordRange(uint8_t coord_type, GeoHashRange *lat_range, 93 | GeoHashRange *long_range); 94 | bool geohashEncode(GeoHashRange lat_range, GeoHashRange long_range, 95 | double latitude, double longitude, uint8_t step, 96 | GeoHashBits *hash); 97 | bool geohashEncodeType(uint8_t coord_type, double latitude, double longitude, 98 | uint8_t step, GeoHashBits *hash); 99 | bool geohashEncodeMercator(double latitude, double longitude, uint8_t step, 100 | GeoHashBits *hash); 101 | bool geohashEncodeWGS84(double latitude, double longitude, uint8_t step, 102 | GeoHashBits *hash); 103 | bool geohashDecode(const GeoHashRange lat_range, const GeoHashRange long_range, 104 | const GeoHashBits hash, GeoHashArea *area); 105 | bool geohashDecodeType(uint8_t coord_type, const GeoHashBits hash, 106 | GeoHashArea *area); 107 | bool geohashDecodeMercator(const GeoHashBits hash, GeoHashArea *area); 108 | bool geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area); 109 | bool geohashDecodeAreaToLatLong(const GeoHashArea *area, double *latlong); 110 | bool geohashDecodeToLatLongType(uint8_t coord_type, const GeoHashBits hash, 111 | double *latlong); 112 | bool geohashDecodeToLatLongWGS84(const GeoHashBits hash, double *latlong); 113 | bool geohashDecodeToLatLongMercator(const GeoHashBits hash, double *latlong); 114 | void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors); 115 | 116 | #if defined(__cplusplus) 117 | } 118 | #endif 119 | #endif /* GEOHASH_H_ */ 120 | -------------------------------------------------------------------------------- /geo/geohash_helper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2014, yinqiwen 3 | * Copyright (c) 2014, Matt Stancliff . 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 22 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 | * THE POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | /* This is a C++ to C conversion from the ardb project. 32 | * This file started out as: 33 | * https://github.com/yinqiwen/ardb/blob/d42503/src/geo/geohash_helper.cpp 34 | */ 35 | 36 | #include "geohash_helper.h" 37 | 38 | #ifndef M_PI 39 | #define M_PI 3.14159265358979323846264338327950288 40 | #endif 41 | 42 | #ifndef M_PI_2 43 | #define M_PI_2 1.57079632679489661923132169163975144 44 | #endif 45 | 46 | #define D_R (M_PI / 180.0) 47 | #define R_MAJOR 6378137.0 48 | #define R_MINOR 6356752.3142 49 | #define RATIO (R_MINOR / R_MAJOR) 50 | #define ECCENT (sqrt(1.0 - (RATIO * RATIO))) 51 | #define COM (0.5 * ECCENT) 52 | 53 | /// @brief The usual PI/180 constant 54 | const double DEG_TO_RAD = 0.017453292519943295769236907684886; 55 | /// @brief Earth's quatratic mean radius for WGS-84 56 | const double EARTH_RADIUS_IN_METERS = 6372797.560856; 57 | 58 | const double MERCATOR_MAX = 20037726.37; 59 | const double MERCATOR_MIN = -20037726.37; 60 | 61 | static inline double deg_rad(double ang) { 62 | return ang * D_R; 63 | } 64 | static inline double rad_deg(double ang) { 65 | return ang / D_R; 66 | } 67 | 68 | double mercator_y(double lat) { 69 | lat = fmin(89.5, fmax(lat, -89.5)); 70 | double phi = deg_rad(lat); 71 | double sinphi = sin(phi); 72 | double con = ECCENT * sinphi; 73 | con = pow((1.0 - con) / (1.0 + con), COM); 74 | double ts = tan(0.5 * (M_PI * 0.5 - phi)) / con; 75 | return 0 - R_MAJOR * log(ts); 76 | } 77 | 78 | double mercator_x(double lon) { 79 | return R_MAJOR * deg_rad(lon); 80 | } 81 | double merc_lon(double x) { 82 | return rad_deg(x) / R_MAJOR; 83 | } 84 | 85 | double merc_lat(double y) { 86 | double ts = exp(-y / R_MAJOR); 87 | double phi = M_PI_2 - 2 * atan(ts); 88 | double dphi = 1.0; 89 | int i; 90 | for (i = 0; fabs(dphi) > 0.000000001 && i < 15; i++) { 91 | double con = ECCENT * sin(phi); 92 | dphi = 93 | M_PI_2 - 2 * atan(ts * pow((1.0 - con) / (1.0 + con), COM)) - phi; 94 | phi += dphi; 95 | } 96 | return rad_deg(phi); 97 | } 98 | 99 | /* You must *ONLY* estimate steps when you are encoding. 100 | * If you are decoding, always decode to GEO_STEP_MAX (26). */ 101 | uint8_t geohashEstimateStepsByRadius(double range_meters) { 102 | uint8_t step = 1; 103 | while (range_meters > 0 && range_meters < MERCATOR_MAX) { 104 | range_meters *= 2; 105 | step++; 106 | } 107 | step--; 108 | if (!step) 109 | step = 26; /* if range = 0, give max resolution */ 110 | return step > 26 ? 26 : step; 111 | } 112 | 113 | double geohashGetXWGS84(double x) { 114 | return merc_lon(x); 115 | } 116 | double geohashGetYWGS84(double y) { 117 | return merc_lat(y); 118 | } 119 | 120 | double geohashGetXMercator(double longtitude) { 121 | if (longtitude > 180 || longtitude < -180) { 122 | return longtitude; 123 | } 124 | return mercator_x(longtitude); 125 | } 126 | double geohashGetYMercator(double latitude) { 127 | if (latitude > 90 || latitude < -90) { 128 | return latitude; 129 | } 130 | return mercator_y(latitude); 131 | } 132 | 133 | int geohashBitsComparator(const GeoHashBits *a, const GeoHashBits *b) { 134 | /* If step not equal, compare on step. Else, compare on bits. */ 135 | return a->step != b->step ? a->step - b->step : a->bits - b->bits; 136 | } 137 | 138 | bool geohashBoundingBox(double latitude, double longitude, double radius_meters, 139 | double *bounds) { 140 | if (!bounds) 141 | return false; 142 | 143 | double latr, lonr; 144 | latr = deg_rad(latitude); 145 | lonr = deg_rad(longitude); 146 | 147 | double distance = radius_meters / EARTH_RADIUS_IN_METERS; 148 | double min_latitude = latr - distance; 149 | double max_latitude = latr + distance; 150 | 151 | /* Note: we're being lazy and not accounting for coordinates near poles */ 152 | double min_longitude, max_longitude; 153 | double difference_longitude = asin(sin(distance) / cos(latr)); 154 | min_longitude = lonr - difference_longitude; 155 | max_longitude = lonr + difference_longitude; 156 | 157 | bounds[0] = rad_deg(min_latitude); 158 | bounds[1] = rad_deg(min_longitude); 159 | bounds[2] = rad_deg(max_latitude); 160 | bounds[3] = rad_deg(max_longitude); 161 | 162 | return true; 163 | } 164 | 165 | GeoHashRadius geohashGetAreasByRadius(uint8_t coord_type, double latitude, 166 | double longitude, double radius_meters) { 167 | GeoHashRange lat_range, long_range; 168 | GeoHashRadius radius = {{0}}; 169 | GeoHashBits hash = {0}; 170 | GeoHashNeighbors neighbors = {{0}}; 171 | GeoHashArea area = {{0}}; 172 | double delta_longitude, delta_latitude; 173 | double min_lat, max_lat, min_lon, max_lon; 174 | int steps; 175 | 176 | if (coord_type == GEO_WGS84_TYPE) { 177 | double bounds[4]; 178 | geohashBoundingBox(latitude, longitude, radius_meters, bounds); 179 | min_lat = bounds[0]; 180 | min_lon = bounds[1]; 181 | max_lat = bounds[2]; 182 | max_lon = bounds[3]; 183 | } else { 184 | delta_latitude = delta_longitude = radius_meters; 185 | min_lat = latitude - delta_latitude; 186 | max_lat = latitude + delta_latitude; 187 | min_lon = longitude - delta_longitude; 188 | max_lon = longitude + delta_longitude; 189 | } 190 | 191 | steps = geohashEstimateStepsByRadius(radius_meters); 192 | 193 | geohashGetCoordRange(coord_type, &lat_range, &long_range); 194 | geohashEncode(lat_range, long_range, latitude, longitude, steps, &hash); 195 | geohashNeighbors(&hash, &neighbors); 196 | geohashDecode(lat_range, long_range, hash, &area); 197 | 198 | if (area.latitude.min < min_lat) { 199 | GZERO(neighbors.south); 200 | GZERO(neighbors.south_west); 201 | GZERO(neighbors.south_east); 202 | } 203 | if (area.latitude.max > max_lat) { 204 | GZERO(neighbors.north); 205 | GZERO(neighbors.north_east); 206 | GZERO(neighbors.north_west); 207 | } 208 | if (area.longitude.min < min_lon) { 209 | GZERO(neighbors.west); 210 | GZERO(neighbors.south_west); 211 | GZERO(neighbors.north_west); 212 | } 213 | if (area.longitude.max > max_lon) { 214 | GZERO(neighbors.east); 215 | GZERO(neighbors.south_east); 216 | GZERO(neighbors.north_east); 217 | } 218 | radius.hash = hash; 219 | radius.neighbors = neighbors; 220 | radius.area = area; 221 | return radius; 222 | } 223 | 224 | GeoHashRadius geohashGetAreasByRadiusWGS84(double latitude, double longitude, 225 | double radius_meters) { 226 | return geohashGetAreasByRadius(GEO_WGS84_TYPE, latitude, longitude, 227 | radius_meters); 228 | } 229 | 230 | GeoHashRadius geohashGetAreasByRadiusMercator(double latitude, double longitude, 231 | double radius_meters) { 232 | return geohashGetAreasByRadius(GEO_MERCATOR_TYPE, latitude, longitude, 233 | radius_meters); 234 | } 235 | 236 | GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash) { 237 | uint64_t bits = hash.bits; 238 | bits <<= (52 - hash.step * 2); 239 | return bits; 240 | } 241 | 242 | /* calculate distance using haversin great circle distance formula */ 243 | double distanceEarth(double lat1d, double lon1d, double lat2d, double lon2d) { 244 | double lat1r, lon1r, lat2r, lon2r, u, v; 245 | lat1r = deg_rad(lat1d); 246 | lon1r = deg_rad(lon1d); 247 | lat2r = deg_rad(lat2d); 248 | lon2r = deg_rad(lon2d); 249 | u = sin((lat2r - lat1r) / 2); 250 | v = sin((lon2r - lon1r) / 2); 251 | return 2.0 * EARTH_RADIUS_IN_METERS * 252 | asin(sqrt(u * u + cos(lat1r) * cos(lat2r) * v * v)); 253 | } 254 | 255 | bool geohashGetDistanceIfInRadius(uint8_t coord_type, double x1, double y1, 256 | double x2, double y2, double radius, 257 | double *distance) { 258 | if (coord_type == GEO_WGS84_TYPE) { 259 | *distance = distanceEarth(y1, x1, y2, x2); 260 | if (*distance > radius) { 261 | return false; 262 | } 263 | } else { 264 | double xx = (x1 - x2) * (x1 - x2); 265 | double yy = (y1 - y2) * (y1 - y2); 266 | double dd = xx + yy; 267 | *distance = dd; 268 | if (dd > (radius * radius)) { 269 | return false; 270 | } 271 | } 272 | return true; 273 | } 274 | 275 | bool geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2, 276 | double y2, double radius, 277 | double *distance) { 278 | return geohashGetDistanceIfInRadius(GEO_WGS84_TYPE, x1, y1, x2, y2, radius, 279 | distance); 280 | } 281 | 282 | bool geohashGetDistanceSquaredIfInRadiusMercator(double x1, double y1, 283 | double x2, double y2, 284 | double radius, 285 | double *distance) { 286 | return geohashGetDistanceIfInRadius(GEO_MERCATOR_TYPE, x1, y1, x2, y2, 287 | radius, distance); 288 | } 289 | 290 | bool geohashVerifyCoordinates(uint8_t coord_type, double x, double y) { 291 | GeoHashRange lat_range, long_range; 292 | geohashGetCoordRange(coord_type, &lat_range, &long_range); 293 | 294 | if (x < long_range.min || x > long_range.max || y < lat_range.min || 295 | y > lat_range.max) { 296 | return false; 297 | } 298 | return true; 299 | } 300 | -------------------------------------------------------------------------------- /geo/geohash_helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2014, yinqiwen 3 | * Copyright (c) 2014, Matt Stancliff . 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 22 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 | * THE POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef GEOHASH_HELPER_HPP_ 32 | #define GEOHASH_HELPER_HPP_ 33 | 34 | #include 35 | #include 36 | #include "geohash.h" 37 | 38 | #define GZERO(s) s.bits = s.step = 0; 39 | #define GISZERO(s) (!s.bits && !s.step) 40 | #define GISNOTZERO(s) (s.bits || s.step) 41 | 42 | typedef uint64_t GeoHashFix52Bits; 43 | typedef uint64_t GeoHashVarBits; 44 | 45 | typedef struct { 46 | GeoHashBits hash; 47 | GeoHashArea area; 48 | GeoHashNeighbors neighbors; 49 | } GeoHashRadius; 50 | 51 | int GeoHashBitsComparator(const GeoHashBits *a, const GeoHashBits *b); 52 | uint8_t geohashEstimateStepsByRadius(double range_meters); 53 | bool geohashBoundingBox(double latitude, double longitude, double radius_meters, 54 | double *bounds); 55 | GeoHashRadius geohashGetAreasByRadius(uint8_t coord_type, double latitude, 56 | double longitude, double radius_meters); 57 | GeoHashRadius geohashGetAreasByRadiusWGS84(double latitude, double longitude, 58 | double radius_meters); 59 | GeoHashRadius geohashGetAreasByRadiusMercator(double latitude, double longitude, 60 | double radius_meters); 61 | GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash); 62 | double geohashGetXMercator(double longtitude); 63 | double geohashGetYMercator(double latitude); 64 | double geohashGetXWGS84(double x); 65 | double geohashGetYWGS84(double y); 66 | bool geohashVerifyCoordinates(uint8_t coord_type, double x, double y); 67 | bool geohashGetDistanceIfInRadius(uint8_t coord_type, double x1, double y1, 68 | double x2, double y2, double radius, 69 | double *distance); 70 | bool geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2, 71 | double y2, double radius, 72 | double *distance); 73 | bool geohashGetDistanceSquaredIfInRadiusMercator(double x1, double y1, 74 | double x2, double y2, 75 | double radius, 76 | double *distance); 77 | 78 | #endif /* GEOHASH_HELPER_HPP_ */ 79 | -------------------------------------------------------------------------------- /geo/geojson.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Matt Stancliff . 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "geojson.h" 31 | 32 | #define L server.lua 33 | 34 | /* ==================================================================== 35 | * The Encoder 36 | * ==================================================================== */ 37 | static sds jsonEncode() { 38 | /* When entering this function, stack is: [1:[geojson table to encode]] */ 39 | lua_getglobal(L, "cjson"); 40 | lua_getfield(L, -1, "encode"); 41 | 42 | /* Stack is now: [1:[geojson table], 2:'cjson', 3:'encode'] */ 43 | 44 | /* Move current top ('encode') to bottom of stack */ 45 | lua_insert(L, 1); 46 | 47 | /* Move current top ('cjson') to bottom of stack so we can 'cjson.encode' */ 48 | lua_insert(L, 1); 49 | 50 | /* Stack is now: [1:'cjson', 2:'encode', 3:[table of geojson to encode]] */ 51 | 52 | /* Call cjson.encode on the element above it on the stack; 53 | * obtain one return value */ 54 | if (lua_pcall(L, 1, 1, 0) != 0) 55 | redisLog(REDIS_WARNING, "Could not encode geojson: %s", 56 | lua_tostring(L, -1)); 57 | 58 | sds geojson = sdsnew(lua_tostring(L, -1)); 59 | 60 | /* We're done. Remove entire stack. Drop mic. Walk away. */ 61 | lua_pop(L, lua_gettop(L)); 62 | 63 | /* Return sds the caller must sdsfree() on their own */ 64 | return geojson; 65 | } 66 | 67 | /* ==================================================================== 68 | * The Lua Helpers 69 | * ==================================================================== */ 70 | static inline void luaCreateFieldFromPrevious(const char *field) { 71 | lua_setfield(L, -2, field); 72 | } 73 | 74 | static inline void luaCreateFieldStr(const char *field, const char *value) { 75 | lua_pushstring(L, value); 76 | luaCreateFieldFromPrevious(field); 77 | } 78 | 79 | /* Creates [Lat, Long] array attached to "coordinates" key */ 80 | static void luaCreateCoordinates(const double x, const double y) { 81 | /* Create array table with two elements */ 82 | lua_createtable(L, 2, 0); 83 | 84 | lua_pushnumber(L, x); 85 | lua_rawseti(L, -2, 1); 86 | lua_pushnumber(L, y); 87 | lua_rawseti(L, -2, 2); 88 | } 89 | 90 | static void luaCreatePropertyNull(void) { 91 | /* Create empty table and give it a name. This is a json {} value. */ 92 | lua_createtable(L, 0, 0); 93 | luaCreateFieldFromPrevious("properties"); 94 | } 95 | 96 | static void _luaCreateProperties(const char *k1, const char *v1, const char *k2, 97 | const char *v2, const int noclose) { 98 | /* we may add additional properties outside of here, so newtable instead of 99 | * fixed-size createtable */ 100 | lua_newtable(L); 101 | 102 | luaCreateFieldStr(k1, v1); 103 | luaCreateFieldStr(k2, v2); 104 | 105 | if (!noclose) 106 | luaCreateFieldFromPrevious("properties"); 107 | } 108 | 109 | static void luaCreateProperties(const char *k1, const char *v1, const char *k2, 110 | const char *v2) { 111 | _luaCreateProperties(k1, v1, k2, v2, 0); 112 | } 113 | 114 | /* ==================================================================== 115 | * The Lua Aggregation Helpers 116 | * ==================================================================== */ 117 | static void attachProperties(const char *set, const char *member) { 118 | if (member) 119 | luaCreateProperties("set", set, "member", member); 120 | else 121 | luaCreatePropertyNull(); 122 | } 123 | 124 | static void attachPropertiesWithDist(const char *set, const char *member, 125 | double dist, const char *units) { 126 | if (member) { 127 | _luaCreateProperties("set", set, "member", member, 1); 128 | if (units) { 129 | /* Add units then distance. After encoding it comes 130 | * out as distance followed by units in the json. */ 131 | lua_pushstring(L, units); 132 | luaCreateFieldFromPrevious("units"); 133 | lua_pushnumber(L, dist); 134 | luaCreateFieldFromPrevious("distance"); 135 | } 136 | 137 | /* We requested to leave the properties table open, but now we 138 | * are done and can close it. */ 139 | luaCreateFieldFromPrevious("properties"); 140 | } else { 141 | luaCreatePropertyNull(); 142 | } 143 | } 144 | 145 | static void createGeometryPoint(const double x, const double y) { 146 | lua_createtable(L, 0, 2); 147 | 148 | /* coordinates = [x, y] */ 149 | luaCreateCoordinates(x, y); 150 | luaCreateFieldFromPrevious("coordinates"); 151 | 152 | /* type = Point */ 153 | luaCreateFieldStr("type", "Point"); 154 | 155 | /* geometry = (coordinates = [x, y]) */ 156 | luaCreateFieldFromPrevious("geometry"); 157 | } 158 | 159 | static void createGeometryBox(const double x1, const double y1, const double x2, 160 | const double y2) { 161 | lua_createtable(L, 0, 2); 162 | 163 | /* Result = [[[x1,y1],[x2,y1],[x2,y2],[x1,y2], [x1,y1]] */ 164 | /* The end coord is the start coord to make a closed polygon */ 165 | lua_createtable(L, 1, 0); 166 | lua_createtable(L, 5, 0); 167 | 168 | /* Bottom left */ 169 | luaCreateCoordinates(x1, y1); 170 | lua_rawseti(L, -2, 1); 171 | 172 | /* Top Left */ 173 | luaCreateCoordinates(x2, y1); 174 | lua_rawseti(L, -2, 2); 175 | 176 | /* Top Right */ 177 | luaCreateCoordinates(x2, y2); 178 | lua_rawseti(L, -2, 3); 179 | 180 | /* Bottom Right */ 181 | luaCreateCoordinates(x1, y2); 182 | lua_rawseti(L, -2, 4); 183 | 184 | /* Bottom Left (Again) */ 185 | luaCreateCoordinates(x1, y1); 186 | lua_rawseti(L, -2, 5); 187 | 188 | /* Set the outer array of our inner array of the inner coords */ 189 | lua_rawseti(L, -2, 1); 190 | 191 | /* Bundle those together in coordinates: [a, b, c, d] */ 192 | luaCreateFieldFromPrevious("coordinates"); 193 | 194 | /* Add type field */ 195 | luaCreateFieldStr("type", "Polygon"); 196 | 197 | luaCreateFieldFromPrevious("geometry"); 198 | } 199 | 200 | static void createFeature() { 201 | /* Features have three fields: type, geometry, and properties */ 202 | lua_createtable(L, 0, 3); 203 | 204 | luaCreateFieldStr("type", "Feature"); 205 | 206 | /* You must call attachProperties on your own */ 207 | } 208 | 209 | static void createCollection(size_t size) { 210 | /* FeatureCollections have two fields: type and features */ 211 | lua_createtable(L, 0, 2); 212 | 213 | luaCreateFieldStr("type", "FeatureCollection"); 214 | } 215 | 216 | static void pointsToCollection(const struct geojsonPoint *pts, const size_t len, 217 | const char *units) { 218 | createCollection(len); 219 | 220 | lua_createtable(L, len, 0); 221 | for (int i = 0; i < len; i++) { 222 | createFeature(); 223 | createGeometryPoint(pts[i].longitude, pts[i].latitude); /* x, y */ 224 | attachPropertiesWithDist(pts[i].set, pts[i].member, pts[i].dist, units); 225 | lua_rawseti(L, -2, i + 1); /* Attach this Feature to "features" array */ 226 | } 227 | luaCreateFieldFromPrevious("features"); 228 | } 229 | 230 | static void latLongToPointFeature(const double latitude, 231 | const double longitude) { 232 | createFeature(); 233 | createGeometryPoint(longitude, latitude); /* geojson is: x,y */ 234 | } 235 | 236 | static void squareToPolygonFeature(const double x1, const double y1, 237 | const double x2, const double y2) { 238 | createFeature(); 239 | createGeometryBox(x1, y1, x2, y2); 240 | } 241 | 242 | /* ==================================================================== 243 | * The Interface Functions 244 | * ==================================================================== */ 245 | sds geojsonFeatureCollection(const struct geojsonPoint *pts, const size_t len, 246 | const char *units) { 247 | pointsToCollection(pts, len, units); 248 | return jsonEncode(); 249 | } 250 | 251 | sds geojsonLatLongToPointFeature(const double latitude, const double longitude, 252 | const char *set, const char *member, 253 | const double dist, const char *units) { 254 | latLongToPointFeature(latitude, longitude); 255 | attachPropertiesWithDist(set, member, dist, units); 256 | return jsonEncode(); 257 | } 258 | 259 | sds geojsonBoxToPolygonFeature(const double y1, const double x1, 260 | const double y2, const double x2, 261 | const char *set, const char *member) { 262 | squareToPolygonFeature(x1, y1, x2, y2); 263 | attachProperties(set, member); 264 | return jsonEncode(); 265 | } 266 | -------------------------------------------------------------------------------- /geo/geojson.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014, Matt Stancliff . 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef __GEOJSON_H__ 31 | #define __GEOJSON_H__ 32 | 33 | #include "redis.h" 34 | #include "geohash_helper.h" 35 | 36 | struct geojsonPoint { 37 | double latitude; 38 | double longitude; 39 | double dist; 40 | char *set; 41 | char *member; 42 | void *userdata; 43 | }; 44 | 45 | sds geojsonLatLongToPointFeature(const double latitude, const double longitude, 46 | const char *set, const char *member, 47 | const double dist, const char *units); 48 | sds geojsonBoxToPolygonFeature(const double x1, const double y1, 49 | const double x2, const double y2, 50 | const char *set, const char *member); 51 | sds geojsonFeatureCollection(const struct geojsonPoint *pts, const size_t len, 52 | const char *units); 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /geo/module.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | #include "geohash_helper.h" 3 | #include "geojson.h" 4 | #include "geo.h" 5 | #include "zset.h" 6 | 7 | /* ==================================================================== 8 | * Bring up / Teardown 9 | * ==================================================================== */ 10 | void *load() { 11 | return NULL; 12 | } 13 | 14 | /* If you reload the module *without* freeing things you allocate in load(), 15 | * then you *will* introduce memory leaks. */ 16 | void cleanup(void *privdata) { 17 | } 18 | 19 | /* ==================================================================== 20 | * Dynamic Redis API Requirements 21 | * ==================================================================== */ 22 | struct redisModule redisModuleDetail = { 23 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 24 | REDIS_VERSION, /* Provided by redis.h */ 25 | "0.3", /* Version of this module (only for reporting) */ 26 | "sh.matt.geo", /* Unique name of this module */ 27 | load, /* Load function pointer (optional) */ 28 | cleanup /* Cleanup function pointer (optional) */ 29 | }; 30 | 31 | struct redisCommand redisCommandTable[] = { 32 | {"geoadd", geoAddCommand, -5, "wm", 0, NULL, 1, 1, 1, 0, 0}, 33 | {"georadius", geoRadiusCommand, -6, "r", 0, NULL, 1, 1, 1, 0, 0}, 34 | {"georadiusbymember", geoRadiusByMemberCommand, -5, "r", 0, NULL, 1, 1, 1, 35 | 0, 0}, 36 | {"geoencode", geoEncodeCommand, -3, "r", 0, NULL, 0, 0, 0, 0, 0}, 37 | {"geodecode", geoDecodeCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 38 | {0} /* Always end your command table with {0} 39 | * If you forget, you will be reminded with a segfault on load. */ 40 | }; 41 | -------------------------------------------------------------------------------- /geo/test.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE /* provides getline() on Linux */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "geohash.h" 12 | 13 | static long long ustime(void) { 14 | struct timeval tv; 15 | long long ust; 16 | 17 | gettimeofday(&tv, NULL); 18 | ust = ((long long)tv.tv_sec) * 1e6; 19 | ust += tv.tv_usec; 20 | return ust; 21 | } 22 | 23 | #define TOTAL 200000 24 | int main(int argc, char *argv[]) { 25 | double latlong[TOTAL * 2] = {0}; 26 | 27 | FILE *fp; 28 | size_t len = 4096; 29 | char *line = malloc(len); 30 | ssize_t read; 31 | bool use_file = false; 32 | 33 | if (argc == 2) { 34 | use_file = true; 35 | /* Warning: we attempt to read your file for exactly 200,000 lines */ 36 | fp = fopen(argv[1], "r"); 37 | } 38 | 39 | if (use_file) { 40 | printf("Loading 200,000 coordinate pairs from source file...\n"); 41 | for (int i = 0; (read = getline(&line, &len, fp)) != -1 && i < TOTAL; 42 | i++) { 43 | char *err; 44 | 45 | latlong[i] = strtod(line, &err); 46 | latlong[i + 1] = strtod(err, &err); 47 | } 48 | } else { 49 | for (int i = 0; i < TOTAL; i++) { 50 | latlong[i] = 40.7126674; 51 | latlong[i + 1] = -74.0131594; 52 | } 53 | } 54 | 55 | printf("Sanity check: first latlong: (%f, %f)\n", latlong[0], latlong[1]); 56 | 57 | printf("Running an encode speed test...\n"); 58 | 59 | long total = sizeof(latlong) / sizeof(*latlong) - 1; 60 | 61 | /* encode speed test */ 62 | #define INNER_BOOST 2000 63 | GeoHashBits hash; 64 | long long start = ustime(); 65 | for (long i = 0; i < total; i++) 66 | for (int j = 0; j < INNER_BOOST; j++) 67 | geohashEncodeWGS84(latlong[i], latlong[i + 1], GEO_STEP_MAX, &hash); 68 | long long end = ustime(); 69 | 70 | double elapsed_seconds = (double)(end - start) / (double)1e6; 71 | printf("Elapsed encode time: %f seconds\n", elapsed_seconds); 72 | printf("Against %d total encodes\n", TOTAL * INNER_BOOST); 73 | printf("Speed: %f encodes per second\n", 74 | (TOTAL * INNER_BOOST) / elapsed_seconds); 75 | printf("(That's %f nanoseconds per encode)\n", 76 | elapsed_seconds * 1e9 / (TOTAL * INNER_BOOST)); 77 | 78 | printf("\n"); 79 | printf("Now running a decode speed test...\n"); 80 | 81 | /* decode speed test */ 82 | GeoHashArea area; 83 | start = ustime(); 84 | for (long i = 0; i < total; i++) 85 | for (int j = 0; j < INNER_BOOST; j++) 86 | geohashDecodeWGS84(hash, &area); 87 | end = ustime(); 88 | 89 | elapsed_seconds = (end - start) / 1e6; 90 | printf("Elapsed decode time: %f seconds\n", elapsed_seconds); 91 | printf("Against %d total decodes \n", TOTAL * INNER_BOOST); 92 | printf("Speed: %f decodes per second\n", 93 | (TOTAL * INNER_BOOST) / elapsed_seconds); 94 | printf("(That's %f nanoseconds per decode)\n", 95 | elapsed_seconds * 1e9 / (TOTAL * INNER_BOOST)); 96 | 97 | exit(EXIT_SUCCESS); 98 | } 99 | -------------------------------------------------------------------------------- /geo/to-redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | here="`dirname $0`" 4 | redis_dir="$here/../../redis" 5 | redis_src_dir="$redis_dir/src" 6 | redis_makefile="$redis_src_dir/Makefile" 7 | geohash_dir="$redis_dir/deps/geohash-int" 8 | 9 | # Copy geohash library to deps 10 | echo "Copying geohash library to deps..." 11 | cp "$here"/geohash* "$geohash_dir" 12 | 13 | if [[ ! -f "$geohash_dir/Makefile" ]]; then 14 | echo "You need to create "$geohash_dir/Makefile" manually, add it to redis/deps/Makefile, then add it to DEPENDENCY_TARGETS in $redis_makefile, then add the objects to the redis-server target, then add the geohash Makefile to deps/Makefile distclean target too." 15 | fi 16 | 17 | # Copy geo commands 18 | echo "Copying geo commands to redis/src..." 19 | cp "$here"/geo.{c,h} "$redis_src_dir" 20 | 21 | # Modify Redis build parameters 22 | grep geo.o "$redis_makefile" > /dev/null 23 | if [[ $? -eq 1 ]]; then 24 | # geo isn't in the Makefile. Let's add it. 25 | echo "Adding geo.o to Makefile" 26 | gsed -i '/^REDIS_SERVER_OBJ/ s/$/ geo.o/' "$redis_makefile" 27 | fi 28 | 29 | # zset.c traverses zsets without generate a client output buffer. 30 | # This is just a placeholder until we have a better zset API that 31 | # can return elements as a list (or the zset functions could take 32 | # a generic function pointer argument to call each time a matching 33 | # result is found). 34 | echo "Copying custom zset functions to redis/src..." 35 | cp "$here"/zset.{c,h} "$redis_src_dir" 36 | 37 | grep " zset.o" "$redis_makefile" > /dev/null 38 | if [[ $? -eq 1 ]]; then 39 | # geo isn't in the Makefile. Let's add it. 40 | echo "Adding zset.o to Makefile" 41 | gsed -i '/^REDIS_SERVER_OBJ/ s/$/ zset.o/' "$redis_makefile" 42 | fi 43 | 44 | # Update primary Redis Command Table 45 | grep georadiusbymember "$redis_src_dir/redis.c" > /dev/null 46 | if [[ $? -eq 1 ]]; then 47 | echo "Please manually add the commands from module.c to $redis_src_dir/redis.c then include these prototypes in redis.h:" 48 | cat $here/geo.h 49 | fi 50 | 51 | grep -- '-I../deps/geohash-int' "$redis_makefile" > /dev/null 52 | if [[ $? -eq 1 ]]; then 53 | echo "Adding ../deps/geohash-int to global include path..." 54 | gsed -i '/^FINAL_CFLAGS\+=/ s#$# -I../deps/geohash-int#' "$redis_makefile" 55 | fi 56 | 57 | echo "Copying geojson helper to redis/src..." 58 | cp "$here"/geojson.{c,h} "$redis_src_dir" 59 | 60 | # Modify Redis build parameters 61 | grep geojson.o "$redis_makefile" > /dev/null 62 | if [[ $? -eq 1 ]]; then 63 | # geo isn't in the Makefile. Let's add it. 64 | echo "Adding geojson.o to Makefile..." 65 | gsed -i '/^REDIS_SERVER_OBJ/ s/$/ geojson.o/' "$redis_makefile" 66 | fi 67 | 68 | grep "static int zslValueLteMax" "$redis_src_dir/t_zset.c" 69 | if [[ $? -eq 0 ]]; then 70 | # we need this to be not-static 71 | echo "Removing static from zslValueLteMax..." 72 | gsed -i 's/static int zslValueLteMax/int zslValueLteMax/' "$redis_src_dir/t_zset.c" 73 | fi 74 | 75 | echo "Copying geo tests..." 76 | cp "$here"/geo.tcl "$redis_dir/tests/unit" 77 | -------------------------------------------------------------------------------- /geo/zset.c: -------------------------------------------------------------------------------- 1 | #include "zset.h" 2 | 3 | /* t_zset.c prototypes (there's no t_zset.h) */ 4 | unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range); 5 | unsigned char *zzlFind(unsigned char *zl, robj *ele, double *score); 6 | int zzlLexValueLteMax(unsigned char *p, zlexrangespec *spec); 7 | 8 | /* Converted from static in t_zset.c: */ 9 | int zslValueLteMax(double value, zrangespec *spec); 10 | 11 | /* ==================================================================== 12 | * Direct Redis DB Interaction 13 | * ==================================================================== */ 14 | 15 | /* zset access is mostly a copy/paste from zscoreCommand() */ 16 | bool zsetScore(robj *zobj, robj *member, double *score) { 17 | if (!zobj || !member) 18 | return false; 19 | 20 | if (zobj->encoding == REDIS_ENCODING_ZIPLIST) { 21 | if (zzlFind(zobj->ptr, member, score) == NULL) 22 | return false; 23 | } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { 24 | zset *zs = zobj->ptr; 25 | dictEntry *de; 26 | 27 | member = tryObjectEncoding(member); 28 | de = dictFind(zs->dict, member); 29 | if (de != NULL) { 30 | *score = *(double *)dictGetVal(de); 31 | } else 32 | return false; 33 | } else { 34 | return false; 35 | } 36 | return true; 37 | } 38 | 39 | /* Largely extracted from genericZrangebyscoreCommand() in t_zset.c */ 40 | /* The zrangebyscoreCommand expects to only operate on a live redisClient, 41 | * but we need results returned to us, not sent over an async socket. */ 42 | list *geozrangebyscore(robj *zobj, double min, double max, int limit) { 43 | /* minex 0 = include min in range; maxex 1 = exclude max in range */ 44 | /* That's: min <= val < max */ 45 | zrangespec range = {.min = min, .max = max, .minex = 0, .maxex = 1}; 46 | list *l = NULL; /* result list */ 47 | 48 | if (zobj->encoding == REDIS_ENCODING_ZIPLIST) { 49 | unsigned char *zl = zobj->ptr; 50 | unsigned char *eptr, *sptr; 51 | unsigned char *vstr = NULL; 52 | unsigned int vlen = 0; 53 | long long vlong = 0; 54 | double score = 0; 55 | 56 | if ((eptr = zzlFirstInRange(zl, &range)) == NULL) { 57 | /* Nothing exists starting at our min. No results. */ 58 | return NULL; 59 | } 60 | 61 | l = listCreate(); 62 | 63 | sptr = ziplistNext(zl, eptr); 64 | 65 | while (eptr && limit--) { 66 | score = zzlGetScore(sptr); 67 | 68 | /* If we fell out of range, break. */ 69 | if (!zslValueLteMax(score, &range)) 70 | break; 71 | 72 | /* We know the element exists. ziplistGet should always succeed */ 73 | ziplistGet(eptr, &vstr, &vlen, &vlong); 74 | if (vstr == NULL) { 75 | listAddNodeTail(l, result_long(score, vlong)); 76 | } else { 77 | listAddNodeTail(l, result_str(score, vstr, vlen)); 78 | } 79 | zzlNext(zl, &eptr, &sptr); 80 | } 81 | } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { 82 | zset *zs = zobj->ptr; 83 | zskiplist *zsl = zs->zsl; 84 | zskiplistNode *ln; 85 | 86 | if ((ln = zslFirstInRange(zsl, &range)) == NULL) { 87 | /* Nothing exists starting at our min. No results. */ 88 | return NULL; 89 | } 90 | 91 | l = listCreate(); 92 | 93 | while (ln && limit--) { 94 | robj *o = ln->obj; 95 | /* Abort when the node is no longer in range. */ 96 | if (!zslValueLteMax(ln->score, &range)) 97 | break; 98 | 99 | if (o->encoding == REDIS_ENCODING_INT) { 100 | listAddNodeTail(l, result_long(ln->score, (long)o->ptr)); 101 | } else { 102 | listAddNodeTail(l, 103 | result_str(ln->score, o->ptr, sdslen(o->ptr))); 104 | } 105 | 106 | ln = ln->level[0].forward; 107 | } 108 | } 109 | if (l) { 110 | listSetFreeMethod(l, (void (*)(void *ptr)) & free_zipresult); 111 | } 112 | 113 | return l; 114 | } 115 | 116 | /* ==================================================================== 117 | * Helpers 118 | * ==================================================================== */ 119 | 120 | /* join 'join' to 'join_to' and free 'join' container */ 121 | void listJoin(list *join_to, list *join) { 122 | /* If the current list has zero size, move join to become join_to. 123 | * If not, append the new list to the current list. */ 124 | if (join_to->len == 0) { 125 | join_to->head = join->head; 126 | } else { 127 | join_to->tail->next = join->head; 128 | join->head->prev = join_to->tail; 129 | join_to->tail = join->tail; 130 | } 131 | 132 | /* Update total element count */ 133 | join_to->len += join->len; 134 | 135 | /* Release original list container. Internal nodes were transferred over. */ 136 | zfree(join); 137 | } 138 | 139 | /* A ziplist member may be either a long long or a string. We create the 140 | * contents of our return zipresult based on what the ziplist contained. */ 141 | static struct zipresult *result(double score, long long v, unsigned char *s, 142 | int len) { 143 | struct zipresult *r = zmalloc(sizeof(*r)); 144 | 145 | /* If string and length, become a string. */ 146 | /* Else, if not string or no length, become a long. */ 147 | if (s && len >= 0) 148 | r->type = ZR_STRING; 149 | else if (!s || len < 0) 150 | r->type = ZR_LONG; 151 | 152 | r->score = score; 153 | switch (r->type) { 154 | case (ZR_LONG): 155 | r->val.v = v; 156 | break; 157 | case (ZR_STRING): 158 | r->val.s = sdsnewlen(s, len); 159 | break; 160 | } 161 | return r; 162 | } 163 | 164 | struct zipresult *result_long(double score, long long v) { 165 | return result(score, v, NULL, -1); 166 | } 167 | 168 | struct zipresult *result_str(double score, unsigned char *str, int len) { 169 | return result(score, 0, str, len); 170 | } 171 | 172 | void free_zipresult(struct zipresult *r) { 173 | if (!r) 174 | return; 175 | 176 | switch (r->type) { 177 | case (ZR_LONG): 178 | break; 179 | case (ZR_STRING): 180 | sdsfree(r->val.s); 181 | break; 182 | } 183 | 184 | zfree(r); 185 | } 186 | -------------------------------------------------------------------------------- /geo/zset.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZSET_H__ 2 | #define __ZSET_H__ 3 | 4 | #include "redis.h" 5 | #include 6 | 7 | #define ZR_LONG 1 8 | #define ZR_STRING 2 9 | struct zipresult { 10 | double score; 11 | union { 12 | long long v; 13 | sds s; 14 | } val; 15 | double distance; /* distance is in meters */ 16 | char type; /* access type for the union */ 17 | }; 18 | 19 | /* Redis DB Access */ 20 | bool zsetScore(robj *zobj, robj *member, double *score); 21 | list *geozrangebyscore(robj *zobj, double min, double max, int limit); 22 | 23 | /* New list operation: append one list to another */ 24 | void listJoin(list *join_to, list *join); 25 | 26 | /* Helpers for returning zrangebyscore results */ 27 | struct zipresult *result_str(double score, unsigned char *str, int len); 28 | struct zipresult *result_long(double score, long long v); 29 | void free_zipresult(struct zipresult *r); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /hmsetif/hmsetif.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | 3 | /* ==================================================================== 4 | * Redis Add-on Module: hmsetif 5 | * Provides commands: hmsetif 6 | * Behaviors: 7 | * - hmsetif - only set multiple values if the first condition matches 8 | * ==================================================================== */ 9 | 10 | /* ==================================================================== 11 | * Command Implementations 12 | * ==================================================================== */ 13 | /* HSETCK myhash checkkey checkval setkey setval */ 14 | void hmsetifCommand(redisClient *c) { 15 | if ((c->argc % 2) == 1) { 16 | addReplyError(c, "wrong number of arguments for HMSETIF"); 17 | return; 18 | } 19 | 20 | robj *o; 21 | if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) 22 | return; 23 | 24 | if (!hashTypeExists(o, c->argv[2])) { 25 | addReply(c, shared.czero); 26 | return; 27 | } 28 | 29 | robj *testobj = hashTypeGetObject(o, c->argv[2]); 30 | if (!equalStringObjects(testobj, c->argv[3])) { 31 | addReply(c, shared.czero); 32 | decrRefCount(testobj); 33 | return; 34 | } 35 | decrRefCount(testobj); 36 | 37 | /* Rearrange command vector for direct hmset */ 38 | /* We need to: remove the checkkey and checkval. Instead 39 | * of rewriting the entire command vector, we 40 | * can accomplish it by: free checkkey/checkval then 41 | * move the last setkey/setval into the position of checkkey/checkval. */ 42 | decrRefCount(c->argv[2]); 43 | decrRefCount(c->argv[3]); 44 | c->argv[2] = c->argv[c->argc - 2]; 45 | c->argv[3] = c->argv[c->argc - 1]; 46 | c->argc -= 2; 47 | hmsetCommand(c); 48 | } 49 | 50 | /* ==================================================================== 51 | * Bring up / Teardown 52 | * ==================================================================== */ 53 | void *load() { 54 | return NULL; 55 | } 56 | 57 | /* If you reload the module *without* freeing things you allocate in load(), 58 | * then you *will* introduce memory leaks. */ 59 | void cleanup(void *privdata) { 60 | } 61 | 62 | /* ==================================================================== 63 | * Dynamic Redis API Requirements 64 | * ==================================================================== */ 65 | struct redisModule redisModuleDetail = { 66 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 67 | REDIS_VERSION, /* Provided by redis.h */ 68 | "0.3", /* Version of this module (only for reporting) */ 69 | "sh.matt.hmsetif", /* Unique name of this module */ 70 | load, /* Load function pointer (optional) */ 71 | cleanup /* Cleanup function pointer (optional) */ 72 | }; 73 | 74 | struct redisCommand redisCommandTable[] = { 75 | {"hmsetif", hmsetifCommand, -6, "wm", 0, NULL, 1, 1, 1, 0, 0}, 76 | {0} /* Always end your command table with {0} 77 | * If you forget, you will be reminded with a segfault on load. */ 78 | }; 79 | -------------------------------------------------------------------------------- /hmsetnx/hmsetnx.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | 3 | /* ==================================================================== 4 | * Redis Add-on Module: hmsetnx 5 | * Provides commands: hmsetnx 6 | * Behaviors: 7 | * - hmsetnx - hash multi-set only if fields don't exist 8 | * ==================================================================== */ 9 | 10 | /* ==================================================================== 11 | * Command Implementations 12 | * ==================================================================== */ 13 | void hmsetnxCommand(redisClient *c) { 14 | /* args 0-N: ["HMSETNX", key, field1, val1, field2, val2, ...] */ 15 | /* Returns 0 if no keys were set or OK if keys were set. */ 16 | int i, busykeys = 0; 17 | robj *o; 18 | 19 | if ((c->argc % 2) == 1) { 20 | addReplyError(c, "wrong number of arguments for HMSETNX"); 21 | return; 22 | } 23 | 24 | if ((o = hashTypeLookupWriteOrCreate(c, c->argv[1])) == NULL) 25 | return; 26 | hashTypeTryConversion(o, c->argv, 2, c->argc - 1); 27 | 28 | /* Handle the NX flag. The HMSETNX semantic is to return zero and don't 29 | * set nothing at all if at least one already key exists. */ 30 | for (i = 2; i < c->argc; i += 2) { 31 | if (hashTypeExists(o, c->argv[i])) { 32 | busykeys++; 33 | } 34 | if (busykeys) { 35 | addReply(c, shared.czero); 36 | return; 37 | } 38 | } 39 | 40 | hmsetCommand(c); 41 | } 42 | 43 | /* ==================================================================== 44 | * Bring up / Teardown 45 | * ==================================================================== */ 46 | void *load() { 47 | return NULL; 48 | } 49 | 50 | /* If you reload the module *without* freeing things you allocate in load(), 51 | * then you *will* introduce memory leaks. */ 52 | void cleanup(void *privdata) { 53 | } 54 | 55 | /* ==================================================================== 56 | * Dynamic Redis API Requirements 57 | * ==================================================================== */ 58 | struct redisModule redisModuleDetail = { 59 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 60 | REDIS_VERSION, /* Provided by redis.h */ 61 | "0.4", /* Version of this module (only for reporting) */ 62 | "sh.matt.hmsetnx", /* Unique name of this module */ 63 | load, /* Load function pointer (optional) */ 64 | cleanup /* Cleanup function pointer (optional) */ 65 | }; 66 | 67 | struct redisCommand redisCommandTable[] = { 68 | {"hmsetnx", hmsetnxCommand, -4, "wm", 0, NULL, 1, 1, 1, 0, 0}, 69 | {0} /* Always end your command table with {0} 70 | * If you forget, you will be reminded with a segfault on load. */ 71 | }; 72 | -------------------------------------------------------------------------------- /json/i_yajl.c: -------------------------------------------------------------------------------- 1 | #include "i_yajl.h" 2 | #include "jsonobj.h" 3 | 4 | /* ==================================================================== 5 | * yajl callbacks 6 | * ==================================================================== */ 7 | 8 | /*#define GEN_AND_RETURN(func) \ 9 | { \ 10 | yajl_gen_status __stat = func; \ 11 | if (__stat == yajl_gen_generation_complete && s_streamredis) { \ 12 | yajl_gen_reset(g, "\n"); \ 13 | __stat = func; \ 14 | } \ 15 | return __stat == yajl_gen_status_ok; \ 16 | } 17 | */ 18 | #define GEN_AND_RETURN(func) return func; 19 | #define previous_container(_l) listNodeValue(listLast(_l)) 20 | 21 | /* ==================================================================== 22 | * Global state for turning json into (jsonObj *) 23 | * ==================================================================== */ 24 | struct ctxstate { 25 | list *mapstack; 26 | struct jsonObj *current; 27 | sds keyname; 28 | }; 29 | 30 | /* ==================================================================== 31 | * yajl parsing callbacks 32 | * ==================================================================== */ 33 | #define assign_name(_obj) \ 34 | do { \ 35 | D("Adding name to object...\n"); \ 36 | (_obj)->name = state->keyname; \ 37 | state->keyname = NULL; \ 38 | } while (0) 39 | 40 | #define process_field(_obj) \ 41 | do { \ 42 | if (!state->current) { \ 43 | state->current = (_obj); \ 44 | } else if (!jsonObjAddField(state->current, (_obj))) { \ 45 | jsonObjFree(_obj); \ 46 | return false; \ 47 | } \ 48 | } while (0) 49 | 50 | static int redis_parse_null(void *ctx) { 51 | struct ctxstate *state = ctx; 52 | 53 | struct jsonObj *o = jsonObjCreate(); 54 | o->type = JSON_TYPE_NULL; 55 | assign_name(o); 56 | process_field(o); 57 | return true; 58 | } 59 | 60 | static int redis_parse_boolean(void *ctx, int boolean) { 61 | struct ctxstate *state = ctx; 62 | 63 | struct jsonObj *o = jsonObjCreate(); 64 | o->type = boolean ? JSON_TYPE_TRUE : JSON_TYPE_FALSE; 65 | assign_name(o); 66 | process_field(o); 67 | return true; 68 | } 69 | 70 | static int redis_parse_number(void *ctx, const char *s, size_t l) { 71 | struct ctxstate *state = ctx; 72 | 73 | struct jsonObj *o = jsonObjNumberAsStringLen((char *)s, l); 74 | assign_name(o); 75 | process_field(o); 76 | return true; 77 | } 78 | 79 | static int redis_parse_string(void *ctx, const unsigned char *stringVal, 80 | size_t stringLen) { 81 | struct ctxstate *state = ctx; 82 | 83 | struct jsonObj *o = jsonObjStringLen((char *)stringVal, stringLen); 84 | assign_name(o); 85 | process_field(o); 86 | return true; 87 | } 88 | 89 | static int redis_parse_map_key(void *ctx, const unsigned char *stringVal, 90 | size_t stringLen) { 91 | struct ctxstate *state = ctx; 92 | state->keyname = sdsnewlen(stringVal, stringLen); 93 | D("Parsed key name [%s]\n", state->keyname); 94 | return true; 95 | } 96 | 97 | static void addRecursiveContainer(struct ctxstate *state, 98 | struct jsonObj *newcontainer) { 99 | assign_name(newcontainer); 100 | 101 | /* Every container except the root is a member of a previous container */ 102 | if (state->current) { 103 | jsonObjAddField(state->current, newcontainer); 104 | listAddNodeTail(state->mapstack, state->current); 105 | } 106 | 107 | D("Setting current container to %p...\n", newcontainer); 108 | state->current = newcontainer; 109 | } 110 | 111 | static void removeRecursiveContainer(struct ctxstate *state) { 112 | /* If this closes the root container, there is no previous for us to 113 | * restore. */ 114 | if (listLength(state->mapstack)) { 115 | state->current = previous_container(state->mapstack); 116 | D("Restoring previous container %p...\n", state->current); 117 | 118 | /* The previous container is now the current container. We're 119 | * tracking it in state->current, so we can remove it from 120 | * the container stack. */ 121 | listDelNode(state->mapstack, listLast(state->mapstack)); 122 | } else { 123 | D("Not restoring previous container because already at root.\n"); 124 | } 125 | } 126 | 127 | static int redis_parse_start_map(void *ctx) { 128 | struct jsonObj *newmap = jsonObjCreateMap(); 129 | 130 | D("Starting map...\n"); 131 | addRecursiveContainer(ctx, newmap); 132 | return 1; 133 | } 134 | 135 | static int redis_parse_end_map(void *ctx) { 136 | D("Ending map...\n"); 137 | removeRecursiveContainer(ctx); 138 | return 1; 139 | } 140 | 141 | static int redis_parse_start_array(void *ctx) { 142 | struct jsonObj *newlist = jsonObjCreateList(); 143 | 144 | D("Starting array...\n"); 145 | addRecursiveContainer(ctx, newlist); 146 | return 1; 147 | } 148 | 149 | static int redis_parse_end_array(void *ctx) { 150 | D("Ending array...\n"); 151 | removeRecursiveContainer(ctx); 152 | return 1; 153 | } 154 | 155 | /* ==================================================================== 156 | * yajl parsing callback collection 157 | * ==================================================================== */ 158 | static yajl_callbacks callbacks = {.yajl_null = redis_parse_null, 159 | .yajl_boolean = redis_parse_boolean, 160 | .yajl_integer = NULL, 161 | .yajl_double = NULL, 162 | .yajl_number = redis_parse_number, 163 | .yajl_string = redis_parse_string, 164 | .yajl_start_map = redis_parse_start_map, 165 | .yajl_map_key = redis_parse_map_key, 166 | .yajl_end_map = redis_parse_end_map, 167 | .yajl_start_array = redis_parse_start_array, 168 | .yajl_end_array = redis_parse_end_array}; 169 | 170 | /* ==================================================================== 171 | * Redis generators 172 | * ==================================================================== */ 173 | static int redis_gen_sds(void *ctx, const sds string) { 174 | yajl_gen g = (yajl_gen)ctx; 175 | GEN_AND_RETURN( 176 | yajl_gen_string(g, (const unsigned char *)string, sdslen(string))); 177 | } 178 | 179 | static int redis_gen_double(void *ctx, const double number) { 180 | char numbuf[128] = {0}; 181 | int sz = snprintf(numbuf, 128, "%f", number); 182 | yajl_gen g = (yajl_gen)ctx; 183 | GEN_AND_RETURN(yajl_gen_number(g, numbuf, sz)); 184 | } 185 | 186 | static int redis_gen_double_from_str(void *ctx, sds number) { 187 | yajl_gen g = (yajl_gen)ctx; 188 | GEN_AND_RETURN(yajl_gen_number(g, (const char *)number, sdslen(number))); 189 | } 190 | 191 | static int redis_gen_start_map(void *ctx) { 192 | yajl_gen g = (yajl_gen)ctx; 193 | GEN_AND_RETURN(yajl_gen_map_open(g)); 194 | } 195 | 196 | static int redis_gen_end_map(void *ctx) { 197 | yajl_gen g = (yajl_gen)ctx; 198 | GEN_AND_RETURN(yajl_gen_map_close(g)); 199 | } 200 | 201 | static int redis_gen_start_list(void *ctx) { 202 | yajl_gen g = (yajl_gen)ctx; 203 | GEN_AND_RETURN(yajl_gen_array_open(g)); 204 | } 205 | 206 | static int redis_gen_end_list(void *ctx) { 207 | yajl_gen g = (yajl_gen)ctx; 208 | GEN_AND_RETURN(yajl_gen_array_close(g)); 209 | } 210 | 211 | /* ==================================================================== 212 | * Redis jsonObj Traversal and Encoding 213 | * ==================================================================== */ 214 | static void encodeJsonObj(yajl_gen g, struct jsonObj *f) { 215 | switch (f->type) { 216 | case JSON_TYPE_NUMBER_AS_STRING: 217 | redis_gen_double_from_str(g, f->content.string); 218 | break; 219 | case JSON_TYPE_NUMBER: 220 | redis_gen_double(g, f->content.number); 221 | break; 222 | case JSON_TYPE_TRUE: 223 | yajl_gen_bool(g, true); 224 | break; 225 | case JSON_TYPE_FALSE: 226 | yajl_gen_bool(g, false); 227 | break; 228 | case JSON_TYPE_NULL: 229 | yajl_gen_null(g); 230 | break; 231 | case JSON_TYPE_STRING: 232 | redis_gen_sds(g, f->content.string); 233 | break; 234 | case JSON_TYPE_LIST: 235 | redis_gen_start_list(g); 236 | for (int i = 0; i < f->content.obj.elements; i++) { 237 | encodeJsonObj(g, f->content.obj.fields[i]); 238 | } 239 | redis_gen_end_list(g); 240 | break; 241 | case JSON_TYPE_MAP: 242 | redis_gen_start_map(g); 243 | for (int i = 0; i < f->content.obj.elements; i++) { 244 | /* Set Name */ 245 | redis_gen_sds(g, f->content.obj.fields[i]->name); 246 | /* Set Value */ 247 | encodeJsonObj(g, f->content.obj.fields[i]); 248 | } 249 | redis_gen_end_map(g); 250 | break; 251 | } 252 | } 253 | 254 | /* ==================================================================== 255 | * Interface Helpers 256 | * ==================================================================== */ 257 | static void *yzmalloc(void *ctx, size_t sz) { 258 | return zmalloc(sz); 259 | } 260 | 261 | static void yzfree(void *ctx, void *ptr) { 262 | zfree(ptr); 263 | } 264 | 265 | static void *yzrealloc(void *ctx, void *ptr, size_t sz) { 266 | return zrealloc(ptr, sz); 267 | } 268 | 269 | yajl_alloc_funcs allocFuncs = { 270 | .malloc = yzmalloc, .free = yzfree, .realloc = yzrealloc, .ctx = NULL}; 271 | 272 | static yajl_handle setupParser(void *ctx) { 273 | yajl_handle hand = yajl_alloc(&callbacks, &allocFuncs, ctx); 274 | 275 | yajl_config(hand, yajl_allow_comments, 1); 276 | 277 | return hand; 278 | } 279 | 280 | static void cleanupParser(yajl_handle hand) { 281 | yajl_free(hand); 282 | } 283 | 284 | static yajl_gen setupEncoder() { 285 | yajl_gen g = yajl_gen_alloc(&allocFuncs); 286 | /* yajl_gen_config(g, yajl_gen_beautify, 1); */ 287 | return g; 288 | } 289 | 290 | static void cleanupEncoder(yajl_gen g) { 291 | yajl_gen_free(g); 292 | } 293 | 294 | static sds encodedJson(yajl_gen g) { 295 | const unsigned char *json_encoded; 296 | size_t len; 297 | 298 | yajl_gen_get_buf(g, &json_encoded, &len); 299 | sds result = sdsnewlen(json_encoded, len); 300 | yajl_gen_clear(g); 301 | 302 | return result; 303 | } 304 | 305 | /* ==================================================================== 306 | * Interface 307 | * ==================================================================== */ 308 | sds yajl_encode(struct jsonObj *f) { 309 | yajl_gen g = setupEncoder(); 310 | 311 | encodeJsonObj(g, f); 312 | sds result = encodedJson(g); 313 | cleanupEncoder(g); 314 | 315 | return result; 316 | } 317 | 318 | struct jsonObj *yajl_decode(sds json, sds *error) { 319 | struct ctxstate *state = zmalloc(sizeof(*state)); 320 | struct jsonObj *root; 321 | 322 | state->mapstack = listCreate(); 323 | state->current = NULL; 324 | state->keyname = NULL; 325 | 326 | yajl_handle hand = setupParser(state); 327 | 328 | yajl_status status = 329 | yajl_parse(hand, (const unsigned char *)json, sdslen(json)); 330 | yajl_complete_parse(hand); 331 | 332 | root = state->current; 333 | if (status != yajl_status_ok) { 334 | if (error) { 335 | unsigned char *err = yajl_get_error( 336 | hand, 1, (const unsigned char *)json, sdslen(json)); 337 | D("YAJL ERROR: %s\n", err); 338 | *error = sdscat(*error, (const char *)err); 339 | yajl_free_error(hand, err); 340 | } 341 | /* Remove any parsed json objects because the parse failed. Whatever 342 | * was created doesn't represent the entire input. */ 343 | jsonObjFree(root); 344 | root = NULL; 345 | } 346 | 347 | cleanupParser(hand); 348 | 349 | listRelease(state->mapstack); 350 | zfree(state); 351 | 352 | return root; 353 | } 354 | -------------------------------------------------------------------------------- /json/i_yajl.h: -------------------------------------------------------------------------------- 1 | #ifndef __I_YAJL_H__ 2 | #define __I_YAJL_H__ 3 | 4 | #include 5 | #include 6 | 7 | #include "json.h" 8 | #include "jsonobj.h" 9 | 10 | sds yajl_encode(struct jsonObj *f); 11 | struct jsonObj *yajl_decode(sds buffer, sds *error); 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /json/json.c: -------------------------------------------------------------------------------- 1 | #include "json.h" 2 | #include "jsonobj.h" 3 | #include "jsonobj_box.h" 4 | #include "jsonobj_get.h" 5 | #include "jsonobj_set.h" 6 | #include "jsonobj_delete.h" 7 | #include "i_yajl.h" 8 | 9 | /* ==================================================================== 10 | * Redis Add-on Module: json 11 | * Provides commands: hgetalljson, hmsetjson 12 | * Behaviors: 13 | * - hgetalljson - same as hgetall, but results returned as JSON map 14 | * - jsonwrap - wrap any Redis command in JSON a key-value map result 15 | * - jsondocvalidate - check if your JSON validates using Redis rules 16 | * - jsondocset - add (or overwrite) JSON document by key 17 | * - jsondocget - get JSON document with existing key 18 | * - jsondocmget - get multiple JSON documents at once 19 | * - jsondocdel - remove JSON documents by key 20 | * - jsondocsetbyjson - add (or overwrite) JSON documents by providing a 21 | * JSON array of maps, each with a field usable for IDs 22 | * - jsondockeys - get all keys (fields) of a JSON document 23 | * - jsonfieldget - get the value of one field of a JSON document 24 | * - jsonfieldset - set the value of one field of a JSON document [NI] 25 | * - jsonfielddel - remove a field from a JSON document or sub-container 26 | * - jsonfieldincrby - increment-by-int a number stored in a map or array 27 | * - jsonfieldincrbyfloat - increment-by-float number stored in map or array 28 | * - jsonfieldrpushx - push a number or string into a homogeneous list 29 | * - jsonfieldrpop - remove and return the last element of a list 30 | * ==================================================================== */ 31 | 32 | struct global g = {0}; 33 | 34 | /* ==================================================================== 35 | * Redis Internal Client Management 36 | * ==================================================================== */ 37 | static redisClient *newFakeClient() { 38 | redisClient *fake_client = createClient(-1); 39 | /* The fake client buffer only works if fd == -1 && flags & LUA_CLIENT */ 40 | fake_client->flags |= REDIS_LUA_CLIENT; 41 | return fake_client; 42 | } 43 | 44 | /* Force fake clients to use the same DB as the caller */ 45 | static void jsonSyncClients(redisClient *c) { 46 | g.c_noreturn->db = g.c->db = c->db; 47 | } 48 | 49 | robj *dbstrTake(sds id) { 50 | return createObject(REDIS_STRING, id); 51 | } 52 | robj *dbstr(sds id) { 53 | return dbstrTake(sdsdup(id)); 54 | } 55 | 56 | void clearClient(redisClient *c) { 57 | /* can't auto-free argv here because some argv are just 58 | * pointers to auto arrays */ 59 | c->argc = 0; 60 | c->argv = NULL; 61 | } 62 | 63 | void freeFakeClientResultBuffer(redisClient *c, sds reply) { 64 | /* If the reply isn't our reply buffer, we created it 65 | * independently and need to free it here. */ 66 | /* Else, it still belongs to the client to deal with. */ 67 | if (reply != c->buf) 68 | sdsfree(reply); 69 | } 70 | 71 | /* result buffer aggregation is taken from scripting.c */ 72 | sds fakeClientResultBuffer(redisClient *c) { 73 | sds reply; 74 | 75 | if (listLength(c->reply) == 0 && c->bufpos < REDIS_REPLY_CHUNK_BYTES) { 76 | /* This is a fast path for the common case of a reply inside the 77 | * client static buffer. Don't create an SDS string but just use 78 | * the client buffer directly. */ 79 | c->buf[c->bufpos] = '\0'; 80 | reply = c->buf; 81 | c->bufpos = 0; 82 | } else { 83 | reply = sdsnewlen(c->buf, c->bufpos); 84 | c->bufpos = 0; 85 | while (listLength(c->reply)) { 86 | robj *o = listNodeValue(listFirst(c->reply)); 87 | 88 | reply = sdscatlen(reply, o->ptr, sdslen(o->ptr)); 89 | listDelNode(c->reply, listFirst(c->reply)); 90 | } 91 | } 92 | 93 | return reply; 94 | } 95 | 96 | /* Don't let people use keys starting with a valid container box value */ 97 | bool validateKeyFormatAndReply(redisClient *c, sds key) { 98 | if (isValidBoxKeyPrefix(key[0])) { 99 | addReplyErrorFormat( 100 | c, "Invalid first character in key: [%c] (binary [%s])", key[0], 101 | bb(key[0])); 102 | return false; 103 | } 104 | 105 | return true; 106 | } 107 | 108 | /* ==================================================================== 109 | * Field Key Helpers 110 | * ==================================================================== */ 111 | /* Join the first (argc - trim) arguments into a colon key */ 112 | static sds genFieldAccessor(redisClient *c, int trim) { 113 | int usable = c->argc - (1 + trim); 114 | sds parts[usable]; 115 | 116 | for (int i = 1; i < usable + 1; i++) { 117 | parts[i - 1] = c->argv[i]->ptr; 118 | } 119 | 120 | return sdsjoin(parts, usable, ":"); 121 | } 122 | 123 | /* Search for 'field_name' as the name of a field in o. 124 | * Returns the value for 'field_name' in o wrapped in an (robj *). */ 125 | static robj *jsonObjFindId(sds field_name, struct jsonObj *o) { 126 | /* Only search for IDs in maps */ 127 | switch (o->type) { 128 | case JSON_TYPE_MAP: 129 | break; 130 | default: 131 | return NULL; 132 | } 133 | 134 | /* for each field in the map, search for field_name and return 135 | * an robj of the value for that field. */ 136 | for (int i = 0; i < o->content.obj.elements; i++) { 137 | struct jsonObj *f = o->content.obj.fields[i]; 138 | if (!strcmp(f->name, field_name)) { 139 | switch (f->type) { 140 | case JSON_TYPE_STRING: 141 | case JSON_TYPE_NUMBER_AS_STRING: 142 | D("Setting found id to: %s\n", f->content.string); 143 | return dbstr(f->content.string); 144 | break; 145 | default: 146 | /* We can't set a container type or immediate type 147 | * (direct number/true/false/null) as a key */ 148 | return NULL; 149 | } 150 | } 151 | } 152 | return NULL; 153 | } 154 | 155 | /* Delete existing document then add new document */ 156 | static int jsonObjAddToDB(robj *key, struct jsonObj *root) { 157 | int deleted; 158 | switch (root->type) { 159 | case JSON_TYPE_MAP: 160 | /* Only add map types! */ 161 | deleted = hgetallAndRecursivelyDeleteEverything(key->ptr); 162 | jsonObjToRedisDB(key, root); 163 | return deleted; 164 | break; 165 | default: 166 | return 1; 167 | break; 168 | } 169 | } 170 | 171 | /* ==================================================================== 172 | * Command Implementations 173 | * ==================================================================== */ 174 | /* jsonwrap assumes all commands return pairs of K V strings in a multibulk */ 175 | void jsonwrapCommand(redisClient *c) { 176 | /* args 0-N: ["jsonwrap", targetCommand, arg1, arg2, ..., argn] */ 177 | jsonSyncClients(c); 178 | genericWrapCommandAndReply(c); 179 | } 180 | 181 | /* More efficient version of jsonwrap for hgetall only */ 182 | void hgetalljsonCommand(redisClient *c) { 183 | addReplyMultiBulkLen(c, c->argc - 1); 184 | 185 | jsonSyncClients(c); 186 | for (int i = 1; i < c->argc; i++) { 187 | hgetallToJsonAndAndReply(c, c->argv[i]->ptr); 188 | } 189 | } 190 | 191 | void jsondocsetCommand(redisClient *c) { 192 | /* args 0-N: ["jsondocset", id, json, [id2, json2, ...]] */ 193 | /* If only K:V strings, submit as one hmset command */ 194 | 195 | /* This function is O(3*N) in the number of total keys submitted: 196 | * - first, we validate the keys 197 | * - second, we process all the json 198 | * - third, we add all the JSON to the DB. 199 | * + we run each loop indepdently because we must be able to abort 200 | * the entire add sequence if any prior aggregate attempt fails 201 | * (e.g. if you have a bad key or bad json, we don't want to add 202 | * _any_ of these documents.) */ 203 | 204 | /* If we don't have an even number of Key/Document pairs, exit. */ 205 | if ((c->argc - 1) % 2 != 0) { 206 | addReplyError( 207 | c, "Invalid number of arguments. Must match keys to documents."); 208 | return; 209 | } 210 | 211 | /* If any of the keys are invalid, exit. */ 212 | for (int i = 1; i < c->argc; i += 2) { 213 | if (!validateKeyFormatAndReply(c, c->argv[i]->ptr)) 214 | return; 215 | } 216 | 217 | int documents = (c->argc - 1) / 2; 218 | 219 | struct jsonObj *additions[documents]; 220 | for (int i = 1; i < c->argc; i += 2) { 221 | struct jsonObj *root = yajl_decode(c->argv[i + 1]->ptr, NULL); 222 | 223 | additions[i / 2] = root; 224 | 225 | if (!root) { 226 | /* Free any jsonObj trees already created */ 227 | for (int j = 0; j < i; j++) 228 | jsonObjFree(additions[j]); 229 | 230 | addReplyErrorFormat(c, "Invalid JSON at document %d. Use " 231 | "JSONDOCVALIDATE [json] for complete error.", 232 | i / 2); 233 | return; 234 | } 235 | } 236 | 237 | jsonSyncClients(c); 238 | int deleted = 0; 239 | /* Now jump keys at positions i*2 and documents at i/2 */ 240 | for (int i = 1; i < c->argc; i += 2) { 241 | struct jsonObj *root = additions[i / 2]; 242 | /* We can add a much simpler "check if key exists" test here first: */ 243 | deleted += jsonObjAddToDB(c->argv[i], root); 244 | jsonObjFree(root); 245 | D("Parsed Json %d!\n", i); 246 | } 247 | 248 | /* Reply += 1 if the document is new; reply += 0 if the document updated 249 | * (where an update is a full Delete/Create cycle) */ 250 | /* Replies with number of new keys set. Existing keys also get 251 | * set, but don't count as new. */ 252 | addReplyLongLong(c, documents - deleted); 253 | } 254 | 255 | void jsondocvalidateCommand(redisClient *c) { 256 | /* args 0-1: ["jsondocvalidate", json] */ 257 | 258 | sds err = sdsempty(); 259 | struct jsonObj *root = yajl_decode(c->argv[1]->ptr, &err); 260 | 261 | if (!root) { 262 | int sz; 263 | sds *error_lines = sdssplitlen(err, sdslen(err), "\n", 1, &sz); 264 | addReplyMultiBulkLen(c, sz + 1); 265 | addReply(c, g.err_parse); 266 | for (int i = 0; i < sz; i++) { 267 | addReplyBulkCBuffer(c, error_lines[i], sdslen(error_lines[i])); 268 | } 269 | sdsfreesplitres(error_lines, sz); 270 | } else { 271 | addReply(c, shared.ok); 272 | } 273 | sdsfree(err); 274 | } 275 | 276 | /* Returns ID(s) of documents added */ 277 | void jsondocsetbyjsonCommand(redisClient *c) { 278 | /* args 0-N: ["jsondocsetbyjson", id-field-name, json] */ 279 | jsonSyncClients(c); 280 | 281 | sds field_name = c->argv[1]->ptr; 282 | struct jsonObj *root = yajl_decode(c->argv[2]->ptr, NULL); 283 | 284 | if (!root) { 285 | addReply(c, g.err_parse); 286 | return; 287 | } 288 | 289 | robj *key; 290 | if (root->type == JSON_TYPE_LIST) { 291 | robj *keys[root->content.obj.elements]; 292 | D("Procesing list!\n"); 293 | /* First, process all documents for names to make sure they are valid 294 | * documents. We don't want to add half the documents then reach a 295 | * failure scenario. */ 296 | for (int i = 0; i < root->content.obj.elements; i++) { 297 | struct jsonObj *o = root->content.obj.fields[i]; 298 | key = jsonObjFindId(field_name, o); 299 | keys[i] = key; 300 | if (!key) { 301 | /* Free any allocated keys so far */ 302 | for (int j = 0; i < j; j++) 303 | decrRefCount(keys[j]); 304 | 305 | jsonObjFree(root); 306 | addReplyErrorFormat( 307 | c, 308 | "field '%s' not found or unusable as key for document %d", 309 | field_name, i); 310 | return; 311 | } 312 | } 313 | /* Now actually add all the documents */ 314 | /* Note how a multi-set gets a multibulk reply while 315 | * a regular one-document set gets just one bulk result. */ 316 | addReplyMultiBulkLen(c, root->content.obj.elements); 317 | for (int i = 0; i < root->content.obj.elements; i++) { 318 | struct jsonObj *o = root->content.obj.fields[i]; 319 | jsonObjAddToDB(keys[i], o); 320 | addReplyBulkCBuffer(c, keys[i]->ptr, sdslen(keys[i]->ptr)); 321 | decrRefCount(keys[i]); 322 | } 323 | } else if (root->type == JSON_TYPE_MAP) { 324 | key = jsonObjFindId(field_name, root); 325 | if (key) { 326 | jsonObjAddToDB(key, root); 327 | addReplyBulkCBuffer(c, key->ptr, sdslen(key->ptr)); 328 | decrRefCount(key); 329 | } else { 330 | addReplyErrorFormat( 331 | c, "field '%s' not found or unusable as key for document", 332 | field_name); 333 | } 334 | } else { 335 | addReplyError(c, "JSON isn't map or array of maps."); 336 | } 337 | 338 | jsonObjFree(root); 339 | } 340 | 341 | void jsondocdelCommand(redisClient *c) { 342 | /* args 0-N: ["jsondocdel", id1, id2, ..., idN] */ 343 | jsonSyncClients(c); 344 | 345 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 346 | return; 347 | 348 | long deleted = 0; 349 | for (int i = 1; i < c->argc; i++) { 350 | deleted += hgetallAndRecursivelyDeleteEverything(c->argv[i]->ptr); 351 | } 352 | addReplyLongLong(c, deleted); 353 | } 354 | 355 | /* a simple HKEYS on the proper hash for the requested document */ 356 | void jsondockeysCommand(redisClient *c) { 357 | /* args 0-N: ["jsondockeys", json-doc-id, [sub-id-1, sub-id-2, ...]] */ 358 | jsonSyncClients(c); 359 | 360 | sds key = genFieldAccessor(c, 0); 361 | 362 | sds found; 363 | int type; 364 | findKeyForHash(key, &found, &type); 365 | if (!found || type != JSON_TYPE_MAP) { 366 | addReplyError(c, "Key isn't a JSON document"); 367 | return; 368 | } 369 | 370 | decrRefCount(c->argv[1]); 371 | c->argv[1] = dbstrTake(found); 372 | sdsfree(key); 373 | hkeysCommand(c); 374 | } 375 | 376 | void jsondocgetCommand(redisClient *c) { 377 | /* args 0-N: ["jsondocget", jsonkey] */ 378 | jsonSyncClients(c); 379 | 380 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 381 | return; 382 | 383 | findJsonAndReply(c, c->argv[1]->ptr); 384 | } 385 | 386 | void jsondocmgetCommand(redisClient *c) { 387 | /* args 0-N: ["jsondocmget", jsonkey1, jsonkey2, ..., jsonkeyN] */ 388 | jsonSyncClients(c); 389 | 390 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 391 | return; 392 | 393 | addReplyMultiBulkLen(c, c->argc - 1); 394 | for (int i = 1; i < c->argc; i++) { 395 | findJsonAndReply(c, c->argv[i]->ptr); 396 | } 397 | } 398 | 399 | void jsonfieldgetCommand(redisClient *c) { 400 | /* args 0-N: ["jsonfieldget", json-dotted-key1, ..., json-dotted-keyN] */ 401 | jsonSyncClients(c); 402 | 403 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 404 | return; 405 | 406 | sds key = genFieldAccessor(c, 1); 407 | sds field = c->argv[c->argc - 1]->ptr; 408 | 409 | D("Asking for Key [%s] and Field [%s]\n", key, field); 410 | findJsonFieldAndReply(c, key, field); 411 | 412 | sdsfree(key); 413 | } 414 | 415 | void jsonfieldsetCommand(redisClient *c) { 416 | /* args 0-N: ["jsonfieldset", json field components, new-json] */ 417 | jsonSyncClients(c); 418 | 419 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 420 | return; 421 | 422 | /* sds key = genFieldAccessor(c, 1); */ 423 | addReplyBulkCString(c, "Thanks for your interest in the JSONFIELDSET " 424 | "command, but it is currently not implemented."); 425 | } 426 | 427 | void jsonfieldincrbyGeneric(redisClient *c, bool use_incrbyfloat) { 428 | /* args 0-N: ["jsonfieldincrby", json field components, incrby number] */ 429 | jsonSyncClients(c); 430 | 431 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 432 | return; 433 | 434 | sds key = genFieldAccessor(c, 2); 435 | sds found; 436 | int type; 437 | int decode_as = findKeyForHash(key, &found, &type); 438 | sdsfree(key); 439 | 440 | if (!found) { 441 | addReplyError(c, "JSON document not found"); 442 | return; 443 | } else if (decode_as == DECODE_INDIVIDUAL) { 444 | /* 'field' is the second to last argv[] element */ 445 | struct jsonObj *f = 446 | hgetToJsonObj(found, decode_as, c->argv[c->argc - 2]->ptr); 447 | switch (f->type) { 448 | case JSON_TYPE_MAP: 449 | case JSON_TYPE_LIST: 450 | case JSON_TYPE_TRUE: 451 | case JSON_TYPE_FALSE: 452 | case JSON_TYPE_NULL: 453 | addReplyError(c, "Requested field not usable for incrby. Can't " 454 | "increment non-number types."); 455 | break; 456 | case JSON_TYPE_NUMBER: 457 | /* found->content.number += incrby; 458 | break; */ 459 | case JSON_TYPE_STRING: 460 | case JSON_TYPE_NUMBER_AS_STRING: 461 | /* strtoi -> += incrby -> store again */ 462 | addReplyError(c, 463 | "Not currently supported on mixed-type containers."); 464 | break; 465 | } 466 | jsonObjFree(f); 467 | return; 468 | } 469 | 470 | /* Target args: 0-3: [_, HASH, FIELD, INCRBY] */ 471 | /* The hincrby* commands don't check argc, so we don't care if 472 | * we have extra arguments after INCRBY. They'll get cleaned up 473 | * when the client exits. */ 474 | decrRefCount(c->argv[1]); 475 | c->argv[1] = dbstrTake(found); 476 | 477 | /* If argc == 4, then the second and third arguments are already okay. 478 | * If argc > 4, we move the last two arguments to positions 3-4 */ 479 | if (c->argc > 4) { 480 | decrRefCount(c->argv[2]); /* goodbye, original argv[2] */ 481 | c->argv[2] = c->argv[c->argc - 2]; /* field is 2nd to last argument */ 482 | c->argv[c->argc - 2] = NULL; 483 | 484 | /* If argc == 5, then argv[3] is the one we moved to argv[2] above. 485 | * We can't release it because it just moved storage locations */ 486 | if (c->argc > 5) 487 | decrRefCount(c->argv[3]); /* goodbye, original argv[3] */ 488 | c->argv[3] = c->argv[c->argc - 1]; /* incrby value is last argument */ 489 | c->argv[c->argc - 1] = NULL; 490 | c->argc -= c->argc > 5 ? 2 : 1; /* if > 5, we removed two. else, we 491 | * removed one and moved the other. */ 492 | } 493 | 494 | if (use_incrbyfloat) 495 | hincrbyfloatCommand(c); 496 | else 497 | hincrbyCommand(c); 498 | } 499 | 500 | void jsonfieldincrbyCommand(redisClient *c) { 501 | jsonfieldincrbyGeneric(c, false); 502 | } 503 | 504 | void jsonfieldincrbyfloatCommand(redisClient *c) { 505 | jsonfieldincrbyGeneric(c, true); 506 | } 507 | 508 | void jsonfieldrpushxCommand(redisClient *c) { 509 | /* args 0-N: ["jsonfieldrpushx", key, sub-key1, sub-key2, ..., new json] */ 510 | jsonSyncClients(c); 511 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 512 | return; 513 | 514 | sds key = genFieldAccessor(c, 1); 515 | sds found; 516 | int type; 517 | int decode_as = findKeyForList(key, &found, &type); 518 | sdsfree(key); 519 | 520 | if (!found) { 521 | addReplyError(c, "JSON List not found"); 522 | return; 523 | } 524 | 525 | /* json is last argument */ 526 | struct jsonObj *o = yajl_decode(c->argv[c->argc - 1]->ptr, NULL); 527 | if (!o) { 528 | addReply(c, g.err_parse); 529 | sdsfree(found); 530 | return; 531 | } else if (o->type == JSON_TYPE_MAP || o->type == JSON_TYPE_LIST) { 532 | /* Implementation outline: 533 | * - get length of parent list 534 | * - create new container type with key sdsAppendColon(key, LENGTH+1) 535 | * - collapse container type to box. 536 | * - rpushx box into the list. */ 537 | addReplyError(c, "Sorry, you can only add basic types (string, number, " 538 | "true/false/null) to a list until somebody finishes " 539 | "this feature."); 540 | jsonObjFree(o); 541 | sdsfree(found); 542 | return; 543 | } else if (decode_as == DECODE_ALL_NUMBER && 544 | (o->type != JSON_TYPE_NUMBER && 545 | o->type != JSON_TYPE_NUMBER_AS_STRING)) { 546 | /* Complete implementation outline: 547 | * - convert homogeneous number list to individual list by: 548 | * - locate parent of list, rename box from JLIST|JHOMOGENEOUS|JNUMBER 549 | * to just JLIST 550 | * - rename this list to 'key' instead of 'found' 551 | * - For each current element of the list, box as number. 552 | * - Now you can add your new non-number type to the list. */ 553 | addReplyError(c, "You must add only numbers to your number-only list."); 554 | jsonObjFree(o); 555 | sdsfree(found); 556 | return; 557 | } else if (decode_as == JSON_TYPE_STRING && (o->type != JSON_TYPE_STRING)) { 558 | /* Complete implementation outline: 559 | * - see details for number above, but replace with string. */ 560 | addReplyError(c, "You must add only strings to your string-only list."); 561 | jsonObjFree(o); 562 | sdsfree(found); 563 | return; 564 | } else if (decode_as == DECODE_INDIVIDUAL) { 565 | jsonObjBoxBasicType(o); 566 | } 567 | 568 | /* Remove entire client argument list after the command */ 569 | for (int i = 1; i < c->argc; i++) 570 | decrRefCount(c->argv[i]); 571 | 572 | c->argc = 3; 573 | /* Re-populate our argument list */ 574 | /* We are reusing the origina c->argv memory, but it'll be 575 | * free'd when the client exists. We are guarnateed to have 576 | * at least four argument pointers available to us, and we only 577 | * need to have three allocated. Perfecto. */ 578 | c->argv[1] = dbstrTake(found); 579 | c->argv[2] = dbstrTake(o->content.string); 580 | o->content.string = NULL; 581 | jsonObjFree(o); 582 | 583 | /* Target args: 0-3: [_, LIST, RAW-STR-OR-NUMBER] */ 584 | rpushxCommand(c); 585 | } 586 | 587 | void jsonfieldrpopCommand(redisClient *c) { 588 | /* args 0-N: ["jsonfielddel", key, sub-key1, sub-key2, ...] */ 589 | jsonSyncClients(c); 590 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 591 | return; 592 | 593 | sds key = genFieldAccessor(c, 0); 594 | sds found; 595 | int type; 596 | int decode_as = findKeyForList(key, &found, &type); 597 | 598 | if (!found) { 599 | addReplyError(c, "JSON List not found"); 600 | return; 601 | } 602 | 603 | rpopRecursiveAndReply(c, found, decode_as); 604 | sdsfree(found); 605 | sdsfree(key); 606 | } 607 | 608 | void jsonfielddelCommand(redisClient *c) { 609 | /* args 0-N: ["jsonfielddel", key, sub-key1, sub-key2, ..., field] */ 610 | jsonSyncClients(c); 611 | 612 | if (!validateKeyFormatAndReply(c, c->argv[1]->ptr)) 613 | return; 614 | 615 | sds key = genFieldAccessor(c, 1); 616 | 617 | long deleted = findAndRecursivelyDelete(key, c->argv[c->argc - 1]->ptr); 618 | 619 | addReplyLongLong(c, deleted); 620 | sdsfree(key); 621 | } 622 | 623 | /* ==================================================================== 624 | * Bring up / Teardown 625 | * ==================================================================== */ 626 | void *load() { 627 | g.c = newFakeClient(); 628 | g.c_noreturn = createClient(-1); 629 | g.err_parse = createObject( 630 | REDIS_STRING, 631 | sdsnew("-ERR JSON Parse Error. You can debug with JSONDOCVALIDATE.")); 632 | return NULL; 633 | } 634 | 635 | /* If you reload the module *without* freeing things you allocate in load(), 636 | * then you *will* introduce memory leaks. */ 637 | void cleanup(void *privdata) { 638 | freeClient(g.c); 639 | freeClient(g.c_noreturn); 640 | decrRefCount(g.err_parse); 641 | } 642 | 643 | /* ==================================================================== 644 | * Dynamic Redis API Requirements 645 | * ==================================================================== */ 646 | struct redisModule redisModuleDetail = { 647 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 648 | REDIS_VERSION, /* Provided by redis.h */ 649 | "0.3", /* Version of this module (only for reporting) */ 650 | "sh.matt.json", /* Unique name of this module */ 651 | load, /* Load function pointer (optional) */ 652 | cleanup /* Cleanup function pointer (optional) */ 653 | }; 654 | 655 | struct redisCommand redisCommandTable[] = { 656 | {"hgetalljson", hgetalljsonCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 657 | {"jsonwrap", jsonwrapCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 658 | {"jsondocvalidate", jsondocvalidateCommand, 2, "r", 0, NULL, 0, 0, 0, 0, 0}, 659 | {"jsondocset", jsondocsetCommand, -3, "r", 0, NULL, 0, 0, 0, 0, 0}, 660 | {"jsondocget", jsondocgetCommand, 2, "r", 0, NULL, 0, 0, 0, 0, 0}, 661 | {"jsondocmget", jsondocmgetCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 662 | {"jsondocdel", jsondocdelCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 663 | {"jsondocsetbyjson", jsondocsetbyjsonCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 664 | 0}, 665 | {"jsondockeys", jsondockeysCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 666 | {"jsonfieldget", jsonfieldgetCommand, -3, "r", 0, NULL, 0, 0, 0, 0, 0}, 667 | {"jsonfieldset", jsonfieldsetCommand, 3, "r", 0, NULL, 0, 0, 0, 0, 0}, 668 | {"jsonfielddel", jsonfielddelCommand, -2, "r", 0, NULL, 0, 0, 0, 0, 0}, 669 | {"jsonfieldincrby", jsonfieldincrbyCommand, -3, "r", 0, NULL, 0, 0, 0, 0, 670 | 0}, 671 | {"jsonfieldincrbyfloat", jsonfieldincrbyfloatCommand, -3, "r", 0, NULL, 0, 672 | 0, 0, 0, 0}, 673 | {"jsonfieldrpushx", jsonfieldrpushxCommand, -4, "r", 0, NULL, 0, 0, 0, 0, 674 | 0}, 675 | {"jsonfieldrpop", jsonfieldrpopCommand, -3, "r", 0, NULL, 0, 0, 0, 0, 0}, 676 | {0} /* Always end your command table with {0} 677 | * If you forget, you will be reminded with a segfault on load. */ 678 | }; 679 | -------------------------------------------------------------------------------- /json/json.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSON_H__ 2 | #define __JSON_H__ 3 | 4 | #include "redis.h" 5 | #include 6 | 7 | #ifdef PRODUCTION 8 | #define D(...) 9 | #else 10 | #define D(...) \ 11 | do { \ 12 | printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__); \ 13 | printf(__VA_ARGS__); \ 14 | } while (0) 15 | #endif 16 | 17 | /* When reading a container type (map/list), we can shortcircut decoding if 18 | * we know the types up front: */ 19 | #define DECODE_UNAVAILABLE 0 20 | #define DECODE_ALL_NUMBER 128 21 | #define DECODE_ALL_STRING 129 22 | #define DECODE_INDIVIDUAL 130 23 | 24 | /* strlen("\r\n") */ 25 | #define SZ_CRLF 2 26 | 27 | /* Inbound type detectors for avoiding unnecessary recursive parsing */ 28 | #define ONLY_STRINGS(_obj) \ 29 | ((_obj)->content.obj.homogeneous && \ 30 | ((_obj)->content.obj.subtype == JSON_TYPE_STRING)) 31 | 32 | #define ONLY_NUMBERS(_obj) \ 33 | ((_obj)->content.obj.homogeneous && \ 34 | ((_obj)->content.obj.subtype == JSON_TYPE_NUMBER_AS_STRING)) 35 | 36 | /* Global things for json module */ 37 | struct global { 38 | redisClient *c; /* fake client acting like real client with output buffer */ 39 | redisClient *c_noreturn; /* fake client without output buffer */ 40 | robj *err_parse; 41 | }; 42 | 43 | /* Created and managed by json.c */ 44 | extern struct global g; 45 | 46 | robj *dbstrTake(sds id); 47 | robj *dbstr(sds id); 48 | void freeFakeClientResultBuffer(redisClient *c, sds reply); 49 | sds fakeClientResultBuffer(redisClient *c); 50 | void clearClient(redisClient *c); 51 | 52 | /* ==================================================================== 53 | * Binary Debugging 54 | * ==================================================================== */ 55 | /* Converts byte to an ASCII string of ones and zeroes */ 56 | /* 'bb' is easy to type and stands for "byte (to) binary (string)" */ 57 | static const char *bb(unsigned char x) { 58 | static char b[9] = {0}; 59 | 60 | b[0] = x & 0x80 ? '1' : '0'; 61 | b[1] = x & 0x40 ? '1' : '0'; 62 | b[2] = x & 0x20 ? '1' : '0'; 63 | b[3] = x & 0x10 ? '1' : '0'; 64 | b[4] = x & 0x08 ? '1' : '0'; 65 | b[5] = x & 0x04 ? '1' : '0'; 66 | b[6] = x & 0x02 ? '1' : '0'; 67 | b[7] = x & 0x01 ? '1' : '0'; 68 | 69 | return b; 70 | } 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /json/jsonobj.c: -------------------------------------------------------------------------------- 1 | #include "jsonobj.h" 2 | 3 | /* ==================================================================== 4 | * jsonObj Constructors 5 | * ==================================================================== */ 6 | struct jsonObj *jsonObjCreate() { 7 | struct jsonObj *root = zmalloc(sizeof(*root)); 8 | root->name = NULL; 9 | root->type = JSON_TYPE_NOTSET; 10 | 11 | return root; 12 | } 13 | 14 | /* How many fields we initially allocate for maps/lists */ 15 | #define INITIAL_FIELD_COUNT 32 16 | 17 | struct jsonObj *jsonObjCreateMap() { 18 | struct jsonObj *root = jsonObjCreate(); 19 | 20 | root->type = JSON_TYPE_MAP; 21 | root->content.obj.fields_sz = INITIAL_FIELD_COUNT; 22 | root->content.obj.homogeneous = false; /* Irrelevant with zero contents */ 23 | root->content.obj.subtype = JSON_TYPE_NOTSET; 24 | root->content.obj.elements = 0; 25 | root->content.obj.fields = zcalloc(sizeof(*root->content.obj.fields) * 26 | root->content.obj.fields_sz); 27 | 28 | return root; 29 | } 30 | 31 | struct jsonObj *jsonObjCreateList() { 32 | struct jsonObj *o = jsonObjCreateMap(); 33 | o->type = JSON_TYPE_LIST; 34 | return o; 35 | } 36 | 37 | struct jsonObj *jsonObjNumberAsStringTake(sds number) { 38 | struct jsonObj *o = jsonObjCreate(); 39 | 40 | o->type = JSON_TYPE_NUMBER_AS_STRING; 41 | o->content.string = number; 42 | 43 | return o; 44 | } 45 | 46 | struct jsonObj *jsonObjNumberAsString(sds number) { 47 | return jsonObjNumberAsStringTake(sdsdup(number)); 48 | } 49 | 50 | struct jsonObj *jsonObjNumberLongLong(long long number) { 51 | return jsonObjNumberAsStringTake(sdsfromlonglong(number)); 52 | } 53 | 54 | struct jsonObj *jsonObjNumber(double number) { 55 | /* If we can represent this double as an integer, do that instead. */ 56 | if (isfinite(number) && (long long)number == number) { 57 | return jsonObjNumberAsStringTake(sdsfromlonglong(number)); 58 | } else { 59 | /* else, create a normal double number object. */ 60 | struct jsonObj *o = jsonObjCreate(); 61 | 62 | o->type = JSON_TYPE_NUMBER; 63 | o->content.number = number; 64 | 65 | return o; 66 | } 67 | } 68 | 69 | struct jsonObj *jsonObjNumberAsStringLen(char *str, size_t len) { 70 | return jsonObjNumberAsStringTake(sdsnewlen(str, len)); 71 | } 72 | 73 | struct jsonObj *jsonObjStringTake(sds string) { 74 | struct jsonObj *o = jsonObjCreate(); 75 | 76 | o->type = JSON_TYPE_STRING; 77 | o->content.string = string; 78 | 79 | return o; 80 | } 81 | 82 | struct jsonObj *jsonObjString(sds string) { 83 | return jsonObjStringTake(sdsdup(string)); 84 | } 85 | 86 | struct jsonObj *jsonObjStringLen(char *string, size_t len) { 87 | return jsonObjStringTake(sdsnewlen(string, len)); 88 | } 89 | 90 | /* ==================================================================== 91 | * jsonObj Updaters 92 | * ==================================================================== */ 93 | void jsonObjTakeName(struct jsonObj *f, sds new_name) { 94 | if (f->name) 95 | sdsfree(f->name); 96 | 97 | f->name = new_name; 98 | } 99 | 100 | void jsonObjUpdateName(struct jsonObj *f, sds new_name) { 101 | jsonObjTakeName(f, sdsdup(new_name)); 102 | } 103 | 104 | bool jsonObjAddField(struct jsonObj *o, struct jsonObj *field) { 105 | if (!o) 106 | return false; 107 | 108 | /* Only add fields to MAP and LIST types. Anything else = error */ 109 | switch (o->type) { 110 | case JSON_TYPE_MAP: 111 | case JSON_TYPE_LIST: 112 | break; 113 | default: 114 | return false; 115 | } 116 | 117 | char subtype = o->content.obj.subtype; 118 | if (subtype == JSON_TYPE_NOTSET) { 119 | /* This is the first field in our object. Claim the type. */ 120 | o->content.obj.homogeneous = true; 121 | o->content.obj.subtype = field->type; 122 | } else if (o->content.obj.homogeneous && (subtype != field->type)) { 123 | /* We're a new field with a different type than all previous fields. 124 | * We've broken the homogeneous condition. */ 125 | o->content.obj.homogeneous = false; 126 | } 127 | 128 | /* If we are at max size, double the pointer allocation. */ 129 | if (o->content.obj.elements == o->content.obj.fields_sz) { 130 | o->content.obj.fields_sz *= 2; 131 | o->content.obj.fields = 132 | zrealloc(o->content.obj.fields, 133 | sizeof(*o->content.obj.fields) * o->content.obj.fields_sz); 134 | } 135 | 136 | o->content.obj.fields[o->content.obj.elements++] = field; 137 | return true; 138 | } 139 | 140 | /* ==================================================================== 141 | * jsonObj Destructor (recursive) 142 | * ==================================================================== */ 143 | void jsonObjFree(struct jsonObj *f) { 144 | if (!f) 145 | return; 146 | 147 | switch (f->type) { 148 | case JSON_TYPE_NUMBER: 149 | break; 150 | case JSON_TYPE_PTR: 151 | case JSON_TYPE_STRING: 152 | case JSON_TYPE_NUMBER_AS_STRING: 153 | sdsfree(f->content.string); 154 | break; 155 | case JSON_TYPE_LIST: 156 | case JSON_TYPE_MAP: 157 | for (int i = 0; i < f->content.obj.elements; i++) { 158 | jsonObjFree(f->content.obj.fields[i]); 159 | } 160 | zfree(f->content.obj.fields); 161 | break; 162 | } 163 | 164 | sdsfree(f->name); 165 | zfree(f); 166 | } 167 | -------------------------------------------------------------------------------- /json/jsonobj.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_H__ 2 | #define __JSONOBJ_H__ 3 | 4 | #include "json.h" 5 | #include "solarisfixes.h" 6 | 7 | #include 8 | /* ==================================================================== 9 | * The Cannonical jsonObj 10 | * ==================================================================== */ 11 | struct jsonObj { 12 | sds name; 13 | union { 14 | double number; 15 | sds string; 16 | struct { 17 | bool homogeneous; /* true if all fields are the same type */ 18 | int fields_sz; /* Total number of allocated fields */ 19 | int elements; /* Number of used fields */ 20 | struct jsonObj **fields; 21 | char subtype; /* If homogeneous, then subtype is type of fields */ 22 | } obj; 23 | } content; 24 | char type; 25 | }; 26 | 27 | /* ==================================================================== 28 | * Types for jsonObj 29 | * ==================================================================== */ 30 | #define JSON_TYPE_NOTSET 0 /* we haven't assigned a type yet */ 31 | #define JSON_TYPE_NUMBER 1 /* double */ 32 | #define JSON_TYPE_STRING 2 /* raw data -> redis hash value */ 33 | #define JSON_TYPE_MAP 3 /* json map -> redis hash */ 34 | #define JSON_TYPE_LIST 4 /* json list -> redis list */ 35 | #define JSON_TYPE_TRUE 5 36 | #define JSON_TYPE_FALSE 6 37 | #define JSON_TYPE_NULL 7 38 | #define JSON_TYPE_NUMBER_AS_STRING 8 39 | #define JSON_TYPE_PTR 9 40 | 41 | /* ==================================================================== 42 | * jsonObj Manipulators 43 | * ==================================================================== */ 44 | struct jsonObj *jsonObjCreate(); 45 | struct jsonObj *jsonObjCreateMap(); 46 | struct jsonObj *jsonObjCreateList(); 47 | 48 | bool jsonObjAddField(struct jsonObj *o, struct jsonObj *field); 49 | 50 | struct jsonObj *jsonObjNumber(double number); 51 | struct jsonObj *jsonObjNumberLongLong(long long number); 52 | struct jsonObj *jsonObjNumberAsStringTake(sds number); 53 | struct jsonObj *jsonObjNumberAsString(sds number); 54 | struct jsonObj *jsonObjNumberAsStringLen(char *str, size_t len); 55 | struct jsonObj *jsonObjStringTake(sds string); 56 | struct jsonObj *jsonObjString(sds string); 57 | struct jsonObj *jsonObjStringLen(char *string, size_t len); 58 | 59 | void jsonObjTakeName(struct jsonObj *f, sds new_name); 60 | void jsonObjUpdateName(struct jsonObj *f, sds new_name); 61 | 62 | void jsonObjFree(struct jsonObj *f); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /json/jsonobj.tcl: -------------------------------------------------------------------------------- 1 | # Run these tests by using the Redis `run-test` command. 2 | # - First, put this file in the redis `tests/` directory. 3 | # - Next, edit the `module-add` line in the first test to point to *your* 4 | # json.so module to load. 5 | # - Finally, run: ./run-test --single jsonobj 6 | 7 | start_server {tags {"jsonobj"}} { 8 | test {JSONDOC - set simple} { 9 | r config set module-add /home/matt/repos/krmt/json.so 10 | 11 | r jsondocset menu { 12 | {"menu": { "header": "SVG Viewer", "items": [ {"id": "Open"}, {"id": "OpenNew", "label": "Open New"}, null, {"id": "ZoomIn", "label": "Zoom In"}, {"id": "ZoomOut", "label": "Zoom Out"}, {"id": "OriginalView", "label": "Original View"}, null, {"id": "Quality"}, {"id": "Pause"}, {"id": "Mute"}, null, {"id": "Find", "label": "Find..."}, {"id": "FindAgain", "label": "Find Again"}, {"id": "Copy"}, {"id": "CopyAgain", "label": "Copy Again"}, {"id": "CopySVG", "label": "Copy SVG"}, {"id": "ViewSVG", "label": "View SVG"}, {"id": "ViewSource", "label": "View Source"}, {"id": "SaveAs", "label": "Save As"}, null, {"id": "Help"}, {"id": "About", "label": "About Adobe CVG Viewer..."} ] }} 13 | } 14 | 15 | } {1} 16 | 17 | test {JSONDOC - remove simple} { 18 | r jsondocdel menu 19 | } {1} 20 | 21 | test {JSONDOC - verify remove cleaned all keys} { 22 | r keys * 23 | } {} 24 | 25 | test {JSONDOC - set complex} { 26 | r jsondocset webapp { 27 | {"web-app": { "servlet": [ { "servlet-name": "cofaxCDS", "servlet-class": "org.cofax.cds.CDSServlet", "init-param": { "configGlossary:installationAt": "Philadelphia, PA", "configGlossary:adminEmail": "ksm@pobox.com", "configGlossary:poweredBy": "Cofax", "configGlossary:poweredByIcon": "/images/cofax.gif", "configGlossary:staticPath": "/content/static", "templateProcessorClass": "org.cofax.WysiwygTemplate", "templateLoaderClass": "org.cofax.FilesTemplateLoader", "templatePath": "templates", "templateOverridePath": "", "defaultListTemplate": "listTemplate.htm", "defaultFileTemplate": "articleTemplate.htm", "useJSP": false, "jspListTemplate": "listTemplate.jsp", "jspFileTemplate": "articleTemplate.jsp", "cachePackageTagsTrack": 200, "cachePackageTagsStore": 200, "cachePackageTagsRefresh": 60, "cacheTemplatesTrack": 100, "cacheTemplatesStore": 50, "cacheTemplatesRefresh": 15, "cachePagesTrack": 200, "cachePagesStore": 100, "cachePagesRefresh": 10, "cachePagesDirtyRead": 10, "searchEngineListTemplate": "forSearchEnginesList.htm", "searchEngineFileTemplate": "forSearchEngines.htm", "searchEngineRobotsDb": "WEB-INF/robots.db", "useDataStore": true, "dataStoreClass": "org.cofax.SqlDataStore", "redirectionClass": "org.cofax.SqlRedirection", "dataStoreName": "cofax", "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", "dataStoreUser": "sa", "dataStorePassword": "dataStoreTestQuery", "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", "dataStoreInitConns": 10, "dataStoreMaxConns": 100, "dataStoreConnUsageLimit": 100, "dataStoreLogLevel": "debug", "maxUrlLength": 500}}, { "servlet-name": "cofaxEmail", "servlet-class": "org.cofax.cds.EmailServlet", "init-param": { "mailHost": "mail1", "mailHostOverride": "mail2"}}, { "servlet-name": "cofaxAdmin", "servlet-class": "org.cofax.cds.AdminServlet"}, { "servlet-name": "fileServlet", "servlet-class": "org.cofax.cds.FileServlet"}, { "servlet-name": "cofaxTools", "servlet-class": "org.cofax.cms.CofaxToolsServlet", "init-param": { "templatePath": "toolstemplates/", "log": 1, "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", "logMaxSize": "", "dataLog": 1, "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", "dataLogMaxSize": "", "removePageCache": "/content/admin/remove?cache=pages&id=", "removeTemplateCache": "/content/admin/remove?cache=templates&id=", "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", "lookInContext": 1, "adminGroupID": 4, "betaServer": true}}], "servlet-mapping": { "cofaxCDS": "/", "cofaxEmail": "/cofaxutil/aemail/*", "cofaxAdmin": "/admin/*", "fileServlet": "/static/*", "cofaxTools": "/tools/*"}, "taglib": { "taglib-uri": "cofax.tld", "taglib-location": "/WEB-INF/tlds/cofax.tld"}}} 28 | } 29 | } {1} 30 | 31 | 32 | test {JSONDOC - remove complex} { 33 | r jsondocdel webapp 34 | 35 | } {1} 36 | 37 | 38 | test {JSONDOC - verify remove cleaned all keys} { 39 | r keys * 40 | } {} 41 | 42 | test {JSONDOC - add sample types} { 43 | r jsondocset simple { 44 | {"a": "b"} 45 | } 46 | 47 | r jsondocset simplelist { 48 | {"a": ["b"]} 49 | } 50 | 51 | r jsondocset simplemap { 52 | {"a": {"b":"c"}} 53 | } 54 | 55 | r jsondocset simplemaplist { 56 | {"a": {"b":["c"]}} 57 | } 58 | 59 | r jsondocset simplelistmap { 60 | {"a": [{"b":["c"]}]} 61 | } 62 | 63 | r jsondocset simplelistmapnumber { 64 | {"a": [{"b":[3]}]} 65 | } 66 | 67 | r jsondocset simplenumber { 68 | {"a": 4} 69 | } 70 | 71 | r jsondocset simpletrue { 72 | {"a": true} 73 | } 74 | 75 | r jsondocset simplefalse { 76 | {"a": false} 77 | } 78 | 79 | r jsondocset simplenull { 80 | {"a": null} 81 | } 82 | } {1} 83 | 84 | test {JSONDOC - remove all basic types} { 85 | r jsondocdel simple simplelist simplemap simplemaplist simplelistmap \ 86 | simplelistmapnumber simplenumber simpletrue simplefalse \ 87 | simplenull 88 | 89 | } {10} 90 | 91 | test {JSONDOC - verify remove cleaned all keys} { 92 | r keys * 93 | } {} 94 | 95 | test {JSONFIELD - basic get string} { 96 | r jsondocset simple { 97 | {"a": "b"} 98 | } 99 | 100 | r jsonfieldget simple a 101 | } {"b"} 102 | 103 | test {JSONFIELD - basic get number} { 104 | r jsondocset simple_number { 105 | {"a": 300} 106 | } 107 | 108 | r jsonfieldget simple_number a 109 | } {300} 110 | 111 | test {JSONFIELD - basic get list (numbers)} { 112 | r jsondocset simple_list_number { 113 | {"a": [1, 2, 3, 4]} 114 | } 115 | 116 | r jsonfieldget simple_list_number a 117 | } {[1,2,3,4]} 118 | 119 | test {JSONFIELD - basic get list (strings)} { 120 | r jsondocset simple_list_string { 121 | {"a": ["a", "b", "c", "d"]} 122 | } 123 | 124 | r jsonfieldget simple_list_string a 125 | } {["a","b","c","d"]} 126 | 127 | test {JSONFIELD - basic get list (with map)} { 128 | r jsondocset simple_list_map { 129 | {"a": [1, {"d":"e"}, 3, 4]} 130 | } 131 | 132 | r jsonfieldget simple_list_map a 133 | } {[1,{"d":"e"},3,4]} 134 | 135 | test {JSONFIELD - basic get map} { 136 | r jsondocset simple_m { 137 | {"a": {"b":"c"}} 138 | } 139 | 140 | r jsonfieldget simple_m a 141 | } {{"b":"c"}} 142 | 143 | test {JSONFIELD - basic get true} { 144 | r jsondocset simple_t { 145 | {"a": true} 146 | } 147 | 148 | r jsonfieldget simple_t a 149 | } {true} 150 | 151 | test {JSONFIELD - basic get false} { 152 | r jsondocset simple_f { 153 | {"a": false} 154 | } 155 | 156 | r jsonfieldget simple_f a 157 | } {false} 158 | 159 | test {JSONFIELD - basic get null} { 160 | r jsondocset simple_n { 161 | {"a": null} 162 | } 163 | 164 | r jsonfieldget simple_n a 165 | } {null} 166 | 167 | test {JSONDOC - remove all test documents} { 168 | r jsondocdel simple simple_number simple_list_number simple_list_string simple_list_map simple_m simple_t simple_f simple_n 169 | } {9} 170 | 171 | test {JSONDOC - verify remove cleaned all keys} { 172 | r keys * 173 | } {} 174 | 175 | test {JSONFIELD - multi-layered get list} { 176 | r jsondocset simple_list_map { 177 | {"a": [1, {"d":"e"}, 3, 4]} 178 | } 179 | 180 | r jsonfieldget simple_list_map a 181 | } {[1,{"d":"e"},3,4]} 182 | 183 | 184 | test {JSONFIELD - multi-layered get first} { 185 | r jsonfieldget simple_list_map a 0 186 | } {1} 187 | 188 | test {JSONFIELD - multi-layered get second} { 189 | r jsonfieldget simple_list_map a 1 190 | } {{"d":"e"}} 191 | 192 | test {JSONFIELD - multi-layered get second key} { 193 | r jsonfieldget simple_list_map a 1 d 194 | } {"e"} 195 | 196 | test {JSONFIELD - multi-layered get third} { 197 | r jsonfieldget simple_list_map a 2 198 | } {3} 199 | 200 | test {JSONFIELD - multi-layered get fourth} { 201 | r jsonfieldget simple_list_map a 3 202 | } {4} 203 | 204 | test {JSONDOC - remove all test documents} { 205 | r jsondocdel simple_list_map 206 | } {1} 207 | 208 | test {JSONDOC - verify remove cleaned all keys} { 209 | r keys * 210 | } {} 211 | 212 | test {JSONFIELD - deep nested get - level 1} { 213 | r jsondocset nest { 214 | {"a": [{"b":[{"c":["d"]}]}]} 215 | } 216 | 217 | r jsonfieldget nest a 218 | } {[{"b":[{"c":["d"]}]}]} 219 | 220 | test {JSONFIELD - deep nested get - level 2} { 221 | r jsonfieldget nest a 0 222 | } {{"b":[{"c":["d"]}]}} 223 | 224 | test {JSONFIELD - deep nested get - level 3} { 225 | r jsonfieldget nest a 0 b 226 | } {[{"c":["d"]}]} 227 | 228 | test {JSONFIELD - deep nested get - level 4} { 229 | r jsonfieldget nest a 0 b 0 230 | } {{"c":["d"]}} 231 | 232 | test {JSONFIELD - deep nested get - level 5} { 233 | r jsonfieldget nest a 0 b 0 c 234 | } {["d"]} 235 | 236 | test {JSONFIELD - deep nested get - level 6} { 237 | r jsonfieldget nest a 0 b 0 c 0 238 | } {"d"} 239 | 240 | test {JSONFIELD - deep nested get - level [too low]} { 241 | r jsonfieldget nest a 0 b 0 c -47 242 | } {} 243 | 244 | test {JSONFIELD - deep nested get - level [too high]} { 245 | r jsonfieldget nest a 0 b 0 c 47 246 | } {} 247 | 248 | test {JSONDOC - remove all test documents} { 249 | r jsondocdel nest 250 | } {1} 251 | 252 | test {JSONDOC - verify remove cleaned all keys} { 253 | r keys * 254 | } {} 255 | 256 | test {JSONFIELDDEL - basic delete} { 257 | r jsondocset basic { 258 | {"a":"b"} 259 | } 260 | r jsonfielddel basic a 261 | } {1} 262 | 263 | test {JSONFIELDDEL - verify document after delete} { 264 | r jsondocget basic 265 | } 266 | 267 | test {JSONFIELDDEL - basic map delete} { 268 | r jsondocset basic_map { 269 | {"a":{"b":"c"}} 270 | } 271 | r jsonfielddel basic_map a 272 | } {1} 273 | 274 | test {JSONFIELDDEL - verify document after delete} { 275 | r jsondocget basic_map 276 | } {} 277 | 278 | test {JSONFIELDDEL - basic list delete} { 279 | r jsondocset basic_list { 280 | {"a":[3,4]} 281 | } 282 | r jsonfielddel basic_list a 283 | } {1} 284 | 285 | test {JSONFIELDDEL - verify document after delete} { 286 | r jsondocget basic_list 287 | } {} 288 | 289 | test {JSONFIELDDEL - basic list delete II} { 290 | r jsondocset basic_list_ii { 291 | {"a":["a", "b"]} 292 | } 293 | r jsonfielddel basic_list_ii a 294 | } {1} 295 | 296 | test {JSONFIELDDEL - verify document after delete} { 297 | r jsondocget basic_list_ii 298 | } {} 299 | 300 | test {JSONFIELDDEL - nested list delete} { 301 | r jsondocset basic_list_ii { 302 | {"a":["a", "b", {"c":"d"}]} 303 | } 304 | r jsonfielddel basic_list_ii a 305 | } {1} 306 | 307 | test {JSONFIELDDEL - verify document after delete} { 308 | r jsondocget basic_list_ii 309 | } {} 310 | 311 | test {JSONFIELDDEL - nested list delete direct positional} { 312 | r jsondocset basic_list_ii_ii { 313 | {"a":["a", "b", {"c":"d"}]} 314 | } 315 | r jsonfielddel basic_list_ii_ii a 1 316 | } {0} 317 | 318 | test {JSONFIELDDEL - verify document after delete} { 319 | r jsondocget basic_list_ii_ii 320 | } {{"a":["a","b",{"c":"d"}]}} 321 | 322 | test {JSONFIELDDEL - nested map to empty map} { 323 | r jsondocset basic_list_ii_ii_ii { 324 | {"a":["a", "b", {"c":"d"}]} 325 | } 326 | r jsonfielddel basic_list_ii_ii_ii a 2 327 | } {1} 328 | 329 | test {JSONFIELDDEL - verify document after nested map delete} { 330 | r jsondocget basic_list_ii_ii_ii 331 | } {{"a":["a","b",{}]}} 332 | 333 | test {JSONFIELDDEL - nested list to empty list} { 334 | r jsondocset basic_list_iii { 335 | {"a":["a", "b", [{"c":"d"}, ["z", "q", {"e":"f"}]]]} 336 | } 337 | r jsonfielddel basic_list_iii a 2 338 | } {1} 339 | 340 | test {JSONFIELDDEL - verify document after nested list delete} { 341 | r jsondocget basic_list_iii 342 | } {{"a":["a","b",[]]}} 343 | 344 | test {JSONFIELDDEL - nested map later field delete} { 345 | r jsondocset basic_map_ii { 346 | {"a":"b", "c":"d"} 347 | } 348 | r jsonfielddel basic_map_ii c 349 | } {1} 350 | 351 | test {JSONFIELDDEL - verify document after delete} { 352 | r jsondocget basic_map_ii 353 | } {{"a":"b"}} 354 | 355 | test {JSONDOC - remove remaining test documents} { 356 | r jsondocdel basic_map_ii basic_list_ii_ii basic_list_ii_ii_ii basic_list_iii 357 | } {4} 358 | 359 | test {JSONDOC - verify remove cleaned all keys} { 360 | r keys * 361 | } {} 362 | 363 | test {JSONDOCSETBYJSON - set map} { 364 | r jsondocsetbyjson id { 365 | {"id":"alpha", "a":"b", "c":"d", "e":"f"} 366 | } 367 | 368 | } {alpha} 369 | 370 | test {JSONDOCSETBYJSON - set map inside list} { 371 | r jsondocsetbyjson id { 372 | [{"id":"beta", "a":"b", "c":"d", "e":"f"}] 373 | } 374 | 375 | } {beta} 376 | 377 | test {JSONDOCSETBYJSON - set multi-map inside list} { 378 | r jsondocsetbyjson id { 379 | [{"id":"gamma", "a":"b", "c":"d", "e":"f"}, 380 | {"a":"b", "id":"delta", "c":"d", "e":"f"}, 381 | {"a":"b", "c":"d", "id":"epsilon", "e":"f"}] 382 | } 383 | 384 | } {gamma delta epsilon} 385 | 386 | test {JSONDOC - remove remaining test documents} { 387 | r jsondocdel alpha beta gamma delta epsilon 388 | } {5} 389 | 390 | test {JSONDOC - verify remove cleaned all keys} { 391 | r keys * 392 | } {} 393 | 394 | test {JSONDOCKEYS - get keys of a map} { 395 | r jsondocset abc { 396 | {"abc": "def", "hij": "def", "zoo": {"number": 4000}} 397 | } 398 | 399 | r jsondockeys abc 400 | } {abc hij zoo} 401 | 402 | test {JSONFIELDINCRBY - increment by integer} { 403 | r jsonfieldincrby abc zoo number 1000 404 | r jsonfieldget abc zoo number 405 | } {5000} 406 | 407 | test {JSONFIELDINCRBYFLOAT - increment by float} { 408 | r jsonfieldincrbyfloat abc zoo number 14.5 409 | r jsonfieldget abc zoo number 410 | } {5014.5} 411 | 412 | test {JSONFIELDRPUSHX - rpush string} { 413 | r jsondocset def { 414 | {"a":["abc", "def"]} 415 | } 416 | r jsonfieldrpushx def a "\"hij\"" 417 | } {3} 418 | 419 | test {JSONFIELDRPUSHX - rpush string verify} { 420 | r jsonfieldget def a 421 | } {["abc","def","hij"]} 422 | 423 | test {JSONFIELDRPUSHX - rpush number} { 424 | r jsondocset hij { 425 | {"a":[1, 2, 3, 4, 5]} 426 | } 427 | r jsonfieldrpushx hij a 6 428 | } {6} 429 | 430 | test {JSONFIELDRPUSHX - rpush string verify} { 431 | r jsonfieldget hij a 432 | } {[1,2,3,4,5,6]} 433 | 434 | test {JSONFIELDRPOP - rpop number} { 435 | r jsonfieldrpop hij a 436 | } {6} 437 | 438 | test {JSONFIELDRPUSHX - rpop string verify} { 439 | r jsonfieldget hij a 440 | } {[1,2,3,4,5]} 441 | 442 | test {JSONFIELDRPOP - rpop number} { 443 | r jsonfieldrpop hij a 444 | } {5} 445 | 446 | test {JSONFIELDRPUSHX - rpop string verify} { 447 | r jsonfieldget hij a 448 | } {[1,2,3,4]} 449 | 450 | test {JSONFIELDRPOP - rpop mixed contents} { 451 | r jsondocset zed { 452 | {"a":[1, "two", 3, {"four": null}]} 453 | } 454 | r jsonfieldrpop zed a 455 | } {{"four":null}} 456 | 457 | test {JSONFIELDRPOP - rpop mixed contents verify} { 458 | r jsonfieldget zed a 459 | } {[1,"two",3]} 460 | 461 | test {JSONFIELDRPOP - rpop mixed contents} { 462 | r jsonfieldrpop zed a 463 | } {3} 464 | 465 | test {JSONFIELDRPOP - rpop mixed contents verify} { 466 | r jsonfieldget zed a 467 | } {[1,"two"]} 468 | 469 | test {JSONFIELDRPOP - rpop mixed contents} { 470 | r jsonfieldrpop zed a 471 | } {"two"} 472 | 473 | test {JSONFIELDRPOP - rpop mixed contents verify} { 474 | r jsonfieldget zed a 475 | } {[1]} 476 | 477 | test {JSONFIELDRPOP - rpop mixed contents last element} { 478 | r jsonfieldrpop zed a 479 | } {1} 480 | 481 | test {JSONFIELDRPOP - rpop mixed contents verify empty} { 482 | r jsonfieldget zed a 483 | } {[]} 484 | 485 | test {JSONDOC - remove remaining test documents} { 486 | r jsondocdel abc def hij zed 487 | } {4} 488 | 489 | test {JSONDOC - verify remove cleaned all keys} { 490 | r keys * 491 | } {} 492 | } 493 | -------------------------------------------------------------------------------- /json/jsonobj_box.c: -------------------------------------------------------------------------------- 1 | #include "jsonobj_box.h" 2 | 3 | /* ==================================================================== 4 | * jsonObj Box Management 5 | * ==================================================================== */ 6 | /* Given a box and a key, return a newly allocated [BOX]key */ 7 | sds boxgen(unsigned char box, sds key) { 8 | sds result = sdsnewlen(&box, 1); 9 | 10 | if (key) 11 | result = sdscatsds(result, key); 12 | 13 | return result; 14 | } 15 | 16 | static unsigned char jsonObjTagHomogeneous(struct jsonObj *t) { 17 | unsigned char boxtype = 0; 18 | 19 | /* Only tag homogeneous for container types */ 20 | switch (t->type) { 21 | case JSON_TYPE_MAP: 22 | D("IS MAP\n"); 23 | boxtype |= JMAP; 24 | break; 25 | case JSON_TYPE_LIST: 26 | D("IS LIST\n"); 27 | boxtype |= JLIST; 28 | break; 29 | default: 30 | return 0; 31 | break; 32 | } 33 | 34 | /* Box based on internal homogeneous type */ 35 | if (t->content.obj.homogeneous) { 36 | D("IS HOMOGENEOUS\n"); 37 | boxtype |= JHOMOGENEOUS; 38 | switch (t->content.obj.subtype) { 39 | case JSON_TYPE_NUMBER: 40 | case JSON_TYPE_NUMBER_AS_STRING: 41 | D("IS NUMBER\n"); 42 | boxtype |= JNUMBER; 43 | break; 44 | case JSON_TYPE_STRING: 45 | D("IS STRING\n"); 46 | boxtype |= JSTRING; 47 | break; 48 | case JSON_TYPE_MAP: 49 | case JSON_TYPE_LIST: 50 | case JSON_TYPE_TRUE: 51 | case JSON_TYPE_FALSE: 52 | case JSON_TYPE_NULL: 53 | /* Containers with these homogeneous subtypes aren't special. 54 | * We still must decode each member individually instead of being 55 | * returning a bulk HGETALL */ 56 | boxtype = 0; 57 | break; 58 | } 59 | D("Returning BOXTYPE: [%s] (subtype: %d)\n", bb(boxtype), 60 | t->content.obj.subtype); 61 | return boxtype; 62 | } else { 63 | return 0; 64 | } 65 | } 66 | 67 | /* Based on jsonObj type, return [BOX] for that type. */ 68 | static unsigned char jsonObjBoxType(struct jsonObj *t) { 69 | if (!t) 70 | return 0; 71 | 72 | /* The first byte of every type will be _boxtype_ 73 | * UNLESS the entire type is homogeneous, then we 74 | * use direct values in the containers and _boxtype_ is 75 | * promoted to the first byte of the key. */ 76 | unsigned char boxtype = 0; 77 | 78 | /* Define the box type byte */ 79 | switch (t->type) { 80 | case JSON_TYPE_MAP: 81 | boxtype = JMAP | jsonObjTagHomogeneous(t); 82 | break; 83 | case JSON_TYPE_LIST: 84 | boxtype = JLIST | jsonObjTagHomogeneous(t); 85 | break; 86 | case JSON_TYPE_STRING: 87 | boxtype = JSTRING; 88 | break; 89 | case JSON_TYPE_NUMBER: 90 | case JSON_TYPE_NUMBER_AS_STRING: 91 | boxtype = JNUMBER; 92 | break; 93 | case JSON_TYPE_TRUE: 94 | boxtype = JTRUE; 95 | break; 96 | case JSON_TYPE_FALSE: 97 | boxtype = JFALSE; 98 | break; 99 | case JSON_TYPE_NULL: 100 | boxtype = JNULL; 101 | break; 102 | case JSON_TYPE_PTR: 103 | D("ERROR - TRIED TO BOX PTR FOR %s\n", t->name); 104 | break; 105 | } 106 | 107 | return boxtype; 108 | } 109 | 110 | /* Return a newly allocated [BOX] for the given object. */ 111 | static sds jsonObjGenerateBox(struct jsonObj *o) { 112 | if (!o) 113 | return NULL; 114 | 115 | char boxtype = jsonObjBoxType(o); 116 | 117 | return boxgen(boxtype, NULL); 118 | } 119 | 120 | struct jsonObj *jsonObjConvertToPtr(struct jsonObj *o) { 121 | struct jsonObj *ptrobj = jsonObjCreate(); 122 | 123 | ptrobj->name = o->name; 124 | ptrobj->type = JSON_TYPE_PTR; 125 | ptrobj->content.string = jsonObjGenerateBox(o); 126 | 127 | D("Converting Obj Name %s to PTR [%s]\n", ptrobj->name, 128 | bb(*ptrobj->content.string)); 129 | 130 | o->name = NULL; /* We took the original name, so don't free it. */ 131 | jsonObjFree(o); 132 | 133 | return ptrobj; 134 | } 135 | 136 | static bool jsonObjBoxNumber(struct jsonObj *o) { 137 | unsigned char box = jsonObjBoxType(o); 138 | sds n; 139 | 140 | switch (o->type) { 141 | case JSON_TYPE_NUMBER: 142 | o->content.string = 143 | boxgen(box, sdscatprintf(sdsempty(), "%f", o->content.number)); 144 | break; 145 | case JSON_TYPE_NUMBER_AS_STRING: 146 | n = o->content.string; 147 | o->content.string = boxgen(box, n); 148 | sdsfree(n); 149 | break; 150 | default: 151 | D("ERROR - Attempted to turn non-number type (%d) into number " 152 | "object.\n", 153 | o->type); 154 | return false; 155 | } 156 | 157 | o->type = JSON_TYPE_PTR; 158 | return true; 159 | } 160 | 161 | static bool jsonObjBoxString(struct jsonObj *o) { 162 | unsigned char box; 163 | sds orig_str = o->content.string; 164 | 165 | box = jsonObjBoxType(o); 166 | o->type = JSON_TYPE_PTR; 167 | 168 | o->content.string = boxgen(box, orig_str); 169 | sdsfree(orig_str); 170 | 171 | return true; 172 | } 173 | 174 | /* If type is not a container, box it. */ 175 | bool jsonObjBoxBasicType(struct jsonObj *o) { 176 | unsigned char box; 177 | 178 | switch (o->type) { 179 | case JSON_TYPE_MAP: 180 | case JSON_TYPE_LIST: 181 | return false; 182 | break; 183 | case JSON_TYPE_NUMBER: 184 | case JSON_TYPE_NUMBER_AS_STRING: 185 | jsonObjBoxNumber(o); 186 | break; 187 | case JSON_TYPE_STRING: 188 | jsonObjBoxString(o); 189 | break; 190 | case JSON_TYPE_TRUE: 191 | case JSON_TYPE_FALSE: 192 | case JSON_TYPE_NULL: 193 | box = jsonObjBoxType(o); 194 | o->type = JSON_TYPE_PTR; 195 | o->content.string = boxgen(box, NULL); 196 | break; 197 | default: 198 | D("ERROR - Tried to box invalid type (%d)\n", o->type); 199 | return false; 200 | } 201 | 202 | return true; 203 | } 204 | 205 | /* inverse of boxBasicType for immediate decode boxes */ 206 | int jsonObjBasicTypeFromBox(unsigned char box) { 207 | if (box & JTRUE) 208 | return JSON_TYPE_TRUE; 209 | else if (box & JFALSE) 210 | return JSON_TYPE_FALSE; 211 | else if (box & JNULL) 212 | return JSON_TYPE_NULL; 213 | else 214 | return -1; 215 | } 216 | 217 | /* When reading a [BOX] from storage, determine the 218 | * action needed to fully decode the type */ 219 | int openBoxAction(unsigned char box) { 220 | switch (box) { 221 | case JMAP: 222 | return BOX_FETCH_MAP_DECODE; 223 | break; 224 | case JLIST: 225 | return BOX_FETCH_LIST_DECODE; 226 | break; 227 | case JNUMBER: 228 | return BOX_PARSE_NUMBER_AFTER_BOX; 229 | break; 230 | case JSTRING: 231 | return BOX_PARSE_STRING_AFTER_BOX; 232 | break; 233 | case JTRUE: 234 | case JFALSE: 235 | case JNULL: 236 | return BOX_DECODE_IMMEDIATE; 237 | break; 238 | case JMAP | JHOMOGENEOUS | JSTRING: 239 | return BOX_FETCH_MAP_STRING; 240 | break; 241 | case JMAP | JHOMOGENEOUS | JNUMBER: 242 | return BOX_FETCH_MAP_NUMBER; 243 | break; 244 | case JLIST | JHOMOGENEOUS | JNUMBER: 245 | return BOX_FETCH_LIST_NUMBER; 246 | break; 247 | case JLIST | JHOMOGENEOUS | JSTRING: 248 | return BOX_FETCH_LIST_STRING; 249 | break; 250 | case JMAP | JHOMOGENEOUS | JNULL: 251 | case JMAP | JHOMOGENEOUS | JTRUE: 252 | case JMAP | JHOMOGENEOUS | JFALSE: 253 | return BOX_FETCH_MAP_DECODE; 254 | break; 255 | case JLIST | JHOMOGENEOUS | JNULL: 256 | case JLIST | JHOMOGENEOUS | JTRUE: 257 | case JLIST | JHOMOGENEOUS | JFALSE: 258 | /* All values in the list will be single boxes */ 259 | /* Can be optimized to simpler fetches, but for now just 260 | * read all the element values. */ 261 | return BOX_FETCH_LIST_DECODE; 262 | break; 263 | default: 264 | D("ERROR - Unknown box type: [%s, %d]\n", bb(box), box); 265 | return 0; 266 | break; 267 | } 268 | } 269 | 270 | /* If we are decoding an ALL* type, generate the proper boxed key so we 271 | * can read the container. */ 272 | static sds boxByDecode(unsigned char typebox, sds key, int decode_as) { 273 | unsigned char box = 0; 274 | 275 | /* Homogeneous storage containers have their keys prefixed 276 | * with their box, so we have to add the box to our depth 277 | * key to find the container. */ 278 | switch (decode_as) { 279 | case DECODE_ALL_NUMBER: 280 | box = typebox | JHOMOGENEOUS | JNUMBER; 281 | break; 282 | case DECODE_ALL_STRING: 283 | box = typebox | JHOMOGENEOUS | JSTRING; 284 | break; 285 | } 286 | 287 | D("Box decode is returning with box [%s] (from [%s] with decode_as: %d)\n", 288 | bb(box), key, decode_as); 289 | if (box) 290 | return boxgen(box, key); 291 | else 292 | return sdsdup(key); 293 | } 294 | 295 | /* List key generation wrapper for boxByDecode */ 296 | sds boxListByDecode(sds key, int decode_as) { 297 | return boxByDecode(JLIST, key, decode_as); 298 | } 299 | 300 | /* Hash/map key generation wrapper for boxByDecode */ 301 | sds boxHashByDecode(sds key, int decode_as) { 302 | return boxByDecode(JMAP, key, decode_as); 303 | } 304 | 305 | /* We generate depth keys by concatenating the top level key 306 | * with each sub-key and join the results with a colon. 307 | * For non-key subtypes (i.e. positions in a list), we use 308 | * the position to denote the original element. */ 309 | sds sdsAppendColon(sds orig, sds append) { 310 | if (!orig) 311 | return NULL; 312 | 313 | if (!append) 314 | return sdsdup(orig); 315 | 316 | return sdscatsds(sdscatlen(sdsdup(orig), ":", 1), append); 317 | } 318 | 319 | sds sdsAppendColonInteger(sds orig, int append) { 320 | sds ll = sdsfromlonglong(append); 321 | sds appended = sdsAppendColon(orig, ll); 322 | sdsfree(ll); 323 | return appended; 324 | } 325 | 326 | /* Generate proper top-level storage keys for homogeneous types so we 327 | * (potentially) don't have to read every member of the container 328 | * when re-reading */ 329 | sds boxKeyIfHomogeneous(struct jsonObj *f, sds key) { 330 | unsigned char box = jsonObjTagHomogeneous(f); 331 | if (box) { 332 | D("Tagging [%s] as homogeneous\n", key); 333 | return boxgen(box, key); 334 | } else { 335 | return sdsdup(key); 336 | } 337 | } 338 | 339 | /* ==================================================================== 340 | * Locate Redis Key by first trying Boxed versions 341 | * ==================================================================== */ 342 | /* The first top-level json hash key *may* be a boxed name to 343 | * denote if we can read all values without decoding further. 344 | * First try [MAP|ALLSTRING]key, then try [MAP|ALLNUMBER]key, then try key. */ 345 | static int findKeyForContainer(unsigned char container, sds key, sds *found_key, 346 | int *type) { 347 | redisClient *c = g.c_noreturn; 348 | 349 | unsigned char only_strings_box = container | JHOMOGENEOUS | JSTRING; 350 | unsigned char only_numbers_box = container | JHOMOGENEOUS | JNUMBER; 351 | 352 | switch (container) { 353 | case JMAP: 354 | *type = JSON_TYPE_MAP; 355 | break; 356 | case JLIST: 357 | *type = JSON_TYPE_LIST; 358 | break; 359 | } 360 | 361 | /* Yes, this is a slight abuse of convention because we're using 362 | * dictFind() directly and not lookupKey() or dbExists(), but 363 | * dictFind lets us pass a sds directly which saves us a lot of 364 | * allocation/free copy/paste here. */ 365 | sds stringbox = boxgen(only_strings_box, key); 366 | if (dictFind(c->db->dict, stringbox)) { 367 | D("Found All Strings Key\n"); 368 | *found_key = stringbox; 369 | return DECODE_ALL_STRING; 370 | } 371 | sdsfree(stringbox); 372 | 373 | sds numbersbox = boxgen(only_numbers_box, key); 374 | if (dictFind(c->db->dict, numbersbox)) { 375 | D("Found All Numbers Key\n"); 376 | *found_key = numbersbox; 377 | return DECODE_ALL_NUMBER; 378 | } 379 | sdsfree(numbersbox); 380 | 381 | dictEntry *b; 382 | if ((b = dictFind(c->db->dict, key))) { 383 | D("Found Mixed Container Key\n"); 384 | *found_key = sdsdup(key); 385 | 386 | robj *bare = dictGetVal(b); 387 | if (bare->type == REDIS_HASH) 388 | *type = JSON_TYPE_MAP; 389 | else if (bare->type == REDIS_LIST) 390 | *type = JSON_TYPE_LIST; 391 | 392 | return DECODE_INDIVIDUAL; 393 | } else { 394 | D("Key not found for this container type.\n"); 395 | *found_key = NULL; 396 | return DECODE_UNAVAILABLE; 397 | } 398 | } 399 | 400 | int findKeyForHash(sds key, sds *found_key, int *type) { 401 | D("Searching For Hash Using [%s]\n", key); 402 | return findKeyForContainer(JMAP, key, found_key, type); 403 | } 404 | 405 | int findKeyForList(sds key, sds *found_key, int *type) { 406 | D("Searching For List Using [%s]\n", key); 407 | return findKeyForContainer(JLIST, key, found_key, type); 408 | } 409 | 410 | bool isValidBoxKeyPrefix(unsigned char box) { 411 | switch (box) { 412 | case JMAP | JHOMOGENEOUS | JSTRING: 413 | case JMAP | JHOMOGENEOUS | JNUMBER: 414 | case JLIST | JHOMOGENEOUS | JNUMBER: 415 | case JLIST | JHOMOGENEOUS | JSTRING: 416 | return true; 417 | break; 418 | default: 419 | return false; 420 | break; 421 | } 422 | } 423 | -------------------------------------------------------------------------------- /json/jsonobj_box.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_BOX_H__ 2 | #define __JSONOBJ_BOX_H__ 3 | 4 | #include "json.h" 5 | #include "jsonobj.h" 6 | 7 | /* ==================================================================== 8 | * Type Bits for Boxing JSON types in Redis Storage 9 | * ==================================================================== */ 10 | #define JBIT(bit) (1 << bit) 11 | 12 | #define JHOMOGENEOUS JBIT(0) 13 | #define JNUMBER JBIT(1) 14 | #define JSTRING JBIT(2) 15 | #define JMAP JBIT(3) 16 | #define JLIST JBIT(4) 17 | #define JTRUE JBIT(5) 18 | #define JFALSE JBIT(6) 19 | #define JNULL JBIT(7) 20 | /* DO NOT define more unary J types. The type box is a char. We 21 | * only have 8 bits (0-7) to work with. */ 22 | 23 | /* Valid boxes are: 24 | * - JTRUE 25 | * - JFALSE 26 | * - JNULL 27 | * - JSTRING 28 | * - JNUMBER 29 | * - JMAP 30 | * - JLIST 31 | * - JMAP|JHOMOGENEOUS|JNUMBER 32 | * - JMAP|JHOMOGENEOUS|JSTRING 33 | * - JLIST|JHOMOGENEOUS|JSTRING 34 | * - JLIST|JHOMOGENEOUS|JNUMBER 35 | * (We don't care about homogeneous container types containing only 36 | * true, false, null, maps, or lists.) 37 | */ 38 | 39 | /* If we need more types, we can use compound types based on individual 40 | * types impossible to co-exist naturally (e.g. NULL|FALSE, MAP|LIST, etc) */ 41 | 42 | /* Note: The combination 01111011 must never be used because that's 43 | * an ASCII '{' and would break cluster slot detection. It should be 44 | * easy to never create it because those bits represent the absurd combination 45 | * (homogeneous | number | map | list | true | false) */ 46 | 47 | /* Actions we need to take when we read [BOX] back from Redis */ 48 | #define BOX_FETCH_MAP_DECODE 64 49 | #define BOX_FETCH_LIST_DECODE 65 50 | #define BOX_PARSE_NUMBER_AFTER_BOX 66 51 | #define BOX_PARSE_STRING_AFTER_BOX 67 52 | #define BOX_DECODE_IMMEDIATE 68 53 | #define BOX_FETCH_MAP_NUMBER 69 54 | #define BOX_FETCH_MAP_STRING 70 55 | #define BOX_FETCH_LIST_NUMBER 71 56 | #define BOX_FETCH_LIST_STRING 72 57 | 58 | sds sdsAppendColon(sds orig, sds append); 59 | sds sdsAppendColonInteger(sds orig, int append); 60 | sds boxListByDecode(sds key, int decode_as); 61 | sds boxHashByDecode(sds key, int decode_as); 62 | int openBoxAction(unsigned char box); 63 | int findKeyForHash(sds key, sds *foundKey, int *type); 64 | int findKeyForList(sds key, sds *foundKey, int *type); 65 | int jsonObjBasicTypeFromBox(unsigned char box); 66 | sds boxKeyIfHomogeneous(struct jsonObj *f, sds key); 67 | bool jsonObjBoxBasicType(struct jsonObj *o); 68 | struct jsonObj *jsonObjConvertToPtr(struct jsonObj *o); 69 | sds boxHashByDecode(sds key, int decode_as); 70 | sds boxListByDecode(sds key, int decode_as); 71 | sds boxgen(unsigned char box, sds key); 72 | bool isValidBoxKeyPrefix(unsigned char box); 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /json/jsonobj_delete.c: -------------------------------------------------------------------------------- 1 | #include "jsonobj_delete.h" 2 | 3 | static struct jsonObj *jsonObjDeleteFromBox(unsigned char box, sds populate_as); 4 | 5 | /* ==================================================================== 6 | * Recusively Delete Entire Document 7 | * ==================================================================== */ 8 | /* Largely copied from lrangeCommand() */ 9 | static int jsonObjGetListFromBufferAndDeleteAll(robj *key, int decode_as) { 10 | redisClient *c = g.c_noreturn; 11 | robj *o; 12 | int start = 0; /* start at beginning */ 13 | int i = 0; /* current positon in the list */ 14 | sds populate_as; 15 | 16 | if (!(o = lookupKeyRead(c->db, key))) 17 | return 0; 18 | 19 | switch (decode_as) { 20 | case DECODE_ALL_NUMBER: 21 | case DECODE_ALL_STRING: 22 | /* delete the list and GET OUT NOW */ 23 | D("Deleting list [%s] (because it has no pointers inside of it).\n", 24 | key->ptr); 25 | return dbDelete(c->db, key); 26 | break; 27 | } 28 | 29 | long llen = listTypeLength(o); 30 | 31 | /* If we get this far, that means every element of this list is a 32 | * boxed type. We must unbox each value and recursively delete if 33 | * it is a pointer to another sub-container. */ 34 | if (o->encoding == REDIS_ENCODING_ZIPLIST) { 35 | unsigned char *p = ziplistIndex(o->ptr, start); 36 | unsigned char *vstr; 37 | unsigned int vlen; 38 | long long vlong; 39 | 40 | while (llen--) { 41 | ziplistGet(p, &vstr, &vlen, &vlong); 42 | if (vstr) { 43 | D("[%s]: List Ziplist decoding str [%s], as (%d)\n", key->ptr, 44 | vstr, decode_as); 45 | /* generate access key for this position in the list */ 46 | populate_as = sdsAppendColonInteger(key->ptr, i); 47 | jsonObjDeleteFromBox(vstr[0], populate_as); 48 | sdsfree(populate_as); 49 | } else { 50 | D("[%s]: List Ziplist decoding number [%lld], as (%d)\n", 51 | key->ptr, vlong, decode_as); 52 | D("ERROR--Trying to decode NUMBER as individual number. Not " 53 | "supported since we can't unbox a number.\n"); 54 | } 55 | p = ziplistNext(o->ptr, p); 56 | i++; 57 | } 58 | } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { 59 | listNode *ln; 60 | 61 | ln = listIndex(o->ptr, start); 62 | 63 | while (llen--) { 64 | D("[%s]: List Linked decoding [%s]\n", key->ptr, ln->value); 65 | /* generate access key for this position in the list */ 66 | populate_as = sdsAppendColonInteger(key->ptr, i); 67 | jsonObjDeleteFromBox(((sds)ln->value)[0], populate_as); 68 | sdsfree(populate_as); 69 | ln = ln->next; 70 | i++; 71 | } 72 | } 73 | return dbDelete(g.c->db, key); 74 | } 75 | 76 | /* Given a depth key and a decode type, generate the top-level boxed Redis 77 | * key we use to access this list, then open the list and delete 78 | * everything inside. */ 79 | static int jsonObjDeleteFromRedisListSdsKey(sds key, int decode_as) { 80 | sds access_key = boxListByDecode(key, decode_as); 81 | robj *akobj = dbstrTake(access_key); 82 | int deleted = 83 | jsonObjGetListFromBufferAndDeleteAll(akobj, DECODE_INDIVIDUAL); 84 | decrRefCount(akobj); 85 | return deleted; 86 | } 87 | 88 | /* Based on box type, either recursively delete more containers or 89 | * return because the current value will be destroyed with this container. */ 90 | static struct jsonObj *jsonObjDeleteFromBox(unsigned char box, 91 | sds populate_as) { 92 | struct jsonObj *o = NULL; 93 | 94 | switch (openBoxAction(box)) { 95 | case BOX_FETCH_MAP_DECODE: 96 | hgetallAndRecursivelyDeleteEverything(populate_as); 97 | break; 98 | case BOX_FETCH_LIST_DECODE: 99 | jsonObjDeleteFromRedisListSdsKey(populate_as, DECODE_INDIVIDUAL); 100 | break; 101 | case BOX_FETCH_MAP_NUMBER: 102 | case BOX_FETCH_LIST_NUMBER: 103 | case BOX_FETCH_MAP_STRING: 104 | case BOX_FETCH_LIST_STRING: 105 | D("deleting container because of homogeneous box [%s]\n", populate_as); 106 | sds actual_key = boxgen(box, populate_as); 107 | robj *akobj = dbstrTake(actual_key); 108 | dbDelete(g.c->db, akobj); 109 | decrRefCount(akobj); 110 | /* can delete list directly */ 111 | break; 112 | case BOX_PARSE_NUMBER_AFTER_BOX: 113 | case BOX_PARSE_STRING_AFTER_BOX: 114 | case BOX_DECODE_IMMEDIATE: 115 | /* These types aren't pointers, so they get deleted 116 | * when the container itself is deleted. */ 117 | break; 118 | default: 119 | D("ERR: Unavailable boxaction: (Boxtype: [%s], boxaction: [%d]\n", 120 | bb(box), openBoxAction(box)); 121 | break; 122 | } 123 | return o; 124 | } 125 | 126 | /* Parse Redis output protocol and extract key/value pairs we need to 127 | * recursively 128 | * delete this map container and all sub-containers too. */ 129 | static int jsonObjDeleteMapFromBuffer(char *buf, int buffer_elements, sds key, 130 | int decode_as) { 131 | sds name = NULL; 132 | char *next = buf; 133 | sds populate_as = NULL; 134 | 135 | /* If buffer_elements is odd, we ignore the last element. */ 136 | /* (Otherise, we end up with a "name" allocated, but it never 137 | * joins gets tied to an object, so it's a memory leak. */ 138 | if (buffer_elements % 2 != 0) { 139 | D("ERROR - Deleting from map with an odd number of elements? Element " 140 | "count: %d\n", 141 | buffer_elements); 142 | buffer_elements--; 143 | } 144 | 145 | /* If we got here without a key, we can't delete the key, so give up. */ 146 | if (!key) 147 | return 0; 148 | 149 | /* Only recursively parse output if we contain sub-containers. 150 | * Else, just delete this hash without any bufer processing. */ 151 | if (decode_as == DECODE_INDIVIDUAL) { 152 | for (int i = 0; i < buffer_elements; i++) { 153 | if (buf[0] != '$') 154 | D("Non bulk type in multibulk! Can't KV this.\n"); 155 | else 156 | buf++; /* skip over '$' */ 157 | 158 | int sz = strtol(buf, &next, 10); 159 | next += SZ_CRLF; 160 | 161 | if (i % 2 == 0) { 162 | name = sdsnewlen(next, sz); 163 | } else { 164 | populate_as = sdsAppendColon(key, name); 165 | jsonObjDeleteFromBox(next[0], populate_as); 166 | sdsfree(populate_as); 167 | sdsfree(name); 168 | } 169 | next += sz + SZ_CRLF; 170 | buf = next; 171 | } 172 | } 173 | 174 | robj *dbkey = dbstr(key); 175 | int deleted = dbDelete(g.c->db, dbkey); 176 | decrRefCount(dbkey); 177 | return deleted; 178 | } 179 | 180 | /* Discover and recurse on multibulk reply contents. 181 | * We assume all multibulk return values are alternating K-V pairs. */ 182 | static int redisKVProtocolRecusivelyDelete(char *reply, sds key, 183 | int decode_as) { 184 | char *p = reply; 185 | char *after = reply + 1; 186 | char *next = NULL; 187 | int sz = strtol(after, &next, 10); 188 | next += SZ_CRLF; /* jump over \r\n */ 189 | 190 | /* The first three are immediate data after their [TYPEID] */ 191 | /* The last two have [TYPEID][SIZE][CRLF] */ 192 | switch (*p) { 193 | case '+': /* Status */ 194 | case '-': /* Error */ 195 | case ':': /* Integer */ 196 | case '$': /* Bulk */ 197 | /* these types can't contain nested containers, so ignore. */ 198 | break; 199 | case '*': /* Multi Bulk */ 200 | return jsonObjDeleteMapFromBuffer(next, sz, key, decode_as); 201 | break; 202 | } 203 | 204 | return 0; 205 | } 206 | 207 | /* Use HGETALL to get an entire hash, the iterate over the hash 208 | * Key/Value pairs to delete any sub-containers. */ 209 | static int hgetallRecursivelyDeleteJsonObj(sds key, int decode_as) { 210 | redisClient *fake_client = g.c; 211 | 212 | fake_client->argc = 2; 213 | robj *argv[2] = {0}; 214 | 215 | D("Access key for read: [%s] (potential box: [%s])\n", key, bb(key[0])); 216 | robj *proper_key = dbstr(key); 217 | 218 | argv[1] = proper_key; 219 | fake_client->argv = argv; 220 | 221 | hgetallCommand(fake_client); 222 | 223 | clearClient(fake_client); 224 | 225 | /* Coalesce client buffer into a single string we can iterate over */ 226 | sds fake_client_buffer = fakeClientResultBuffer(fake_client); 227 | 228 | /* Delete ALL THE THINGS */ 229 | int deleted = redisKVProtocolRecusivelyDelete(fake_client_buffer, 230 | proper_key->ptr, decode_as); 231 | freeFakeClientResultBuffer(fake_client, fake_client_buffer); 232 | decrRefCount(proper_key); 233 | 234 | return deleted; 235 | } 236 | 237 | /* Locate the storage hash for the given key by first trying the two options 238 | * for boxed hashes, then falling back to the key itself. */ 239 | int hgetallAndRecursivelyDeleteEverything(sds key) { 240 | /* We don't know if the top level has will be [BOX]KEY or KEY, so try 241 | * a few first and pick the best one. */ 242 | sds found; 243 | int type; 244 | int decode_as = findKeyForHash(key, &found, &type); 245 | 246 | if (!found) 247 | return 0; 248 | 249 | D("Found hash: [%s] from original key: [%s], decode as: %d\n", found, key, 250 | decode_as); 251 | 252 | int deleted = hgetallRecursivelyDeleteJsonObj(found, decode_as); 253 | sdsfree(found); 254 | return deleted; 255 | } 256 | 257 | static int deleteHashField(sds key, sds field) { 258 | robj *keyobj = dbstr(key); 259 | robj *fieldobj = dbstr(field); 260 | D("Deleting hash field [%s] on key [%s]\n", field, key); 261 | robj *hash = lookupKey(g.c->db, keyobj); 262 | int deleted = hashTypeDelete(hash, fieldobj); 263 | /* If this was the last field in the hash, we have to delete 264 | * the hash key itself too. */ 265 | if (!hashTypeLength(hash)) 266 | dbDelete(g.c->db, keyobj); 267 | decrRefCount(fieldobj); 268 | decrRefCount(keyobj); 269 | return deleted; 270 | } 271 | 272 | /* Generic deletion for fields and not top-level document keys */ 273 | int findAndRecursivelyDelete(sds key, sds field) { 274 | sds found; 275 | int type; 276 | int deleted = 0; 277 | int decode_as = findKeyForHash(key, &found, &type); 278 | 279 | robj *keyobj; 280 | sds combined; 281 | sds found_orig = found; 282 | switch (decode_as) { 283 | case DECODE_ALL_NUMBER: 284 | case DECODE_ALL_STRING: 285 | /* we can hdel directly because no nested containers exist */ 286 | deleted = deleteHashField(found, field); 287 | break; 288 | case DECODE_INDIVIDUAL: 289 | /* If key is a *mixed* CONTAINER type, we can try to construct the 290 | * top-level key of this field and delete it directly */ 291 | /* If the top level key isn't found, then the field either does not 292 | * exist *or* is a non-container-type (string, number, t/f/n) in a 293 | * mixed container. */ 294 | combined = sdsAppendColon(found, field); 295 | decode_as = findKeyForHash(combined, &found, &type); 296 | if (decode_as == DECODE_UNAVAILABLE) { 297 | /* Unavailable = hash not found; search for list */ 298 | decode_as = findKeyForList(combined, &found, &type); 299 | if (decode_as == DECODE_UNAVAILABLE) { 300 | /* Unavailable == List not found, so we can delete the field 301 | * directly 302 | * of this mixed-content hash. */ 303 | deleted = deleteHashField(found_orig, field); 304 | } else { 305 | /* We found a top-level list we can recursively delete */ 306 | keyobj = dbstr(found); 307 | deleted = 308 | jsonObjGetListFromBufferAndDeleteAll(keyobj, decode_as); 309 | if (deleted) 310 | deleteHashField(found_orig, field); 311 | decrRefCount(keyobj); 312 | } 313 | } else { 314 | /* Else, key:field is a top level hash and we can eradicate it. */ 315 | /* it's either a hash or a list. for now, we're being lazy and just 316 | * trying to delete both types. */ 317 | switch (type) { 318 | case JSON_TYPE_MAP: 319 | deleted = hgetallRecursivelyDeleteJsonObj(found, decode_as); 320 | break; 321 | case JSON_TYPE_LIST: 322 | keyobj = dbstr(found); 323 | deleted += 324 | jsonObjGetListFromBufferAndDeleteAll(keyobj, decode_as); 325 | decrRefCount(keyobj); 326 | break; 327 | } 328 | /* If we deleted the value for the field, also delete the field */ 329 | if (deleted) 330 | deleteHashField(found_orig, field); 331 | } 332 | sdsfree(found); 333 | sdsfree(combined); 334 | break; 335 | case DECODE_UNAVAILABLE: 336 | /* NOTE: currently we don't support removing *elements* of lists, but 337 | * we do support removing entire lists, and as a side effect of some of 338 | * these cases, we support turning sub-maps into empty maps ('{}') and 339 | * sub lists into empty lists ('[]') */ 340 | /* First, search for this entire key:field as a list, then delete 341 | * it if it exists. */ 342 | combined = sdsAppendColon(key, field); 343 | decode_as = findKeyForList(combined, &found, &type); 344 | keyobj = dbstr(combined); 345 | switch (decode_as) { 346 | case DECODE_ALL_NUMBER: 347 | case DECODE_ALL_STRING: 348 | /* we can delete this top key directly */ 349 | deleted = dbDelete(g.c->db, keyobj); 350 | break; 351 | case DECODE_INDIVIDUAL: 352 | /* we can recursively delete this entire list */ 353 | deleted = jsonObjGetListFromBufferAndDeleteAll(keyobj, decode_as); 354 | break; 355 | case DECODE_UNAVAILABLE: 356 | /* abort. combined key:field list doesn't exist. */ 357 | break; 358 | } 359 | decrRefCount(keyobj); 360 | sdsfree(found); 361 | sdsfree(combined); 362 | } 363 | sdsfree(found_orig); 364 | return deleted; 365 | } 366 | -------------------------------------------------------------------------------- /json/jsonobj_delete.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_DELETE_H__ 2 | #define __JSONOBJ_DELETE_H__ 3 | 4 | #include "json.h" 5 | #include "jsonobj.h" 6 | #include "jsonobj_box.h" 7 | 8 | int hgetallAndRecursivelyDeleteEverything(sds key); 9 | int findAndRecursivelyDelete(sds key, sds field); 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /json/jsonobj_field_set.c: -------------------------------------------------------------------------------- 1 | #include "jsonobj_field_set.h" 2 | -------------------------------------------------------------------------------- /json/jsonobj_field_set.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_FIELD_SET__ 2 | #define __JSONOBJ_FIELD_SET__ 3 | 4 | #include "json.h" 5 | #include "jsonobj_set.h" 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /json/jsonobj_get.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_GET_H__ 2 | #define __JSONOBJ_GET_H__ 3 | 4 | #include "json.h" 5 | #include "jsonobj.h" 6 | #include "jsonobj_box.h" 7 | #include "jsonobj_delete.h" 8 | #include "i_yajl.h" 9 | 10 | void hgetallToJsonAndAndReply(redisClient *c, sds key); 11 | void genericWrapCommandAndReply(redisClient *c); 12 | void findJsonAndReply(redisClient *c, sds search_key); 13 | void findJsonFieldAndReply(redisClient *c, sds container_key, sds field); 14 | 15 | struct jsonObj *jsonObjFromBoxedBuffer(sds buffer, sds populate_as); 16 | struct jsonObj *hgetToJsonObj(sds key, int decode_as, sds field); 17 | void rpopRecursiveAndReply(redisClient *c, sds key, int decode_as); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /json/jsonobj_set.c: -------------------------------------------------------------------------------- 1 | #include "jsonobj_set.h" 2 | 3 | /* ==================================================================== 4 | * Create nested Redis types from jsonObj map/array 5 | * ==================================================================== */ 6 | /* mostly copied from pushGenericCommand() in t_list.c */ 7 | static bool storeListInsideRedisFromJsonObjFields(struct jsonObj *o, sds id) { 8 | if (!o || o->type != JSON_TYPE_LIST) 9 | return false; 10 | 11 | redisClient *c = g.c_noreturn; 12 | 13 | sds actual_key = boxKeyIfHomogeneous(o, id); 14 | robj *list_key = dbstrTake(actual_key); 15 | 16 | robj *lobj = lookupKeyWrite(c->db, list_key); 17 | 18 | /* If the list already exists, we're going to delete it so we 19 | * have a fresh start for our blanket add. If someone adds the same 20 | * document again, they should get the document they set without 21 | * any influence from past elements. */ 22 | if (lobj) { 23 | dbDelete(c->db, lobj); 24 | lobj = NULL; 25 | } 26 | 27 | /* We are guaranteed lobj doesn't exist here because if we 28 | * found it above, we deleted it. */ 29 | if (!lobj) { 30 | lobj = createZiplistObject(); 31 | dbAdd(c->db, list_key, lobj); 32 | } 33 | decrRefCount(list_key); 34 | 35 | for (int j = 0; j < o->content.obj.elements; j++) { 36 | struct jsonObj *field = o->content.obj.fields[j]; 37 | robj *ro = dbstr(field->content.string); 38 | ro = tryObjectEncoding(ro); 39 | listTypePush(lobj, ro, REDIS_TAIL); 40 | decrRefCount(ro); 41 | } 42 | return true; 43 | } 44 | 45 | /* Direct Mode assumes zero nesting of types. The jsonObj is *only* top level 46 | * string field->value pairs. */ 47 | static void hmsetDirectFromJson(struct jsonObj *r, sds key) { 48 | redisClient *c = g.c_noreturn; 49 | 50 | int total_pairs = r->content.obj.elements; 51 | c->argc = total_pairs * 2 + 2; /* +2 = [cmd, hash key] */ 52 | robj *argv[c->argc]; 53 | 54 | sds total_key = boxKeyIfHomogeneous(r, key); 55 | argv[0] = NULL; 56 | argv[1] = dbstrTake(total_key); 57 | 58 | D("Setting direct %s with %d pairs\n", key, total_pairs); 59 | for (int i = 0; i < total_pairs; i++) { 60 | struct jsonObj *o = r->content.obj.fields[i]; 61 | 62 | if (!o) 63 | D("BAD—OBJECT IS NULL AT %d\n", i); 64 | 65 | if (o->type == JSON_TYPE_PTR) 66 | D("[%s] (%s) -> [%s]\n", key, o->name, bb(*o->content.string)); 67 | else 68 | D("[%s] (%s) -> (%s)\n", key, o->name, o->content.string); 69 | 70 | argv[(i * 2) + 2] = dbstr(o->name); 71 | 72 | switch (o->type) { 73 | case JSON_TYPE_PTR: 74 | case JSON_TYPE_STRING: 75 | case JSON_TYPE_NUMBER_AS_STRING: 76 | argv[(i * 2) + 3] = dbstr(o->content.string); 77 | break; 78 | default: 79 | D("SETTING NON-STRING (%d) IN HMSET at %d for name: %s!\n", o->type, 80 | i, o->name); 81 | break; 82 | } 83 | } 84 | c->argv = argv; 85 | hmsetCommand(c); 86 | for (int i = 1; i < c->argc; i++) { 87 | decrRefCount(argv[i]); 88 | } 89 | clearClient(c); 90 | } 91 | 92 | /* Shrink jsonObj fields to (maybe boxed) strings for simple types */ 93 | static bool compactRepresentation(struct jsonObj *o) { 94 | switch (o->type) { 95 | case JSON_TYPE_MAP: 96 | case JSON_TYPE_LIST: 97 | /* If we are a container type, process the container below. */ 98 | break; 99 | default: 100 | /* Else, we have an immediate type. */ 101 | jsonObjBoxBasicType(o); 102 | return true; 103 | break; 104 | } 105 | 106 | if (o->content.obj.homogeneous) { 107 | switch (o->content.obj.subtype) { 108 | case JSON_TYPE_STRING: 109 | case JSON_TYPE_NUMBER_AS_STRING: 110 | /* If our fields are homogeneous strings or numbers, we use no 111 | * boxing 112 | * because the box type is promoted to the front of the key for this 113 | * container, not for the individual values themselves. */ 114 | return true; 115 | break; 116 | case JSON_TYPE_NUMBER: 117 | /* Not entirely sure if we ever encode NUMBER directly. We always 118 | * seem to use NUMBER_AS_STRING. Double check. */ 119 | case JSON_TYPE_TRUE: 120 | case JSON_TYPE_FALSE: 121 | case JSON_TYPE_NULL: 122 | /* for True/False/Null homogeneous, we could avoid encoding them 123 | * directly and just append the number to return in the list 124 | * when we are reading. That would break someone trying to 125 | * update the list itself though. */ 126 | for (int i = 0; i < o->content.obj.elements; i++) { 127 | jsonObjBoxBasicType(o->content.obj.fields[i]); 128 | } 129 | return true; 130 | break; 131 | case JSON_TYPE_MAP: 132 | case JSON_TYPE_LIST: 133 | /* These cases would catch a container with only lists or only 134 | * maps. 135 | * It doesn't make sense to have a container type tagged as being 136 | * homogeneous with other container types, since you have to iterate 137 | * over the sub-containers anyway. */ 138 | break; 139 | } 140 | } 141 | return false; 142 | } 143 | 144 | /* Generate the correct depth key for finding sub-containers of 145 | * higher level containers. */ 146 | static sds populateKey(sds base, struct jsonObj *root, struct jsonObj *field, 147 | int i) { 148 | sds populate_as = NULL; 149 | sds position = NULL; 150 | 151 | /* If we have a name, we're making sub-types */ 152 | if (field->name) { 153 | /* as we recurse more, this grows as: ID:subID1:subID2:...:subIDk */ 154 | populate_as = sdsAppendColon(base, field->name); 155 | } else { 156 | /* If we are putting a map or list inside of a list, we need a special 157 | * positional designator. 158 | * Lists are guaranteed to not have names themselves. */ 159 | /* If anybody inserts or delets from the list, we must rename all our 160 | * positional designators in the list. */ 161 | switch (root->type) { 162 | case JSON_TYPE_LIST: 163 | switch (field->type) { 164 | case JSON_TYPE_MAP: 165 | case JSON_TYPE_LIST: 166 | position = sdsfromlonglong(i); 167 | populate_as = sdsAppendColon(base, position); 168 | sdsfree(position); 169 | break; 170 | } 171 | } 172 | } 173 | 174 | if (populate_as) 175 | return populate_as; 176 | else 177 | return sdsdup(base); 178 | } 179 | 180 | /* Take a jsonObj and create it in Redis as a tree of hashes/lists as 181 | * appropriate. */ 182 | static bool hmsetIndirectFromJson(struct jsonObj *root, sds id) { 183 | /* If the root type isn't a homogeneous container, process all sub-fields */ 184 | if (!compactRepresentation(root)) { 185 | /* This object is either a map or a list. */ 186 | int elements = root->content.obj.elements; 187 | 188 | D("Element count: %d\n", elements); 189 | /* For each field in this map or list... */ 190 | for (int i = 0; i < elements; i++) { 191 | struct jsonObj *o = root->content.obj.fields[i]; 192 | sds populate_as = populateKey(id, root, o, i); 193 | 194 | D("Root type, element type and subname: %d, %d, %s\n", root->type, 195 | o->type, populate_as); 196 | 197 | /* Actions: - if map or list, recursively create sub-containers 198 | * - if string, number, true, false, null: box in place. */ 199 | switch (o->type) { 200 | case JSON_TYPE_LIST: 201 | D("TYPE L\n"); 202 | case JSON_TYPE_MAP: 203 | D("TYPE M\n"); 204 | hmsetIndirectFromJson(o, populate_as); 205 | root->content.obj.fields[i] = jsonObjConvertToPtr(o); 206 | break; 207 | default: 208 | D("type D\n"); 209 | jsonObjBoxBasicType(o); 210 | break; 211 | } 212 | sdsfree(populate_as); 213 | } 214 | } 215 | #ifndef PRODUCTION 216 | /* D is too noisy and we need spacing between iterations. Only do this 217 | * double line printf if we are in D-debug mode. */ 218 | printf("\n\n"); 219 | #endif 220 | /* Now, all the way down here, every field inside the root jsonObj hierarchy 221 | * has been created as proper types, and their entires have been replaced 222 | * with boxed type pointers all the way up the hierarchy. Now we are 223 | * guaranteed a few things: the json is now one level of fields with boxed 224 | * pointers - OR - this is a homogeneous container with only strings 225 | * from the beginning. */ 226 | 227 | switch (root->type) { 228 | case JSON_TYPE_MAP: 229 | D("Top level ID for Map: %s\n", id); 230 | hmsetDirectFromJson(root, id); 231 | break; 232 | case JSON_TYPE_LIST: 233 | D("Top level ID for List: %s\n", id); 234 | storeListInsideRedisFromJsonObjFields(root, id); 235 | break; 236 | } 237 | return true; 238 | } 239 | 240 | /* Public interface for serializing a jsonObj to Redis */ 241 | void jsonObjToRedisDB(robj *id, struct jsonObj *r) { 242 | hmsetIndirectFromJson(r, id->ptr); 243 | } 244 | -------------------------------------------------------------------------------- /json/jsonobj_set.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSONOBJ_SET_H__ 2 | #define __JSONOBJ_SET_H__ 3 | 4 | #include "json.h" 5 | #include "jsonobj.h" 6 | #include "jsonobj_box.h" 7 | 8 | void jsonObjToRedisDB(robj *id, struct jsonObj *r); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /poong/poong.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | 3 | void pingCommandLocal(redisClient *c) { 4 | sds pooong = sdsnew("+POOOOOOOOOONG\r\n"); 5 | /* Redis uses refcounted objects. You must decrement the refcount 6 | * every time you create an object. */ 7 | robj *o = createObject(REDIS_STRING, pooong); /* refcount + 1 */ 8 | /* addReply retains the object too. The object will be free'd 9 | * after addReply is done with it. */ 10 | addReply(c, o); 11 | decrRefCount(o); /* refcount - 1 */ 12 | } 13 | 14 | void *load() { 15 | fprintf(stderr, "%s: Loaded at %d\n", __FILE__, __LINE__); 16 | return NULL; 17 | } 18 | 19 | void cleanup(void *privdata) { 20 | fprintf(stderr, "%s: Cleaning up at %d\n", __FILE__, __LINE__); 21 | } 22 | 23 | struct redisModule redisModuleDetail = { 24 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 25 | REDIS_VERSION, /* Provided by redis.h */ 26 | "0.0001", /* Version of this module (only for reporting) */ 27 | "sh.matt.test.pong", /* Unique name of this module */ 28 | load, /* Load function pointer (optional) */ 29 | cleanup /* Cleanup function pointer (optional) */ 30 | }; 31 | 32 | struct redisCommand redisCommandTable[] = { 33 | /* See the primary command table in redis.c for details 34 | * about what each field means. 35 | * Some fields are more important than others. You can't 36 | * just copy and paste an existing field into a new field 37 | * and expect it to work. */ 38 | /* name, function, arity, sflags, [internal], keyFunction, 39 | * keyFirstPos, keyLastPos, keyStep, [internal], [internal] */ 40 | {"poong", pingCommandLocal, 1, "rt", 0, NULL, 0, 0, 0, 0, 0}, 41 | {"pooooong", pingCommandLocal, 1, "rt", 0, NULL, 0, 0, 0, 0, 0}, 42 | {"pooong", pingCommandLocal, 1, "rt", 0, NULL, 0, 0, 0, 0, 0}, 43 | {"pinger", pingCommand, 1, "rt", 0, NULL, 0, 0, 0, 0, 44 | 0}, /* function from Redis */ 45 | {0} /* Always end your command table with {0} 46 | * If you forget, you will be reminded with a segfault on load. */ 47 | }; 48 | -------------------------------------------------------------------------------- /scriptname/scriptname.c: -------------------------------------------------------------------------------- 1 | #include "redis.h" 2 | 3 | /* ==================================================================== 4 | * Redis Add-on Module: scriptname 5 | * Provides commands: scriptname, evalname 6 | * Behaviors: 7 | * - scriptName - bind name to a loaded script SHA (CRUD) 8 | * - evalName - run a Lua script by name instead of SHA 9 | * ==================================================================== */ 10 | 11 | /* Global things for this module */ 12 | struct global { 13 | dict *names; /* Map of name -> SHA */ 14 | struct err { 15 | robj *nosha, *noname; 16 | } err; 17 | }; 18 | 19 | /* ==================================================================== 20 | * Global struct to store our persistent configuration. 21 | * If this wasn't a module, we'd use the global 'server' struct instead. 22 | * ==================================================================== */ 23 | static struct global g = {0}; 24 | 25 | /* ==================================================================== 26 | * Dict Helpers 27 | * ==================================================================== */ 28 | static void *dictSdsDup(void *privdata, const void *string) { 29 | DICT_NOTUSED(privdata); 30 | return sdsnew(string); 31 | } 32 | 33 | /* Script name -> script sha dict */ 34 | /* Note how we assign every dup and free pointer so we don't 35 | * have to manually create or free objects we put in the dict. */ 36 | static dictType namesTableDictType = { 37 | dictSdsCaseHash, /* hash function */ 38 | dictSdsDup, /* key dup */ 39 | dictSdsDup, /* val dup */ 40 | dictSdsKeyCompare, /* key compare */ 41 | dictSdsDestructor, /* key destructor */ 42 | dictSdsDestructor /* val destructor */ 43 | }; 44 | 45 | /* ==================================================================== 46 | * Command Implementations 47 | * ==================================================================== */ 48 | /* scriptNameCommand() has compound sub-arguments, so it looks slightly more 49 | * convoluted than it actually is. Just read each if/else branch as 50 | * if it were an individual command. */ 51 | void scriptNameCommand(redisClient *c) { 52 | char *req = c->argv[1]->ptr; 53 | sds script_name = c->argv[2]->ptr; 54 | 55 | if (c->argc == 4 && !strcasecmp(req, "set")) { 56 | sds target_sha = c->argv[3]->ptr; 57 | 58 | if (sdslen(target_sha) != 40 || 59 | dictFind(server.lua_scripts, target_sha) == NULL) { 60 | addReply(c, g.err.nosha); 61 | return; 62 | } 63 | 64 | /* If name doesn't exist, dictReplace == dictAdd */ 65 | dictReplace(g.names, script_name, target_sha); 66 | 67 | addReplyBulkCBuffer(c, script_name, sdslen(script_name)); 68 | } else if (c->argc == 3 && !strcasecmp(req, "get")) { 69 | sds found; 70 | if ((found = dictFetchValue(g.names, script_name))) { 71 | addReplyBulkCBuffer(c, found, sdslen(found)); 72 | } else { 73 | addReply(c, g.err.noname); 74 | } 75 | } else if (c->argc == 2 && !strcasecmp(req, "getall")) { 76 | dictIterator *di; 77 | dictEntry *de; 78 | 79 | unsigned long sz = dictSize(g.names); 80 | 81 | if (!sz) { 82 | addReply(c, shared.emptymultibulk); 83 | return; 84 | } 85 | 86 | /* Multiply by 2 because the size of the dict is the number of keys. 87 | * We are returning keys *and* values, so length is dictSize * 2 */ 88 | addReplyMultiBulkLen(c, sz * 2); 89 | 90 | di = dictGetIterator(g.names); 91 | while ((de = dictNext(di))) { 92 | addReplyBulkCString(c, dictGetKey(de)); 93 | addReplyBulkCString(c, dictGetVal(de)); 94 | } 95 | dictReleaseIterator(di); 96 | } else if (c->argc == 3 && !strcasecmp(req, "del")) { 97 | sds deleted; 98 | 99 | if ((deleted = dictFetchValue(g.names, script_name))) { 100 | dictDelete(g.names, script_name); 101 | addReplyBulkCBuffer(c, deleted, sdslen(deleted)); 102 | } else { 103 | addReply(c, g.err.noname); 104 | } 105 | } else { 106 | addReplyError(c, "Unknown scriptName subcommand or arg count"); 107 | } 108 | } 109 | 110 | void evalName(redisClient *c, char *name) { 111 | sds hash; 112 | robj *sha; 113 | 114 | /* if len(name) == 40, then this *could* be a script hash. 115 | * Try to look it up as a hash first. */ 116 | if (sdslen(name) == 40 && dictFind(server.lua_scripts, name)) { 117 | hash = name; 118 | } else { 119 | hash = dictFetchValue(g.names, name); 120 | /* If we didn't find it, or if we did find it but for some 121 | * reason the len(hash) != 40, ... */ 122 | if (!hash || sdslen(hash) != 40) { 123 | addReply(c, g.err.nosha); 124 | return; 125 | } 126 | } 127 | 128 | /* the redisClient object uses robj * for fields, 129 | * so here we convert the sds hash into an robj of 130 | * an sds */ 131 | sha = createStringObject(hash, sdslen(hash)); 132 | rewriteClientCommandArgument(c, 1, sha); 133 | decrRefCount(sha); /* No leaking memory from our created robj */ 134 | 135 | /* Now run the script as normal, just like we never existed. *poof* */ 136 | evalGenericCommand(c, 1); 137 | } 138 | 139 | void evalNameCommand(redisClient *c) { 140 | evalName(c, c->argv[1]->ptr); 141 | } 142 | 143 | /* ==================================================================== 144 | * Bring up / Teardown 145 | * ==================================================================== */ 146 | void *load() { 147 | g.names = dictCreate(&namesTableDictType, NULL); 148 | g.err.nosha = createObject( 149 | REDIS_STRING, 150 | sdsnew("-NOSCRIPT Target SHA not found. Please use SCRIPT LOAD.\r\n")); 151 | g.err.noname = createObject( 152 | REDIS_STRING, 153 | sdsnew( 154 | "-NONAME Script name not found. Please use SCRIPTNAME SET.\r\n")); 155 | return NULL; 156 | } 157 | 158 | /* If you reload the module *without* freeing things you allocate in load(), 159 | * then you *will* introduce memory leaks. */ 160 | void cleanup(void *privdata) { 161 | /* dictRelease will free every key, every value, then the dict itself. */ 162 | dictRelease(g.names); 163 | decrRefCount(g.err.nosha); 164 | decrRefCount(g.err.noname); 165 | } 166 | 167 | /* ==================================================================== 168 | * Dynamic Redis API Requirements 169 | * ==================================================================== */ 170 | struct redisModule redisModuleDetail = { 171 | REDIS_MODULE_COMMAND, /* Tell Dynamic Redis our module provides commands */ 172 | REDIS_VERSION, /* Provided by redis.h */ 173 | "0.3", /* Version of this module (only for reporting) */ 174 | "sh.matt.scriptName", /* Unique name of this module */ 175 | load, /* Load function pointer (optional) */ 176 | cleanup /* Cleanup function pointer (optional) */ 177 | }; 178 | 179 | struct redisCommand redisCommandTable[] = { 180 | {"scriptName", scriptNameCommand, -2, "s", 0, NULL, 0, 0, 0, 0, 0}, 181 | #if DYN_FEATURE_CLUSTER == false 182 | {"evalName", evalNameCommand, -3, "s", 0, NULL, 0, 0, 0, 0, 0}, 183 | #else 184 | /* evalGetKeys exists for Cluster in redis-unstable */ 185 | {"evalName", evalNameCommand, -3, "s", 0, evalGetKeys, 0, 0, 0, 0, 0}, 186 | #endif 187 | {0} /* Always end your command table with {0} 188 | * If you forget, you will be reminded with a segfault on load. */ 189 | }; 190 | --------------------------------------------------------------------------------