├── .gitignore ├── Makefile ├── README.md ├── c_src ├── erl_nif.h ├── erl_nif_compat.h ├── lzf.h ├── lzfP.h ├── lzf_c.c ├── lzf_d.c └── nsync_nifs.c ├── rebar └── src ├── lzf.erl ├── nsync.app.src ├── nsync.erl ├── nsync.hrl ├── nsync_utils.erl ├── rdb_load.erl └── redis_text_proto.erl /.gitignore: -------------------------------------------------------------------------------- 1 | ebin/* 2 | src/*.swp 3 | src/*/*.swp 4 | c_src/*.o 5 | priv/*.so 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: compile 2 | 3 | compile: 4 | @./rebar compile 5 | 6 | clean: 7 | @./rebar clean 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | nsync is a redis master/slave replication client written in Erlang. When the nsync process starts, it opens a tcp socket connected to the master redis server and issues a "SYNC" command. The redis master asynchronously dumps its dataset to disk and transfers it to the nsync slave as an rdb dump. The master then streams subsequent updates to the slave using the redis text protocol. 4 | 5 | # Build 6 | 7 | $ ./rebar compile 8 | 9 | # Run 10 | 11 | $ erl -pa ebin 12 | $ nsync:start_link(). 13 | 14 | # Options 15 | 16 | nsync:start_link([Opt]). 17 | Opt = {host, Host} | 18 | {port, Port} | 19 | {pass, Pass} | 20 | {callback, Callback} | 21 | {block, Block} | 22 | {timeout, Timeout} 23 | Host = list() % redis master host 24 | Port = integer() % redis master port 25 | Pass = list() % redis master password 26 | Block = boolean() % block calling pid during sync process 27 | Timeout = integer() % timeout of blocking sync process 28 | Callback = {M, F} | {M, F, A} | Fun 29 | M = atom() 30 | F = atom() 31 | A = list() 32 | Fun = fun() 33 | -------------------------------------------------------------------------------- /c_src/erl_nif.h: -------------------------------------------------------------------------------- 1 | /* 2 | * %CopyrightBegin% 3 | * 4 | * Copyright Ericsson AB 2009-2010. All Rights Reserved. 5 | * 6 | * The contents of this file are subject to the Erlang Public License, 7 | * Version 1.1, (the "License"); you may not use this file except in 8 | * compliance with the License. You should have received a copy of the 9 | * Erlang Public License along with this software. If not, it can be 10 | * retrieved online at http://www.erlang.org/. 11 | * 12 | * Software distributed under the License is distributed on an "AS IS" 13 | * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 14 | * the License for the specific language governing rights and limitations 15 | * under the License. 16 | * 17 | * %CopyrightEnd% 18 | */ 19 | 20 | /* Include file for writers of Native Implemented Functions. 21 | */ 22 | 23 | #ifndef __ERL_NIF_H__ 24 | #define __ERL_NIF_H__ 25 | 26 | 27 | #include "erl_drv_nif.h" 28 | 29 | /* Version history: 30 | ** 0.1: R13B03 31 | ** 1.0: R13B04 32 | ** 2.0: R14A 33 | */ 34 | #define ERL_NIF_MAJOR_VERSION 2 35 | #define ERL_NIF_MINOR_VERSION 0 36 | 37 | #include 38 | 39 | #ifdef SIZEOF_CHAR 40 | # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR 41 | # undef SIZEOF_CHAR 42 | #endif 43 | #ifdef SIZEOF_SHORT 44 | # define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT 45 | # undef SIZEOF_SHORT 46 | #endif 47 | #ifdef SIZEOF_INT 48 | # define SIZEOF_INT_SAVED__ SIZEOF_INT 49 | # undef SIZEOF_INT 50 | #endif 51 | #ifdef SIZEOF_LONG 52 | # define SIZEOF_LONG_SAVED__ SIZEOF_LONG 53 | # undef SIZEOF_LONG 54 | #endif 55 | #ifdef SIZEOF_LONG_LONG 56 | # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG 57 | # undef SIZEOF_LONG_LONG 58 | #endif 59 | #ifdef HALFWORD_HEAP_EMULATOR 60 | # define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR 61 | # undef HALFWORD_HEAP_EMULATOR 62 | #endif 63 | #include "erl_int_sizes_config.h" 64 | 65 | #ifdef __cplusplus 66 | extern "C" { 67 | #endif 68 | 69 | #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) 70 | typedef unsigned __int64 ErlNifUInt64; 71 | typedef __int64 ErlNifSInt64; 72 | #elif SIZEOF_LONG == 8 73 | typedef unsigned long ErlNifUInt64; 74 | typedef long ErlNifSInt64; 75 | #elif SIZEOF_LONG_LONG == 8 76 | typedef unsigned long long ErlNifUInt64; 77 | typedef long long ErlNifSInt64; 78 | #else 79 | #error No 64-bit integer type 80 | #endif 81 | 82 | #ifdef HALFWORD_HEAP_EMULATOR 83 | typedef unsigned int ERL_NIF_TERM; 84 | #else 85 | typedef unsigned long ERL_NIF_TERM; 86 | #endif 87 | 88 | struct enif_environment_t; 89 | typedef struct enif_environment_t ErlNifEnv; 90 | 91 | typedef struct 92 | { 93 | const char* name; 94 | unsigned arity; 95 | ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); 96 | }ErlNifFunc; 97 | 98 | typedef struct enif_entry_t 99 | { 100 | int major; 101 | int minor; 102 | const char* name; 103 | int num_of_funcs; 104 | ErlNifFunc* funcs; 105 | int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); 106 | int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); 107 | int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); 108 | void (*unload) (ErlNifEnv*, void* priv_data); 109 | }ErlNifEntry; 110 | 111 | 112 | 113 | typedef struct 114 | { 115 | size_t size; 116 | unsigned char* data; 117 | 118 | /* Internals (avert your eyes) */ 119 | ERL_NIF_TERM bin_term; 120 | void* ref_bin; 121 | }ErlNifBinary; 122 | 123 | typedef struct enif_resource_type_t ErlNifResourceType; 124 | typedef void ErlNifResourceDtor(ErlNifEnv*, void*); 125 | typedef enum 126 | { 127 | ERL_NIF_RT_CREATE = 1, 128 | ERL_NIF_RT_TAKEOVER = 2 129 | }ErlNifResourceFlags; 130 | 131 | typedef enum 132 | { 133 | ERL_NIF_LATIN1 = 1 134 | }ErlNifCharEncoding; 135 | 136 | typedef struct 137 | { 138 | ERL_NIF_TERM pid; /* internal, may change */ 139 | }ErlNifPid; 140 | 141 | typedef ErlDrvSysInfo ErlNifSysInfo; 142 | 143 | typedef struct ErlDrvTid_ *ErlNifTid; 144 | typedef struct ErlDrvMutex_ ErlNifMutex; 145 | typedef struct ErlDrvCond_ ErlNifCond; 146 | typedef struct ErlDrvRWLock_ ErlNifRWLock; 147 | typedef int ErlNifTSDKey; 148 | 149 | typedef ErlDrvThreadOpts ErlNifThreadOpts; 150 | 151 | #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) 152 | # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS 153 | typedef struct { 154 | # include "erl_nif_api_funcs.h" 155 | } TWinDynNifCallbacks; 156 | extern TWinDynNifCallbacks WinDynNifCallbacks; 157 | # undef ERL_NIF_API_FUNC_DECL 158 | #endif 159 | 160 | #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) 161 | # define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME) 162 | # include "erl_nif_api_funcs.h" 163 | /* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */ 164 | 165 | #else /* non windows or included from emulator itself */ 166 | 167 | # define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) extern RET_TYPE NAME ARGS 168 | # include "erl_nif_api_funcs.h" 169 | # undef ERL_NIF_API_FUNC_DECL 170 | #endif 171 | 172 | 173 | #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) 174 | # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; 175 | # define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) 176 | # define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) 177 | #else 178 | # define ERL_NIF_INIT_GLOB 179 | # define ERL_NIF_INIT_BODY 180 | # if defined(VXWORKS) 181 | # define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _init(void) 182 | # else 183 | # define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) 184 | # endif 185 | #endif 186 | 187 | 188 | #ifdef __cplusplus 189 | } 190 | # define ERL_NIF_INIT_PROLOGUE extern "C" { 191 | # define ERL_NIF_INIT_EPILOGUE } 192 | #else 193 | # define ERL_NIF_INIT_PROLOGUE 194 | # define ERL_NIF_INIT_EPILOGUE 195 | #endif 196 | 197 | 198 | #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ 199 | ERL_NIF_INIT_PROLOGUE \ 200 | ERL_NIF_INIT_GLOB \ 201 | ERL_NIF_INIT_DECL(NAME) \ 202 | { \ 203 | static ErlNifEntry entry = \ 204 | { \ 205 | ERL_NIF_MAJOR_VERSION, \ 206 | ERL_NIF_MINOR_VERSION, \ 207 | #NAME, \ 208 | sizeof(FUNCS) / sizeof(*FUNCS), \ 209 | FUNCS, \ 210 | LOAD, RELOAD, UPGRADE, UNLOAD \ 211 | }; \ 212 | ERL_NIF_INIT_BODY; \ 213 | return &entry; \ 214 | } \ 215 | ERL_NIF_INIT_EPILOGUE 216 | 217 | 218 | #endif /* __ERL_NIF_H__ */ 219 | 220 | -------------------------------------------------------------------------------- /c_src/erl_nif_compat.h: -------------------------------------------------------------------------------- 1 | #ifndef ERL_NIF_COMPAT_H_ 2 | #define ERL_NIF_COMPAT_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif /* __cplusplus */ 7 | 8 | #include "erl_nif.h" 9 | 10 | #if ERL_NIF_MAJOR_VERSION == 1 && ERL_NIF_MINOR_VERSION == 0 11 | 12 | #define enif_open_resource_type_compat enif_open_resource_type 13 | #define enif_alloc_resource_compat enif_alloc_resource 14 | #define enif_release_resource_compat enif_release_resource 15 | #define enif_alloc_binary_compat enif_alloc_binary 16 | #define enif_realloc_binary_compat enif_realloc_binary 17 | #define enif_release_binary_compat enif_release_binary 18 | #define enif_alloc_compat enif_alloc 19 | #define enif_free_compat enif_free 20 | 21 | #endif /* R13B04 */ 22 | 23 | #if ERL_NIF_MAJOR_VERSION == 2 && ERL_NIF_MINOR_VERSION == 0 24 | 25 | #define enif_open_resource_type_compat(E, N, D, F, T) \ 26 | enif_open_resource_type(E, NULL, N, D, F, T) 27 | 28 | #define enif_alloc_resource_compat(E, T, S) \ 29 | enif_alloc_resource(T, S) 30 | 31 | #define enif_release_resource_compat(E, H) \ 32 | enif_release_resource(H) 33 | 34 | #define enif_alloc_binary_compat(E, S, B) \ 35 | enif_alloc_binary(S, B) 36 | 37 | #define enif_release_binary_compat(E, B) \ 38 | enif_release_binary(B) 39 | 40 | #define enif_realloc_binary_compat(E, B, S) \ 41 | enif_realloc_binary(B, S) 42 | 43 | #define enif_alloc_compat(E, S) \ 44 | enif_alloc(S) 45 | 46 | #define enif_free_compat(E, P) \ 47 | enif_free(P) 48 | 49 | #endif /* R14 */ 50 | 51 | 52 | 53 | #ifdef __cplusplus 54 | } 55 | #endif /* __cplusplus */ 56 | 57 | #endif /* ERL_NIF_COMPAT_H_ */ 58 | -------------------------------------------------------------------------------- /c_src/lzf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2000-2008 Marc Alexander Lehmann 3 | * 4 | * Redistribution and use in source and binary forms, with or without modifica- 5 | * tion, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- 16 | * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- 18 | * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- 22 | * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 23 | * OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * Alternatively, the contents of this file may be used under the terms of 26 | * the GNU General Public License ("GPL") version 2 or any later version, 27 | * in which case the provisions of the GPL are applicable instead of 28 | * the above. If you wish to allow the use of your version of this file 29 | * only under the terms of the GPL and not to allow others to use your 30 | * version of this file under the BSD license, indicate your decision 31 | * by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL. If you do not delete the 33 | * provisions above, a recipient may use your version of this file under 34 | * either the BSD or the GPL. 35 | */ 36 | 37 | #ifndef LZF_H 38 | #define LZF_H 39 | 40 | /*********************************************************************** 41 | ** 42 | ** lzf -- an extremely fast/free compression/decompression-method 43 | ** http://liblzf.plan9.de/ 44 | ** 45 | ** This algorithm is believed to be patent-free. 46 | ** 47 | ***********************************************************************/ 48 | 49 | #define LZF_VERSION 0x0105 /* 1.5, API version */ 50 | 51 | /* 52 | * Compress in_len bytes stored at the memory block starting at 53 | * in_data and write the result to out_data, up to a maximum length 54 | * of out_len bytes. 55 | * 56 | * If the output buffer is not large enough or any error occurs return 0, 57 | * otherwise return the number of bytes used, which might be considerably 58 | * more than in_len (but less than 104% of the original size), so it 59 | * makes sense to always use out_len == in_len - 1), to ensure _some_ 60 | * compression, and store the data uncompressed otherwise (with a flag, of 61 | * course. 62 | * 63 | * lzf_compress might use different algorithms on different systems and 64 | * even different runs, thus might result in different compressed strings 65 | * depending on the phase of the moon or similar factors. However, all 66 | * these strings are architecture-independent and will result in the 67 | * original data when decompressed using lzf_decompress. 68 | * 69 | * The buffers must not be overlapping. 70 | * 71 | * If the option LZF_STATE_ARG is enabled, an extra argument must be 72 | * supplied which is not reflected in this header file. Refer to lzfP.h 73 | * and lzf_c.c. 74 | * 75 | */ 76 | unsigned int 77 | lzf_compress (const void *const in_data, unsigned int in_len, 78 | void *out_data, unsigned int out_len); 79 | 80 | /* 81 | * Decompress data compressed with some version of the lzf_compress 82 | * function and stored at location in_data and length in_len. The result 83 | * will be stored at out_data up to a maximum of out_len characters. 84 | * 85 | * If the output buffer is not large enough to hold the decompressed 86 | * data, a 0 is returned and errno is set to E2BIG. Otherwise the number 87 | * of decompressed bytes (i.e. the original length of the data) is 88 | * returned. 89 | * 90 | * If an error in the compressed data is detected, a zero is returned and 91 | * errno is set to EINVAL. 92 | * 93 | * This function is very fast, about as fast as a copying loop. 94 | */ 95 | unsigned int 96 | lzf_decompress (const void *const in_data, unsigned int in_len, 97 | void *out_data, unsigned int out_len); 98 | 99 | #endif 100 | 101 | -------------------------------------------------------------------------------- /c_src/lzfP.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2000-2007 Marc Alexander Lehmann 3 | * 4 | * Redistribution and use in source and binary forms, with or without modifica- 5 | * tion, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- 16 | * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- 18 | * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- 22 | * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 23 | * OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * Alternatively, the contents of this file may be used under the terms of 26 | * the GNU General Public License ("GPL") version 2 or any later version, 27 | * in which case the provisions of the GPL are applicable instead of 28 | * the above. If you wish to allow the use of your version of this file 29 | * only under the terms of the GPL and not to allow others to use your 30 | * version of this file under the BSD license, indicate your decision 31 | * by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL. If you do not delete the 33 | * provisions above, a recipient may use your version of this file under 34 | * either the BSD or the GPL. 35 | */ 36 | 37 | #ifndef LZFP_h 38 | #define LZFP_h 39 | 40 | #define STANDALONE 1 /* at the moment, this is ok. */ 41 | 42 | #ifndef STANDALONE 43 | # include "lzf.h" 44 | #endif 45 | 46 | /* 47 | * Size of hashtable is (1 << HLOG) * sizeof (char *) 48 | * decompression is independent of the hash table size 49 | * the difference between 15 and 14 is very small 50 | * for small blocks (and 14 is usually a bit faster). 51 | * For a low-memory/faster configuration, use HLOG == 13; 52 | * For best compression, use 15 or 16 (or more, up to 23). 53 | */ 54 | #ifndef HLOG 55 | # define HLOG 16 56 | #endif 57 | 58 | /* 59 | * Sacrifice very little compression quality in favour of compression speed. 60 | * This gives almost the same compression as the default code, and is 61 | * (very roughly) 15% faster. This is the preferred mode of operation. 62 | */ 63 | #ifndef VERY_FAST 64 | # define VERY_FAST 1 65 | #endif 66 | 67 | /* 68 | * Sacrifice some more compression quality in favour of compression speed. 69 | * (roughly 1-2% worse compression for large blocks and 70 | * 9-10% for small, redundant, blocks and >>20% better speed in both cases) 71 | * In short: when in need for speed, enable this for binary data, 72 | * possibly disable this for text data. 73 | */ 74 | #ifndef ULTRA_FAST 75 | # define ULTRA_FAST 0 76 | #endif 77 | 78 | /* 79 | * Unconditionally aligning does not cost very much, so do it if unsure 80 | */ 81 | #ifndef STRICT_ALIGN 82 | # define STRICT_ALIGN !(defined(__i386) || defined (__amd64)) 83 | #endif 84 | 85 | /* 86 | * You may choose to pre-set the hash table (might be faster on some 87 | * modern cpus and large (>>64k) blocks, and also makes compression 88 | * deterministic/repeatable when the configuration otherwise is the same). 89 | */ 90 | #ifndef INIT_HTAB 91 | # define INIT_HTAB 0 92 | #endif 93 | 94 | /* 95 | * Avoid assigning values to errno variable? for some embedding purposes 96 | * (linux kernel for example), this is neccessary. NOTE: this breaks 97 | * the documentation in lzf.h. 98 | */ 99 | #ifndef AVOID_ERRNO 100 | # define AVOID_ERRNO 0 101 | #endif 102 | 103 | /* 104 | * Wether to pass the LZF_STATE variable as argument, or allocate it 105 | * on the stack. For small-stack environments, define this to 1. 106 | * NOTE: this breaks the prototype in lzf.h. 107 | */ 108 | #ifndef LZF_STATE_ARG 109 | # define LZF_STATE_ARG 0 110 | #endif 111 | 112 | /* 113 | * Wether to add extra checks for input validity in lzf_decompress 114 | * and return EINVAL if the input stream has been corrupted. This 115 | * only shields against overflowing the input buffer and will not 116 | * detect most corrupted streams. 117 | * This check is not normally noticable on modern hardware 118 | * (<1% slowdown), but might slow down older cpus considerably. 119 | */ 120 | #ifndef CHECK_INPUT 121 | # define CHECK_INPUT 1 122 | #endif 123 | 124 | /*****************************************************************************/ 125 | /* nothing should be changed below */ 126 | 127 | typedef unsigned char u8; 128 | 129 | typedef const u8 *LZF_STATE[1 << (HLOG)]; 130 | 131 | #if !STRICT_ALIGN 132 | /* for unaligned accesses we need a 16 bit datatype. */ 133 | # include 134 | # if USHRT_MAX == 65535 135 | typedef unsigned short u16; 136 | # elif UINT_MAX == 65535 137 | typedef unsigned int u16; 138 | # else 139 | # undef STRICT_ALIGN 140 | # define STRICT_ALIGN 1 141 | # endif 142 | #endif 143 | 144 | #if ULTRA_FAST 145 | # if defined(VERY_FAST) 146 | # undef VERY_FAST 147 | # endif 148 | #endif 149 | 150 | #if INIT_HTAB 151 | # ifdef __cplusplus 152 | # include 153 | # else 154 | # include 155 | # endif 156 | #endif 157 | 158 | #endif 159 | 160 | -------------------------------------------------------------------------------- /c_src/lzf_c.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2000-2008 Marc Alexander Lehmann 3 | * 4 | * Redistribution and use in source and binary forms, with or without modifica- 5 | * tion, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- 16 | * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- 18 | * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- 22 | * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 23 | * OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * Alternatively, the contents of this file may be used under the terms of 26 | * the GNU General Public License ("GPL") version 2 or any later version, 27 | * in which case the provisions of the GPL are applicable instead of 28 | * the above. If you wish to allow the use of your version of this file 29 | * only under the terms of the GPL and not to allow others to use your 30 | * version of this file under the BSD license, indicate your decision 31 | * by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL. If you do not delete the 33 | * provisions above, a recipient may use your version of this file under 34 | * either the BSD or the GPL. 35 | */ 36 | 37 | #include "lzfP.h" 38 | 39 | #define HSIZE (1 << (HLOG)) 40 | 41 | /* 42 | * don't play with this unless you benchmark! 43 | * decompression is not dependent on the hash function 44 | * the hashing function might seem strange, just believe me 45 | * it works ;) 46 | */ 47 | #ifndef FRST 48 | # define FRST(p) (((p[0]) << 8) | p[1]) 49 | # define NEXT(v,p) (((v) << 8) | p[2]) 50 | # if ULTRA_FAST 51 | # define IDX(h) ((( h >> (3*8 - HLOG)) - h ) & (HSIZE - 1)) 52 | # elif VERY_FAST 53 | # define IDX(h) ((( h >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) 54 | # else 55 | # define IDX(h) ((((h ^ (h << 5)) >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) 56 | # endif 57 | #endif 58 | /* 59 | * IDX works because it is very similar to a multiplicative hash, e.g. 60 | * ((h * 57321 >> (3*8 - HLOG)) & (HSIZE - 1)) 61 | * the latter is also quite fast on newer CPUs, and compresses similarly. 62 | * 63 | * the next one is also quite good, albeit slow ;) 64 | * (int)(cos(h & 0xffffff) * 1e6) 65 | */ 66 | 67 | #if 0 68 | /* original lzv-like hash function, much worse and thus slower */ 69 | # define FRST(p) (p[0] << 5) ^ p[1] 70 | # define NEXT(v,p) ((v) << 5) ^ p[2] 71 | # define IDX(h) ((h) & (HSIZE - 1)) 72 | #endif 73 | 74 | #define MAX_LIT (1 << 5) 75 | #define MAX_OFF (1 << 13) 76 | #define MAX_REF ((1 << 8) + (1 << 3)) 77 | 78 | #if __GNUC__ >= 3 79 | # define expect(expr,value) __builtin_expect ((expr),(value)) 80 | # define inline inline 81 | #else 82 | # define expect(expr,value) (expr) 83 | # define inline static 84 | #endif 85 | 86 | #define expect_false(expr) expect ((expr) != 0, 0) 87 | #define expect_true(expr) expect ((expr) != 0, 1) 88 | 89 | /* 90 | * compressed format 91 | * 92 | * 000LLLLL ; literal 93 | * LLLooooo oooooooo ; backref L 94 | * 111ooooo LLLLLLLL oooooooo ; backref L+7 95 | * 96 | */ 97 | 98 | unsigned int 99 | lzf_compress (const void *const in_data, unsigned int in_len, 100 | void *out_data, unsigned int out_len 101 | #if LZF_STATE_ARG 102 | , LZF_STATE htab 103 | #endif 104 | ) 105 | { 106 | #if !LZF_STATE_ARG 107 | LZF_STATE htab; 108 | #endif 109 | const u8 **hslot; 110 | const u8 *ip = (const u8 *)in_data; 111 | u8 *op = (u8 *)out_data; 112 | const u8 *in_end = ip + in_len; 113 | u8 *out_end = op + out_len; 114 | const u8 *ref; 115 | 116 | /* off requires a type wide enough to hold a general pointer difference. 117 | * ISO C doesn't have that (size_t might not be enough and ptrdiff_t only 118 | * works for differences within a single object). We also assume that no 119 | * no bit pattern traps. Since the only platform that is both non-POSIX 120 | * and fails to support both assumptions is windows 64 bit, we make a 121 | * special workaround for it. 122 | */ 123 | #if defined (WIN32) && defined (_M_X64) 124 | unsigned _int64 off; /* workaround for missing POSIX compliance */ 125 | #else 126 | unsigned long off; 127 | #endif 128 | unsigned int hval; 129 | int lit; 130 | 131 | if (!in_len || !out_len) 132 | return 0; 133 | 134 | #if INIT_HTAB 135 | memset (htab, 0, sizeof (htab)); 136 | # if 0 137 | for (hslot = htab; hslot < htab + HSIZE; hslot++) 138 | *hslot++ = ip; 139 | # endif 140 | #endif 141 | 142 | lit = 0; op++; /* start run */ 143 | 144 | hval = FRST (ip); 145 | while (ip < in_end - 2) 146 | { 147 | hval = NEXT (hval, ip); 148 | hslot = htab + IDX (hval); 149 | ref = *hslot; *hslot = ip; 150 | 151 | if (1 152 | #if INIT_HTAB 153 | && ref < ip /* the next test will actually take care of this, but this is faster */ 154 | #endif 155 | && (off = ip - ref - 1) < MAX_OFF 156 | && ip + 4 < in_end 157 | && ref > (u8 *)in_data 158 | #if STRICT_ALIGN 159 | && ref[0] == ip[0] 160 | && ref[1] == ip[1] 161 | && ref[2] == ip[2] 162 | #else 163 | && *(u16 *)ref == *(u16 *)ip 164 | && ref[2] == ip[2] 165 | #endif 166 | ) 167 | { 168 | /* match found at *ref++ */ 169 | unsigned int len = 2; 170 | unsigned int maxlen = in_end - ip - len; 171 | maxlen = maxlen > MAX_REF ? MAX_REF : maxlen; 172 | 173 | op [- lit - 1] = lit - 1; /* stop run */ 174 | op -= !lit; /* undo run if length is zero */ 175 | 176 | if (expect_false (op + 3 + 1 >= out_end)) 177 | return 0; 178 | 179 | for (;;) 180 | { 181 | if (expect_true (maxlen > 16)) 182 | { 183 | len++; if (ref [len] != ip [len]) break; 184 | len++; if (ref [len] != ip [len]) break; 185 | len++; if (ref [len] != ip [len]) break; 186 | len++; if (ref [len] != ip [len]) break; 187 | 188 | len++; if (ref [len] != ip [len]) break; 189 | len++; if (ref [len] != ip [len]) break; 190 | len++; if (ref [len] != ip [len]) break; 191 | len++; if (ref [len] != ip [len]) break; 192 | 193 | len++; if (ref [len] != ip [len]) break; 194 | len++; if (ref [len] != ip [len]) break; 195 | len++; if (ref [len] != ip [len]) break; 196 | len++; if (ref [len] != ip [len]) break; 197 | 198 | len++; if (ref [len] != ip [len]) break; 199 | len++; if (ref [len] != ip [len]) break; 200 | len++; if (ref [len] != ip [len]) break; 201 | len++; if (ref [len] != ip [len]) break; 202 | } 203 | 204 | do 205 | len++; 206 | while (len < maxlen && ref[len] == ip[len]); 207 | 208 | break; 209 | } 210 | 211 | len -= 2; /* len is now #octets - 1 */ 212 | ip++; 213 | 214 | if (len < 7) 215 | { 216 | *op++ = (off >> 8) + (len << 5); 217 | } 218 | else 219 | { 220 | *op++ = (off >> 8) + ( 7 << 5); 221 | *op++ = len - 7; 222 | } 223 | 224 | *op++ = off; 225 | lit = 0; op++; /* start run */ 226 | 227 | ip += len + 1; 228 | 229 | if (expect_false (ip >= in_end - 2)) 230 | break; 231 | 232 | #if ULTRA_FAST || VERY_FAST 233 | --ip; 234 | # if VERY_FAST && !ULTRA_FAST 235 | --ip; 236 | # endif 237 | hval = FRST (ip); 238 | 239 | hval = NEXT (hval, ip); 240 | htab[IDX (hval)] = ip; 241 | ip++; 242 | 243 | # if VERY_FAST && !ULTRA_FAST 244 | hval = NEXT (hval, ip); 245 | htab[IDX (hval)] = ip; 246 | ip++; 247 | # endif 248 | #else 249 | ip -= len + 1; 250 | 251 | do 252 | { 253 | hval = NEXT (hval, ip); 254 | htab[IDX (hval)] = ip; 255 | ip++; 256 | } 257 | while (len--); 258 | #endif 259 | } 260 | else 261 | { 262 | /* one more literal byte we must copy */ 263 | if (expect_false (op >= out_end)) 264 | return 0; 265 | 266 | lit++; *op++ = *ip++; 267 | 268 | if (expect_false (lit == MAX_LIT)) 269 | { 270 | op [- lit - 1] = lit - 1; /* stop run */ 271 | lit = 0; op++; /* start run */ 272 | } 273 | } 274 | } 275 | 276 | if (op + 3 > out_end) /* at most 3 bytes can be missing here */ 277 | return 0; 278 | 279 | while (ip < in_end) 280 | { 281 | lit++; *op++ = *ip++; 282 | 283 | if (expect_false (lit == MAX_LIT)) 284 | { 285 | op [- lit - 1] = lit - 1; /* stop run */ 286 | lit = 0; op++; /* start run */ 287 | } 288 | } 289 | 290 | op [- lit - 1] = lit - 1; /* end run */ 291 | op -= !lit; /* undo run if length is zero */ 292 | 293 | return op - (u8 *)out_data; 294 | } 295 | 296 | -------------------------------------------------------------------------------- /c_src/lzf_d.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2000-2007 Marc Alexander Lehmann 3 | * 4 | * Redistribution and use in source and binary forms, with or without modifica- 5 | * tion, are permitted provided that the following conditions are met: 6 | * 7 | * 1. Redistributions of source code must retain the above copyright notice, 8 | * this list of conditions and the following disclaimer. 9 | * 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- 16 | * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- 18 | * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- 22 | * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 23 | * OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * 25 | * Alternatively, the contents of this file may be used under the terms of 26 | * the GNU General Public License ("GPL") version 2 or any later version, 27 | * in which case the provisions of the GPL are applicable instead of 28 | * the above. If you wish to allow the use of your version of this file 29 | * only under the terms of the GPL and not to allow others to use your 30 | * version of this file under the BSD license, indicate your decision 31 | * by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL. If you do not delete the 33 | * provisions above, a recipient may use your version of this file under 34 | * either the BSD or the GPL. 35 | */ 36 | 37 | #include "lzfP.h" 38 | 39 | #if AVOID_ERRNO 40 | # define SET_ERRNO(n) 41 | #else 42 | # include 43 | # define SET_ERRNO(n) errno = (n) 44 | #endif 45 | 46 | /* 47 | #if (__i386 || __amd64) && __GNUC__ >= 3 48 | # define lzf_movsb(dst, src, len) \ 49 | asm ("rep movsb" \ 50 | : "=D" (dst), "=S" (src), "=c" (len) \ 51 | : "0" (dst), "1" (src), "2" (len)); 52 | #endif 53 | */ 54 | 55 | unsigned int 56 | lzf_decompress (const void *const in_data, unsigned int in_len, 57 | void *out_data, unsigned int out_len) 58 | { 59 | u8 const *ip = (const u8 *)in_data; 60 | u8 *op = (u8 *)out_data; 61 | u8 const *const in_end = ip + in_len; 62 | u8 *const out_end = op + out_len; 63 | 64 | do 65 | { 66 | unsigned int ctrl = *ip++; 67 | 68 | if (ctrl < (1 << 5)) /* literal run */ 69 | { 70 | ctrl++; 71 | 72 | if (op + ctrl > out_end) 73 | { 74 | SET_ERRNO (E2BIG); 75 | return 0; 76 | } 77 | 78 | #if CHECK_INPUT 79 | if (ip + ctrl > in_end) 80 | { 81 | SET_ERRNO (EINVAL); 82 | return 0; 83 | } 84 | #endif 85 | 86 | #ifdef lzf_movsb 87 | lzf_movsb (op, ip, ctrl); 88 | #else 89 | do 90 | *op++ = *ip++; 91 | while (--ctrl); 92 | #endif 93 | } 94 | else /* back reference */ 95 | { 96 | unsigned int len = ctrl >> 5; 97 | 98 | u8 *ref = op - ((ctrl & 0x1f) << 8) - 1; 99 | 100 | #if CHECK_INPUT 101 | if (ip >= in_end) 102 | { 103 | SET_ERRNO (EINVAL); 104 | return 0; 105 | } 106 | #endif 107 | if (len == 7) 108 | { 109 | len += *ip++; 110 | #if CHECK_INPUT 111 | if (ip >= in_end) 112 | { 113 | SET_ERRNO (EINVAL); 114 | return 0; 115 | } 116 | #endif 117 | } 118 | 119 | ref -= *ip++; 120 | 121 | if (op + len + 2 > out_end) 122 | { 123 | SET_ERRNO (E2BIG); 124 | return 0; 125 | } 126 | 127 | if (ref < (u8 *)out_data) 128 | { 129 | SET_ERRNO (EINVAL); 130 | return 0; 131 | } 132 | 133 | #ifdef lzf_movsb 134 | len += 2; 135 | lzf_movsb (op, ref, len); 136 | #else 137 | *op++ = *ref++; 138 | *op++ = *ref++; 139 | 140 | do 141 | *op++ = *ref++; 142 | while (--len); 143 | #endif 144 | } 145 | } 146 | while (ip < in_end); 147 | 148 | return op - (u8 *)out_data; 149 | } 150 | 151 | -------------------------------------------------------------------------------- /c_src/nsync_nifs.c: -------------------------------------------------------------------------------- 1 | // most of this nif was shamelessly copy/pasted from erlz, so included 2 | // below is the original license... 3 | // ------------------------------------------------------------------- 4 | // 5 | // erlz: Erlang bindings for the fastlz compression library 6 | // 7 | // Copyright (c) 2010 Hypothetical Labs, Inc. All Rights Reserved. 8 | // 9 | // This file is provided to you under the Apache License, 10 | // Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License. You may obtain 12 | // a copy of the License at 13 | // 14 | // http://www.apache.org/licenses/LICENSE-2.0 15 | // 16 | // Unless required by applicable law or agreed to in writing, 17 | // software distributed under the License is distributed on an 18 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 19 | // KIND, either express or implied. See the License for the 20 | // specific language governing permissions and limitations 21 | // under the License. 22 | // 23 | // ------------------------------------------------------------------- 24 | #include "erl_nif.h" 25 | #include "erl_nif_compat.h" 26 | #include "lzf.h" 27 | #include 28 | 29 | extern int decompress(int x); 30 | 31 | int decompress1(ErlNifEnv *env, ErlNifBinary *source, ErlNifBinary *target) { 32 | int retval = 0; 33 | int bufsize; 34 | double expansion_factor = 1.1; 35 | unsigned int result; 36 | while(expansion_factor < 2.5) { 37 | bufsize = (int) source->size * expansion_factor; 38 | bufsize = bufsize < 66 ? 66 : bufsize; 39 | enif_alloc_binary_compat(env, bufsize, target); 40 | result = lzf_decompress(source->data, source->size, target->data, target->size); 41 | if (result) { 42 | enif_realloc_binary_compat(env, target, result); 43 | retval = 1; 44 | break; 45 | } 46 | else { 47 | enif_release_binary_compat(env, target); 48 | } 49 | expansion_factor += 0.1; 50 | } 51 | return retval; 52 | } 53 | 54 | int compress1(ErlNifEnv *env, ErlNifBinary *source, ErlNifBinary *target) { 55 | int retval = 0; 56 | int bufsize; 57 | double expansion_factor = 1.1; 58 | int result; 59 | while(expansion_factor < 2.5) { 60 | bufsize = (int) source->size * expansion_factor; 61 | bufsize = bufsize < 66 ? 66 : bufsize; 62 | enif_alloc_binary_compat(env, bufsize, target); 63 | result = lzf_compress(source->data, source->size, target->data, target->size); 64 | if (result) { 65 | enif_realloc_binary_compat(env, target, result); 66 | retval = 1; 67 | break; 68 | } 69 | else { 70 | enif_release_binary_compat(env, target); 71 | } 72 | expansion_factor += 0.1; 73 | } 74 | return retval; 75 | } 76 | 77 | static ERL_NIF_TERM decompress_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { 78 | ERL_NIF_TERM retval; 79 | ErlNifBinary source; 80 | ErlNifBinary target; 81 | if (argc != 1 || !enif_inspect_binary(env, argv[0], &source)) { 82 | return enif_make_badarg(env); 83 | } 84 | if (decompress1(env, &source, &target)) { 85 | retval = enif_make_binary(env, &target); 86 | } 87 | else { 88 | retval = enif_make_badarg(env); 89 | } 90 | return retval; 91 | } 92 | 93 | ERL_NIF_TERM compress_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { 94 | ERL_NIF_TERM retval; 95 | ErlNifBinary source; 96 | ErlNifBinary target; 97 | 98 | if (argc != 1 || !enif_inspect_binary(env, argv[0], &source)) { 99 | return enif_make_badarg(env); 100 | } 101 | if (compress1(env, &source, &target)) { 102 | retval = enif_make_binary(env, &target); 103 | } 104 | else { 105 | retval = enif_make_badarg(env); 106 | } 107 | return retval; 108 | } 109 | 110 | static ErlNifFunc nif_funcs[] = { 111 | {"decompress", 1, decompress_nif}, 112 | {"compress", 1, compress_nif} 113 | }; 114 | 115 | ERL_NIF_INIT(lzf, nif_funcs, NULL, NULL, NULL, NULL); 116 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jkvor/nsync/dab4de60c87a90515d7903f487d7e5fc9a5d6d2f/rebar -------------------------------------------------------------------------------- /src/lzf.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(lzf). 24 | -export([compress/1, decompress/1]). 25 | 26 | -on_load(init/0). 27 | 28 | init() -> 29 | case erlang:load_nif("./priv/nsync_drv", 0) of 30 | ok -> ok; 31 | _ -> 32 | Filename = filename:join([code:lib_dir(nsync), "priv", "nsync_drv"]), 33 | ok = erlang:load_nif(Filename, 0) 34 | end. 35 | 36 | compress(_X) -> 37 | exit(nif_library_not_loaded). 38 | 39 | decompress(_X) -> 40 | exit(nif_library_not_loaded). 41 | 42 | -------------------------------------------------------------------------------- /src/nsync.app.src: -------------------------------------------------------------------------------- 1 | {application, nsync, 2 | [ 3 | {description, "Erlang redis replication client"}, 4 | {vsn, "1.0"}, 5 | {registered, []}, 6 | {applications, [kernel,stdlib]}, 7 | {env, []} 8 | ]}. 9 | -------------------------------------------------------------------------------- /src/nsync.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(nsync). 24 | -behaviour(gen_server). 25 | 26 | %% gen_server callbacks 27 | -export([init/1, 28 | handle_call/3, 29 | handle_cast/2, 30 | handle_info/2, 31 | terminate/2, 32 | code_change/3]). 33 | 34 | %% API 35 | -export([start_link/0, start_link/1]). 36 | 37 | -include("nsync.hrl"). 38 | 39 | -record(state, {callback, caller_pid, socket, opts, 40 | state, buffer, rdb_state}). 41 | 42 | -define(TIMEOUT, 30000). 43 | -define(CALLBACK_MODS, [nsync_string, 44 | nsync_list, 45 | nsync_set, 46 | nsync_zset, 47 | nsync_hash, 48 | nsync_pubsub]). 49 | 50 | %%==================================================================== 51 | %% API functions 52 | %%==================================================================== 53 | start_link() -> 54 | start_link([]). 55 | 56 | start_link(Opts) -> 57 | Timeout = proplists:get_value(timeout, Opts, 1000 * 60 * 6), 58 | case proplists:get_value(block, Opts) of 59 | true -> 60 | case gen_server:start_link(?MODULE, [Opts, self()], []) of 61 | {ok, Pid} -> 62 | receive 63 | {Pid, load_complete} -> 64 | {ok, Pid} 65 | after Timeout -> 66 | {error, timeout} 67 | end; 68 | Err -> 69 | Err 70 | end; 71 | _ -> 72 | gen_server:start_link(?MODULE, [Opts, undefined], []) 73 | end. 74 | 75 | %%==================================================================== 76 | %% gen_server callbacks 77 | %%==================================================================== 78 | init([Opts, CallerPid]) -> 79 | case init_state(Opts, CallerPid) of 80 | {ok, State} -> 81 | {ok, State}; 82 | Error -> 83 | {stop, Error} 84 | end. 85 | 86 | handle_call(_Request, _From, State) -> 87 | {reply, ignore, State}. 88 | 89 | handle_cast(_Msg, State) -> 90 | {noreply, State}. 91 | 92 | handle_info({tcp, Socket, Data}, #state{callback=Callback, 93 | caller_pid=CallerPid, 94 | socket=Socket, 95 | state=loading, 96 | rdb_state=RdbState}=State) -> 97 | NewState = 98 | case rdb_load:packet(RdbState, Data, Callback) of 99 | {eof, Rest} -> 100 | case CallerPid of 101 | undefined -> ok; 102 | _ -> CallerPid ! {self(), load_complete} 103 | end, 104 | nsync_utils:do_callback(Callback, [{load, eof}]), 105 | {ok, Rest1} = redis_text_proto:parse_commands(Rest, Callback), 106 | State#state{state=up, buffer=Rest1}; 107 | {error, <<"-LOADING", _/binary>> = Msg} -> 108 | error_logger:info_report([?MODULE, Msg]), 109 | timer:sleep(1000), 110 | init_sync(Socket), 111 | State; 112 | RdbState1 -> 113 | State#state{rdb_state=RdbState1} 114 | end, 115 | inet:setopts(Socket, [{active, once}]), 116 | {noreply, NewState}; 117 | 118 | handle_info({tcp, Socket, Data}, #state{callback=Callback, 119 | socket=Socket, 120 | buffer=Buffer}=State) -> 121 | {ok, Rest} = redis_text_proto:parse_commands(<>, Callback), 122 | inet:setopts(Socket, [{active, once}]), 123 | {noreply, State#state{buffer=Rest}}; 124 | 125 | handle_info({tcp_closed, _}, #state{callback=Callback, 126 | opts=Opts, 127 | socket=Socket}=State) -> 128 | catch gen_tcp:close(Socket), 129 | nsync_utils:do_callback(Callback, [{error, closed}]), 130 | case reconnect(Opts) of 131 | {ok, State1} -> 132 | {noreply, State1}; 133 | Error -> 134 | {stop, Error, State} 135 | end; 136 | 137 | handle_info({tcp_error, _ ,_}, #state{callback=Callback, 138 | opts=Opts, 139 | socket=Socket}=State) -> 140 | catch gen_tcp:close(Socket), 141 | nsync_utils:do_callback(Callback, [{error, closed}]), 142 | case reconnect(Opts) of 143 | {ok, State1} -> 144 | {noreply, State1}; 145 | Error -> 146 | {stop, Error, State} 147 | end; 148 | 149 | handle_info(_Info, State) -> 150 | {noreply, State}. 151 | 152 | terminate(_Reason, _State) -> 153 | ok. 154 | 155 | code_change(_OldVsn, State, _Extra) -> 156 | {ok, State}. 157 | 158 | %%==================================================================== 159 | %% internal functions 160 | %%==================================================================== 161 | reconnect(Opts) -> 162 | case init_state(Opts, undefined) of 163 | {ok, State} -> 164 | {ok, State}; 165 | {error, Err} when Err == econnrefused; Err == closed -> 166 | timer:sleep(250), 167 | reconnect(Opts); 168 | Err -> 169 | Err 170 | end. 171 | 172 | init_state(Opts, CallerPid) -> 173 | Host = proplists:get_value(host, Opts, "localhost"), 174 | Port = proplists:get_value(port, Opts, 6379), 175 | Auth = proplists:get_value(pass, Opts), 176 | Callback = 177 | case proplists:get_value(callback, Opts) of 178 | undefined -> default_callback(); 179 | Cb -> Cb 180 | end, 181 | case open_socket(Host, Port) of 182 | {ok, Socket} -> 183 | case authenticate(Socket, Auth) of 184 | ok -> 185 | init_sync(Socket), 186 | inet:setopts(Socket, [{active, once}]), 187 | {ok, #state{ 188 | callback=Callback, 189 | caller_pid=CallerPid, 190 | socket=Socket, 191 | opts=Opts, 192 | state=loading, 193 | buffer = <<>> 194 | }}; 195 | Error -> 196 | Error 197 | end; 198 | Error -> 199 | Error 200 | end. 201 | 202 | open_socket(Host, Port) when is_list(Host), is_integer(Port) -> 203 | gen_tcp:connect(Host, Port, [binary, {packet, raw}, {active, false}]); 204 | 205 | open_socket(_Host, _Port) -> 206 | {error, invalid_host_or_port}. 207 | 208 | authenticate(_Socket, undefined) -> 209 | ok; 210 | 211 | authenticate(Socket, Auth) -> 212 | case gen_tcp:send(Socket, [<<"AUTH ">>, Auth, <<"\r\n">>]) of 213 | ok -> 214 | case gen_tcp:recv(Socket, 0, ?TIMEOUT) of 215 | {ok, <<"OK\r\n">>} -> 216 | ok; 217 | {ok, <<"+OK\r\n">>} -> 218 | ok; 219 | Error -> 220 | Error 221 | end; 222 | Error -> 223 | Error 224 | end. 225 | 226 | default_callback() -> 227 | fun(Arg) -> 228 | io:format("nsync ~1000p~n", [Arg]) 229 | end. 230 | 231 | init_sync(Socket) -> 232 | gen_tcp:send(Socket, <<"SYNC\r\n">>). 233 | 234 | -------------------------------------------------------------------------------- /src/nsync.hrl: -------------------------------------------------------------------------------- 1 | -define(REDIS_STRING, 0). 2 | -define(REDIS_LIST, 1). 3 | -define(REDIS_SET, 2). 4 | -define(REDIS_ZSET, 3). 5 | -define(REDIS_HASH, 4). 6 | 7 | -define(REDIS_EXPIRETIME, 253). 8 | -define(REDIS_SELECTDB, 254). 9 | -define(REDIS_EOF, 255). 10 | 11 | -define(REDIS_RDB_6BITLEN, 0). 12 | -define(REDIS_RDB_14BITLEN, 1). 13 | -define(REDIS_RDB_32BITLEN, 2). 14 | -define(REDIS_RDB_ENCVAL, 3). 15 | 16 | -define(REDIS_RDB_ENC_INT8, 0). % 8 bit signed integer 17 | -define(REDIS_RDB_ENC_INT16, 1). % 16 bit signed integer 18 | -define(REDIS_RDB_ENC_INT32, 2). % 32 bit signed integer 19 | -define(REDIS_RDB_ENC_LZF, 3). % string compressed with FASTLZ 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/nsync_utils.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(nsync_utils). 24 | -export([do_callback/2]). 25 | 26 | do_callback({M,F}, Args) when is_atom(M), is_atom(F) -> 27 | apply(M, F, Args); 28 | 29 | do_callback({M,F,A}, Args) when is_atom(M), is_atom(F), is_list(A) -> 30 | apply(M, F, A ++ Args); 31 | 32 | do_callback(Fun, Args) when is_function(Fun) -> 33 | apply(Fun, Args); 34 | 35 | do_callback(_Cb, _Args) -> 36 | exit("Invalid callback"). 37 | -------------------------------------------------------------------------------- /src/rdb_load.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(rdb_load). 24 | -export([packet/3]). 25 | 26 | -record(state, {first = true, buffer = <<>>}). 27 | 28 | -include("nsync.hrl"). 29 | 30 | packet(State, Data, Callback) when State == undefined orelse State#state.first == true -> 31 | Data1 = 32 | case State of 33 | undefined -> Data; 34 | #state{buffer=Buffer} -> <> 35 | end, 36 | case check_packet(Data1) of 37 | {ok, line, Line} -> 38 | {error, Line}; 39 | {ok, Rest} -> 40 | case parse_len(Rest) of 41 | {ok, _Len, Rest1} -> 42 | case parse_rdb_version(Rest1) of 43 | {ok, Rest2, Vsn} -> 44 | Vsn /= <<"0001">> andalso exit({error, vsn_not_supported}), 45 | packet(#state{buffer = <<>>, first = false}, Rest2, Callback); 46 | {error, eof} -> 47 | #state{buffer = Data} 48 | end; 49 | {error, eof} -> 50 | #state{buffer = Data} 51 | end; 52 | {error, eof} -> 53 | #state{buffer = Data} 54 | end; 55 | 56 | packet(#state{buffer=Buffer}, Data, Callback) -> 57 | case parse(<>, Callback) of 58 | {ok, Rest} -> 59 | #state{buffer = Rest, first = false}; 60 | {eof, Rest} -> 61 | {eof, Rest} 62 | end. 63 | 64 | check_packet(<> = Data) when Char == $-; Char == $+ -> 65 | case read_line(Data) of 66 | {ok, Line, <<>>} -> 67 | {ok, line, Line}; 68 | {ok, _Line, Rest} -> 69 | {ok, Rest}; 70 | {error, eof} -> 71 | {error, eof} 72 | end; 73 | 74 | check_packet(<<"\r\n", Data/binary>>) -> 75 | {ok, Data}; 76 | 77 | check_packet(<<"\n", Data/binary>>) -> 78 | {ok, Data}; 79 | 80 | check_packet(Data) -> 81 | {ok, Data}. 82 | 83 | read_line(Data) -> 84 | case binary:split(Data, <<"\r\n">>) of 85 | [_] -> {error, eof}; 86 | [] -> {error, eof}; 87 | [Line,Rest] -> 88 | {ok, Line, Rest} 89 | end. 90 | 91 | parse_len(<<>>) -> 92 | {error, eof}; 93 | 94 | parse_len(<<"\n">>) -> 95 | {error, eof}; 96 | 97 | parse_len(<<"$", Rest/binary>>) -> 98 | case read_line(Rest) of 99 | {ok, Line, Rest1} -> 100 | {ok, list_to_integer(binary_to_list(Line)), Rest1}; 101 | {error, eof} -> 102 | {error, eof} 103 | end. 104 | 105 | parse_rdb_version(<<"REDIS", Vsn:4/binary, Rest/binary>>) -> 106 | {ok, Rest, Vsn}; 107 | 108 | parse_rdb_version(_) -> 109 | {error, eof}. 110 | 111 | parse(<<>>, _Callback) -> 112 | {ok, <<>>}; 113 | 114 | parse(Data, Callback) -> 115 | {ok, Type, Rest} = rdb_type(Data), 116 | parse(Type, Rest, Callback). 117 | 118 | parse(?REDIS_EXPIRETIME, Data, Callback) -> 119 | case Data of 120 | <<_Time:32/unsigned-integer, Rest/binary>> -> 121 | parse(Rest, Callback); 122 | _ -> 123 | {ok, <>} 124 | end; 125 | 126 | parse(?REDIS_EOF, Rest, _Callback) -> 127 | {eof, Rest}; 128 | 129 | parse(?REDIS_SELECTDB, Data, Callback) -> 130 | case catch rdb_len(Data) of 131 | {'EXIT', {error, eof}} -> 132 | {ok, <>}; 133 | {ok, _Enc, _Db, Rest} -> 134 | parse(Rest, Callback) 135 | end; 136 | 137 | parse(Type, <<>>, _Callback) -> 138 | {ok, <>}; 139 | 140 | parse(Type, Data, Callback) -> 141 | case catch rdb_string_object(Data) of 142 | {'EXIT', {error, eof}} -> 143 | {ok, <>}; 144 | {'EXIT', Err} -> 145 | exit(Err); 146 | {ok, Key, Rest} -> 147 | case catch rdb_load_object(Type, Rest) of 148 | {'EXIT', {error, eof}} -> 149 | {ok, <>}; 150 | {'EXIT', Err} -> 151 | exit(Err); 152 | {ok, Val, Rest1} -> 153 | write(Callback, Key, Val), 154 | parse(Rest1, Callback) 155 | end 156 | end. 157 | 158 | write(Callback, Key, Val) -> 159 | nsync_utils:do_callback(Callback, [{load, Key, Val}]), 160 | ok. 161 | 162 | rdb_type(<>) -> 163 | {ok, Type, Rest}; 164 | 165 | rdb_type(<<>>) -> 166 | exit({error, eof}). 167 | 168 | rdb_len(<<>>) -> 169 | exit({error, eof}); 170 | 171 | rdb_len(<>) -> 172 | case ((Type band 16#C0) bsr 6) of 173 | ?REDIS_RDB_6BITLEN -> 174 | {ok, false, Type band 16#3F, Rest}; 175 | ?REDIS_RDB_ENCVAL -> 176 | {ok, true, Type band 16#3F, Rest}; 177 | ?REDIS_RDB_14BITLEN -> 178 | case Rest of 179 | <> -> 180 | {ok, false, ((Type band 16#3F) bsl 8) bor Next, Rest1}; 181 | _ -> 182 | exit({error, eof}) 183 | end; 184 | _ -> 185 | case Rest of 186 | <> -> 187 | case <> of 188 | <> -> 189 | {ok, false, Val, Rest1}; 190 | _ -> 191 | exit({error, eof}) 192 | end; 193 | _ -> 194 | exit({error, eof}) 195 | end 196 | end. 197 | 198 | rdb_string_object(Data) -> 199 | rdb_generic_string_object(Data, false). 200 | 201 | rdb_encoded_string_object(Data) -> 202 | rdb_generic_string_object(Data, true). 203 | 204 | rdb_generic_string_object(Data, _Encode) -> 205 | {ok, Enc, Len, Rest} = rdb_len(Data), 206 | case Enc of 207 | true -> 208 | case Len of 209 | ?REDIS_RDB_ENC_INT8 -> 210 | rdb_integer_object(Len, Rest); 211 | ?REDIS_RDB_ENC_INT16 -> 212 | rdb_integer_object(Len, Rest); 213 | ?REDIS_RDB_ENC_INT32 -> 214 | rdb_integer_object(Len, Rest); 215 | ?REDIS_RDB_ENC_LZF -> 216 | rdb_lzf_string_object(Rest); 217 | _ -> 218 | exit("Unknown RDB encoding type") 219 | end; 220 | false -> 221 | case Rest of 222 | <> -> 223 | {ok, Str, Rest1}; 224 | _ -> 225 | exit({error, eof}) 226 | end 227 | end. 228 | 229 | rdb_integer_object(?REDIS_RDB_ENC_INT8, <>) -> 230 | {ok, integer_to_list(Char), Rest}; 231 | 232 | rdb_integer_object(?REDIS_RDB_ENC_INT8, _) -> 233 | exit({error, eof}); 234 | 235 | rdb_integer_object(?REDIS_RDB_ENC_INT16, <>) -> 236 | {ok, integer_to_list(A bor (B bsl 8)), Rest}; 237 | 238 | rdb_integer_object(?REDIS_RDB_ENC_INT16, _) -> 239 | exit({error, eof}); 240 | 241 | rdb_integer_object(?REDIS_RDB_ENC_INT32, <>) -> 242 | {ok, integer_to_list(A bor (B bsl 8) bor (C bsl 16) bor (D bsl 24)), Rest}; 243 | 244 | rdb_integer_object(?REDIS_RDB_ENC_INT32, _) -> 245 | exit({error, eof}); 246 | 247 | rdb_integer_object(_Type, _Data) -> 248 | exit("Unknown RDB integer encoding type"). 249 | 250 | %% TODO: parse doubles 251 | rdb_double_value(Data) -> 252 | rdb_encoded_string_object(Data). 253 | 254 | rdb_lzf_string_object(Data) -> 255 | {ok, _Enc1, LzfLen, Rest} = rdb_len(Data), 256 | {ok, _Enc2, _UncompLen, Rest1} = rdb_len(Rest), 257 | case Rest1 of 258 | <> -> 259 | case (catch lzf:decompress(LzfEnc)) of 260 | {'EXIT', _Err} -> 261 | error_logger:error_msg("failed lzf_decompress(~p)~n", [LzfEnc]), 262 | {ok, <<"">>, Rest2}; 263 | Str -> 264 | {ok, Str, Rest2} 265 | end; 266 | _ -> 267 | exit({error, eof}) 268 | end. 269 | 270 | rdb_load_object(_Type, <<>>) -> 271 | exit({error, eof}); 272 | 273 | rdb_load_object(?REDIS_STRING, Data) -> 274 | rdb_encoded_string_object(Data); 275 | 276 | rdb_load_object(?REDIS_LIST, Data) -> 277 | {ok, _Enc, Size, Rest} = rdb_len(Data), 278 | parse_list_vals(Size, Rest, []); 279 | 280 | rdb_load_object(?REDIS_SET, Data) -> 281 | {ok, _Enc, Size, Rest} = rdb_len(Data), 282 | parse_list_vals(Size, Rest, []); 283 | 284 | rdb_load_object(?REDIS_ZSET, Data) -> 285 | {ok, _Enc, Size, Rest} = rdb_len(Data), 286 | parse_zset_vals(Size, Rest, []); 287 | 288 | rdb_load_object(?REDIS_HASH, Data) -> 289 | {ok, _Enc, Size, Rest} = rdb_len(Data), 290 | parse_hash_props(Size, Rest, dict:new()); 291 | 292 | rdb_load_object(_Type, _Data) -> 293 | io:format("unknown object type: ~p~n", [_Type]), 294 | exit("Unknown object type"). 295 | 296 | parse_list_vals(0, Rest, Acc) -> 297 | {ok, lists:reverse(Acc), Rest}; 298 | 299 | parse_list_vals(Size, Rest, Acc) -> 300 | {ok, Str, Rest1} = rdb_encoded_string_object(Rest), 301 | parse_list_vals(Size-1, Rest1, [Str|Acc]). 302 | 303 | parse_zset_vals(0, Rest, Acc) -> 304 | {ok, lists:sort(Acc), Rest}; 305 | 306 | parse_zset_vals(Size, Rest, Acc) -> 307 | {ok, Str, Rest1} = rdb_encoded_string_object(Rest), 308 | {ok, Score, Rest2} = rdb_double_value(Rest1), 309 | parse_zset_vals(Size-1, Rest2, [{Score, Str}|Acc]). 310 | 311 | parse_hash_props(0, Rest, Acc) -> 312 | {ok, Acc, Rest}; 313 | 314 | parse_hash_props(Size, Rest, Acc) -> 315 | {ok, Key, Rest1} = rdb_encoded_string_object(Rest), 316 | {ok, Val, Rest2} = rdb_encoded_string_object(Rest1), 317 | parse_hash_props(Size-1, Rest2, dict:store(Key, Val, Acc)). 318 | -------------------------------------------------------------------------------- /src/redis_text_proto.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redis_text_proto). 24 | -export([parse_commands/2]). 25 | 26 | -include("nsync.hrl"). 27 | 28 | parse_commands(<<>>, _Callback) -> 29 | {ok, <<>>}; 30 | 31 | parse_commands(<<"*", Rest/binary>>, Callback) -> 32 | case read_line(Rest, <<>>) of 33 | {ok, Num0, Rest1} -> 34 | Num = list_to_integer(binary_to_list(Num0)), 35 | case parse_num_commands(Rest1, Num, []) of 36 | {ok, [Cmd|Args], Rest2} -> 37 | Cmd1 = string:to_lower(binary_to_list(Cmd)), 38 | nsync_utils:do_callback(Callback, [{cmd, Cmd1, Args}]), 39 | parse_commands(Rest2, Callback); 40 | {error, eof} -> 41 | {ok, <<"*", Rest/binary>>} 42 | end; 43 | {error, eof} -> 44 | {ok, <<"*", Rest/binary>>} 45 | end; 46 | 47 | parse_commands(Buffer, _Callback) -> 48 | case read_line(Buffer, <<>>) of 49 | {ok, _Line, Rest} -> 50 | {ok, Rest}; 51 | {error, eof} -> 52 | {ok, Buffer} 53 | end. 54 | 55 | read_line(<<"\r\n", Rest/binary>>, Acc) -> 56 | {ok, Acc, Rest}; 57 | 58 | read_line(<<"\r", _Rest/binary>>, _Acc) -> 59 | {error, eof}; 60 | 61 | read_line(<<>>, _Acc) -> 62 | {error, eof}; 63 | 64 | read_line(<>, Acc) -> 65 | read_line(Rest, <>). 66 | 67 | parse_num_commands(Rest, 0, Acc) -> 68 | {ok, lists:reverse(Acc), Rest}; 69 | 70 | parse_num_commands(<<"$", Rest/binary>>, Num, Acc) -> 71 | case read_line(Rest, <<>>) of 72 | {ok, Size0, Rest1} -> 73 | Size = list_to_integer(binary_to_list(Size0)), 74 | case read_string(Size, Rest1) of 75 | {ok, Cmd, Rest2} -> 76 | parse_num_commands(Rest2, Num-1, [Cmd|Acc]); 77 | {error, eof} -> 78 | {error, eof} 79 | end; 80 | {error, eof} -> 81 | {error, eof} 82 | end; 83 | 84 | parse_num_commands(_, _Num, _Acc) -> 85 | {error, eof}. 86 | 87 | read_string(Size, Data) -> 88 | case Data of 89 | <> -> 90 | {ok, Cmd, Rest}; 91 | _ -> 92 | {error, eof} 93 | end. 94 | --------------------------------------------------------------------------------