├── .github └── workflows │ ├── Dockerfile │ ├── autobuild.yaml │ ├── package.yaml │ ├── release.yaml │ ├── tests.yaml │ └── zdb-process ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── bin └── .keep ├── examples ├── Makefile └── zdb-example.c ├── libzdb ├── Makefile ├── api.c ├── api.h ├── bootstrap.c ├── bootstrap.h ├── crc32.c ├── crc32.h ├── data.c ├── data.h ├── filesystem.c ├── filesystem.h ├── hook.c ├── hook.h ├── index.c ├── index.h ├── index_branch.c ├── index_branch.h ├── index_get.c ├── index_get.h ├── index_loader.c ├── index_loader.h ├── index_scan.c ├── index_scan.h ├── index_seq.c ├── index_seq.h ├── index_set.c ├── index_set.h ├── libzdb.c ├── libzdb.h ├── libzdb_private.h ├── namespace.c ├── namespace.h ├── security.c ├── security.h ├── settings.c ├── settings.h ├── sha1.c ├── sha1.h └── sockets.h ├── tests ├── Makefile ├── pipeline │ ├── Makefile │ └── pipeline.c ├── run.sh ├── tests.c ├── tests.h ├── tests_user.h ├── zdb_default.c ├── zdb_history.c ├── zdb_lowlevel.c ├── zdb_misc.c ├── zdb_namespace.c ├── zdb_payload.c ├── zdb_scan.c ├── zdb_utils.c └── zdb_utils.h ├── tools ├── Makefile ├── README.md ├── compaction │ ├── Makefile │ ├── branches.c │ ├── branches.h │ ├── compaction.c │ ├── compaction.h │ ├── namespace.c │ ├── validity.c │ └── validity.h ├── hooks-monitor │ └── zdb-hooks-monitor.py ├── incremental-update │ └── incremental.py ├── index-dump │ ├── Makefile │ └── index-dump.c ├── index-rebuild │ ├── Makefile │ └── index-rebuild.c ├── integrity-check │ ├── Makefile │ └── integrity-check.c ├── live-stats │ └── zdb-live-stats.py ├── namespace-dump │ ├── Makefile │ └── namespace-dump.c ├── namespace-editor │ ├── Makefile │ └── namespace-editor.c └── quick-compaction │ ├── Makefile │ └── quick-compact.c ├── utilities ├── README.md ├── db-compacted │ ├── mkdb-quick.py │ └── mkdb-small.py ├── db-mirror │ ├── Makefile │ ├── db-keepsync.c │ ├── db-mirror.c │ ├── db-mirror.h │ └── db-prepare.c ├── db-replicate │ ├── Makefile │ └── db-replicate.c └── db-sync │ ├── Makefile │ └── db-sync.c └── zdbd ├── Makefile ├── auth.c ├── auth.h ├── commands.c ├── commands.h ├── commands_auth.c ├── commands_auth.h ├── commands_dataset.c ├── commands_dataset.h ├── commands_get.c ├── commands_get.h ├── commands_history.c ├── commands_history.h ├── commands_namespace.c ├── commands_namespace.h ├── commands_replicate.c ├── commands_replicate.h ├── commands_scan.c ├── commands_scan.h ├── commands_set.c ├── commands_set.h ├── commands_system.c ├── commands_system.h ├── redis.c ├── redis.h ├── socket_epoll.c ├── socket_kqueue.c ├── zdbd.c └── zdbd.h /.github/workflows/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest AS build 2 | 3 | RUN apk add alpine-sdk 4 | 5 | RUN git clone -b development-v2 https://github.com/threefoldtech/0-db /zdb && \ 6 | cd /zdb/libzdb && \ 7 | make release && \ 8 | cd ../zdbd && \ 9 | make STATIC=1 release 10 | 11 | FROM busybox:latest 12 | 13 | COPY --from=build /zdb/zdbd/zdb /bin/zdb 14 | COPY zdb-process /bin/ 15 | 16 | RUN chmod +x /bin/zdb-process 17 | 18 | WORKDIR / 19 | 20 | ENTRYPOINT ["/bin/zdb-process"] 21 | -------------------------------------------------------------------------------- /.github/workflows/autobuild.yaml: -------------------------------------------------------------------------------- 1 | name: Build release 2 | on: [push] 3 | 4 | jobs: 5 | debug: 6 | name: Create debug artifacts 7 | runs-on: ubuntu-latest 8 | container: 9 | image: alpine:latest 10 | env: 11 | GITHUB_REF: ${{ github.ref }} 12 | GITHUB_SHA: ${{ github.sha }} 13 | steps: 14 | - name: Checkout the repository 15 | uses: actions/checkout@master 16 | 17 | - name: Install dependencies 18 | run: | 19 | apk add alpine-sdk 20 | 21 | - name: Get versions 22 | id: versions 23 | run: | 24 | echo "branch=${GITHUB_REF#refs/*/}" >> "$GITHUB_OUTPUT" 25 | echo "commit=${GITHUB_SHA:0:10}" >> "$GITHUB_OUTPUT" 26 | 27 | - name: Building 0-db 28 | run: | 29 | cd libzdb 30 | make 31 | cd .. 32 | 33 | cd zdbd 34 | make STATIC=1 35 | cd .. 36 | 37 | make 38 | 39 | - name: Create archive 40 | run: | 41 | tar -czf threefoldtech-0-db-${{ steps.versions.outputs.branch }}-${{ steps.versions.outputs.commit }}.tar.gz bin 42 | 43 | - name: Publish flist (tf-autobuilder, ${{ steps.versions.outputs.commit }}) 44 | uses: threefoldtech/publish-flist@master 45 | with: 46 | threefold: ${{ secrets.HUB_TOKEN }} 47 | action: upload 48 | name: threefoldtech-0-db-${{ steps.versions.outputs.branch }}-${{ steps.versions.outputs.commit }}.tar.gz 49 | 50 | release: 51 | name: Create release artifacts 52 | runs-on: ubuntu-latest 53 | container: 54 | image: alpine:latest 55 | env: 56 | GITHUB_REF: ${{ github.ref }} 57 | GITHUB_SHA: ${{ github.sha }} 58 | steps: 59 | - name: Checkout the repository 60 | uses: actions/checkout@master 61 | 62 | - name: Install dependencies 63 | run: | 64 | apk add alpine-sdk 65 | 66 | - name: Get versions 67 | id: versions 68 | run: | 69 | echo "branch=${GITHUB_REF#refs/*/}" >> "$GITHUB_OUTPUT" 70 | echo "commit=${GITHUB_SHA:0:10}" >> "$GITHUB_OUTPUT" 71 | 72 | - name: Building 0-db 73 | run: | 74 | cd libzdb 75 | make release 76 | cd .. 77 | 78 | cd zdbd 79 | make release STATIC=1 80 | cd .. 81 | 82 | make 83 | 84 | - name: Create archive 85 | run: | 86 | tar -czf threefoldtech-0-db-release-${{ steps.versions.outputs.branch }}-${{ steps.versions.outputs.commit }}.tar.gz bin 87 | 88 | - name: Publish flist (tf-autobuilder, ${{ steps.versions.outputs.commit }}) 89 | if: success() 90 | uses: threefoldtech/publish-flist@master 91 | with: 92 | threefold: ${{ secrets.HUB_TOKEN }} 93 | action: upload 94 | name: threefoldtech-0-db-release-${{ steps.versions.outputs.branch }}-${{ steps.versions.outputs.commit }}.tar.gz 95 | 96 | -------------------------------------------------------------------------------- /.github/workflows/package.yaml: -------------------------------------------------------------------------------- 1 | name: Publish update 2 | on: 3 | push: 4 | branches: ['development-v2'] 5 | 6 | env: 7 | REGISTRY: ghcr.io 8 | IMAGE_NAME: ${{ github.repository }} 9 | 10 | jobs: 11 | build-and-push-image: 12 | name: Create GitHub Package 13 | runs-on: ubuntu-latest 14 | 15 | permissions: 16 | contents: read 17 | packages: write 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v4 22 | 23 | - name: Log in to the Container registry 24 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 25 | with: 26 | registry: ${{ env.REGISTRY }} 27 | username: ${{ github.actor }} 28 | password: ${{ secrets.GITHUB_TOKEN }} 29 | 30 | - name: Extract metadata (tags, labels) for Docker 31 | id: meta 32 | uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 33 | with: 34 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 35 | 36 | - name: Build and push Docker image 37 | uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 38 | with: 39 | context: .github/workflows 40 | push: true 41 | tags: ${{ steps.meta.outputs.tags }} 42 | labels: ${{ steps.meta.outputs.labels }} 43 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Build release 2 | on: 3 | release: 4 | types: [created] 5 | 6 | env: 7 | REGISTRY: ghcr.io 8 | IMAGE_NAME: ${{ github.repository }} 9 | 10 | jobs: 11 | static: 12 | name: Create static release artifact 13 | runs-on: ubuntu-latest 14 | container: 15 | image: alpine:3.13 16 | steps: 17 | - name: Checkout the repository 18 | uses: actions/checkout@master 19 | 20 | - name: Install dependencies 21 | run: | 22 | apk add alpine-sdk 23 | 24 | - name: Build production 0-db 25 | run: | 26 | cd libzdb 27 | make release 28 | 29 | cd ../zdbd 30 | make STATIC=1 release 31 | 32 | version=$(grep ZDBD_VERSION zdbd.h | awk '{ print $3 }' | sed s/'"'//g) 33 | cp zdb ../zdb-${version}-linux-amd64-static 34 | 35 | - name: Upload the artifacts 36 | uses: skx/github-action-publish-binaries@master 37 | env: 38 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 39 | with: 40 | args: 'zdb-*-linux-amd64-static' 41 | 42 | build-and-push-image: 43 | name: Create GitHub Package 44 | runs-on: ubuntu-latest 45 | 46 | permissions: 47 | contents: read 48 | packages: write 49 | 50 | steps: 51 | - name: Checkout repository 52 | uses: actions/checkout@v4 53 | 54 | - name: Log in to the Container registry 55 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 56 | with: 57 | registry: ${{ env.REGISTRY }} 58 | username: ${{ github.actor }} 59 | password: ${{ secrets.GITHUB_TOKEN }} 60 | 61 | - name: Extract metadata (tags, labels) for Docker 62 | id: meta 63 | uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 64 | with: 65 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 66 | 67 | - name: Build and push Docker image 68 | uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 69 | with: 70 | context: .github/workflows 71 | push: true 72 | tags: ${{ steps.meta.outputs.tags }} 73 | labels: ${{ steps.meta.outputs.labels }} 74 | 75 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests and Coverage 2 | on: [push] 3 | jobs: 4 | generate: 5 | name: Run tests 6 | runs-on: ubuntu-20.04 7 | steps: 8 | - name: Checkout the repository 9 | uses: actions/checkout@master 10 | 11 | - name: Install dependencies 12 | run: | 13 | sudo apt-get update 14 | sudo apt-get install -y pkg-config libhiredis-dev build-essential curl 15 | 16 | - name: Building 0-db 17 | run: | 18 | make COVERAGE=1 19 | cd tests 20 | make 21 | cd .. 22 | 23 | - name: Run tests suite 24 | run: | 25 | bash tests/run.sh 26 | 27 | - name: Upload codecov 28 | uses: codecov/codecov-action@v1 29 | 30 | -------------------------------------------------------------------------------- /.github/workflows/zdb-process: -------------------------------------------------------------------------------- 1 | #!/bin/ash 2 | zdb="/bin/zdb" 3 | args="" 4 | 5 | [[ ! -z "${DATADIR}" ]] && args="--data ${DATADIR}" 6 | [[ ! -z "${INDEXDIR}" ]] && args="${args} --index ${INDEXDIR}" 7 | [[ ! -z "${DATASIZE}" ]] && args="${args} --datasize ${DATASIZE}" 8 | [[ ! -z "${ADMIN}" ]] && args="${args} --admin ${ADMIN}" 9 | [[ ! -z "${PROTECT}" ]] && args="${args} --protect" 10 | 11 | ${zdb} ${args} 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # C 2 | 3 | # Prerequisites 4 | *.d 5 | 6 | # Object files 7 | *.o 8 | *.ko 9 | *.obj 10 | *.elf 11 | 12 | # Linker output 13 | *.ilk 14 | *.map 15 | *.exp 16 | 17 | # Precompiled Headers 18 | *.gch 19 | *.pch 20 | 21 | # Libraries 22 | *.lib 23 | *.a 24 | *.la 25 | *.lo 26 | 27 | # Shared objects (inc. Windows DLLs) 28 | *.dll 29 | *.so 30 | *.so.* 31 | *.dylib 32 | 33 | # Executables 34 | *.exe 35 | *.out 36 | *.app 37 | *.i*86 38 | *.x86_64 39 | *.hex 40 | 41 | # Debug files 42 | *.dSYM/ 43 | *.su 44 | *.idb 45 | *.pdb 46 | 47 | # Kernel Module Compile Results 48 | *.mod* 49 | *.cmd 50 | .tmp_versions/ 51 | modules.order 52 | Module.symvers 53 | Mkfile.old 54 | dkms.conf 55 | bin/zdb* 56 | 57 | 58 | # binary files 59 | bin/zdb 60 | bin/zdb-integrity-check 61 | bin/zdb-index-dump 62 | bin/zdbtests 63 | src/zdb 64 | tools/integrity-check/integrity-check 65 | tools/index-dump/index-dump 66 | tools/index-rebuild/index-rebuild 67 | tools/compaction/compaction 68 | tools/namespace-editor/namespace-editor 69 | tests/zdbtests 70 | utilities/db-sync/db-sync 71 | utilities/db-replicate/db-replicate 72 | utilities/db-mirror/db-mirror 73 | tests/pipeline/pipeline 74 | zos.log 75 | zdbd/zdb 76 | zdbd/zdbd 77 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all release: 2 | $(MAKE) -C libzdb $@ 3 | $(MAKE) -C zdbd $@ 4 | $(MAKE) -C tools $@ 5 | 6 | cp -f zdbd/zdb bin/ 7 | cp -f tools/integrity-check/integrity-check bin/zdb-integrity-check 8 | cp -f tools/index-dump/index-dump bin/zdb-index-dump 9 | # cp -f tools/compaction/compaction bin/zdb-compaction 10 | cp -f tools/namespace-editor/namespace-editor bin/zdb-namespace-editor 11 | cp -f tools/namespace-dump/namespace-dump bin/zdb-namespace-dump 12 | 13 | clean: 14 | $(MAKE) -C libzdb $@ 15 | $(MAKE) -C zdbd $@ 16 | $(MAKE) -C tools $@ 17 | 18 | mrproper: 19 | $(MAKE) -C libzdb $@ 20 | $(MAKE) -C zdbd $@ 21 | $(MAKE) -C tools $@ 22 | $(RM) bin/* 23 | 24 | install: 25 | $(MAKE) -C zdbd $@ 26 | -------------------------------------------------------------------------------- /bin/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/threefoldtech/0-db/a6904f6d4193bdc5d926da1e65ebb75a4558e067/bin/.keep -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = zdb-example 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -O0 -W -Wall -Wextra -msse4.2 -Wno-implicit-fallthrough -I../libzdb 6 | LDFLAGS += -rdynamic ../libzdb/libzdb.a 7 | 8 | all: $(EXEC) 9 | 10 | $(EXEC): $(OBJ) 11 | $(CC) -o $@ $^ $(LDFLAGS) 12 | 13 | %.o: %.c 14 | $(CC) $(CFLAGS) -c $< 15 | 16 | clean: 17 | $(RM) *.o 18 | 19 | mrproper: clean 20 | $(RM) $(EXEC) 21 | -------------------------------------------------------------------------------- /examples/zdb-example.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "libzdb.h" 6 | 7 | void dump_type(zdb_api_type_t status) { 8 | char *msg = zdb_api_debug_type(status); 9 | printf("[+] example: " COLOR_CYAN "%s" COLOR_RESET "\n", msg); 10 | } 11 | 12 | int stuff(zdb_settings_t *settings) { 13 | (void) settings; 14 | namespace_t *ns = namespace_get_default(); 15 | 16 | // 17 | // set a key 18 | // 19 | char *key = "MyKey"; 20 | char *data = "Hello World !"; 21 | 22 | zdb_api_t *reply = zdb_api_set(ns, key, strlen(key), data, strlen(data)); 23 | dump_type(reply->status); 24 | zdb_api_reply_free(reply); 25 | 26 | // 27 | // ensure existance 28 | // 29 | printf("[+] example: checking if key exists\n"); 30 | reply = zdb_api_exists(ns, key, strlen(key)); 31 | dump_type(reply->status); 32 | zdb_api_reply_free(reply); 33 | 34 | // 35 | // looking up for this entry 36 | // 37 | reply = zdb_api_get(ns, key, strlen(key)); 38 | dump_type(reply->status); 39 | 40 | if(reply->status == ZDB_API_ENTRY) { 41 | zdb_api_entry_t *entry = reply->payload; 42 | printf("[+] entry: key: <%.*s>\n", (int) entry->key.size, entry->key.payload); 43 | printf("[+] entry: data: <%.*s>\n", (int) entry->payload.size, entry->payload.payload); 44 | } 45 | 46 | zdb_api_reply_free(reply); 47 | 48 | // 49 | // checking consistancy 50 | // 51 | printf("[+] example: checking data consistancy\n"); 52 | reply = zdb_api_check(ns, key, strlen(key)); 53 | dump_type(reply->status); 54 | zdb_api_reply_free(reply); 55 | 56 | // 57 | // deleting the key 58 | // 59 | printf("[+] example: deleting key\n"); 60 | reply = zdb_api_del(ns, key, strlen(key)); 61 | dump_type(reply->status); 62 | zdb_api_reply_free(reply); 63 | 64 | // 65 | // checking if key was well removed 66 | // 67 | printf("[+] example: checking for key suppression\n"); 68 | reply = zdb_api_exists(ns, key, strlen(key)); 69 | dump_type(reply->status); 70 | zdb_api_reply_free(reply); 71 | 72 | 73 | 74 | return 0; 75 | } 76 | 77 | // int main(int argc, char *argv[]) { 78 | int main() { 79 | printf("[*] 0-db example small example code\n"); 80 | printf("[*] 0-db engine v%s\n", zdb_version()); 81 | printf("[*] 0-db engine revision: %s\n", zdb_revision()); 82 | 83 | // initialize default settings 84 | zdb_settings_t *zdb_settings = zdb_initialize(); 85 | 86 | // set a dump id, you should always set an id, if possible 87 | // unique, this id is mainly used to differenciate running 88 | // zdb when using hook system 89 | zdb_id_set("example-run"); 90 | 91 | // each zdb have an instance id generated randomly 92 | // when initialized 93 | printf("[+] instance id: %u\n", zdb_instanceid_get()); 94 | 95 | // set custom data and index path for our database 96 | // keep all others settings by default 97 | zdb_settings->datapath = "/tmp/zdb-example/data"; 98 | zdb_settings->indexpath = "/tmp/zdb-example/index"; 99 | // zdb_settings->mode = ZDB_MODE_SEQUENTIAL; 100 | 101 | // open the database 102 | zdb_open(zdb_settings); 103 | 104 | // --------------------- 105 | stuff(zdb_settings); 106 | // --------------------- 107 | 108 | // closing and memory freeing everything 109 | // related to database stuff 110 | zdb_close(zdb_settings); 111 | 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /libzdb/Makefile: -------------------------------------------------------------------------------- 1 | LIB = libzdb 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -fPIC -std=gnu11 -O0 -W -Wall -Wextra -Wno-implicit-fallthrough 6 | LDFLAGS += -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | # grab version from git, if possible 15 | REVISION := $(shell git describe --abbrev=8 --dirty --always --tags) 16 | ifeq ($(REVISION),) 17 | REVISION := $(shell grep ZDB_VERSION libzdb.h | awk '{ print $$3 }' | sed s/'"'//g) 18 | endif 19 | 20 | # add revision to build 21 | CFLAGS += -DZDB_REVISION=\"$(REVISION)\" 22 | 23 | ifeq ($(STATIC),1) 24 | LDFLAGS += -static 25 | endif 26 | 27 | ifeq ($(COVERAGE),1) 28 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 29 | LDFLAGS += -lgcov --coverage 30 | endif 31 | 32 | ifeq ($(PROFILE),1) 33 | CFLAGS += -pg 34 | LDFLAGS += -pg 35 | endif 36 | 37 | all: $(LIB).a $(LIB).so 38 | 39 | release: CFLAGS += -DRELEASE -O2 40 | release: clean $(LIB).a $(LIB).so 41 | 42 | $(LIB).so: $(OBJ) 43 | $(CC) -shared -o $@ $^ $(LDFLAGS) 44 | 45 | $(LIB).a: $(OBJ) 46 | ar rcs $(LIB).a $(OBJ) 47 | 48 | %.o: %.c 49 | $(CC) $(CFLAGS) -c $< 50 | 51 | clean: 52 | $(RM) *.o 53 | 54 | mrproper: clean 55 | $(RM) $(LIB).a $(LIB).so 56 | $(RM) *.gcno *.gcda *.gcov 57 | -------------------------------------------------------------------------------- /libzdb/api.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_API_H 2 | #define __ZDB_API_H 3 | 4 | typedef enum zdb_api_type_t { 5 | ZDB_API_SUCCESS, 6 | ZDB_API_FAILURE, 7 | ZDB_API_ENTRY, 8 | ZDB_API_UP_TO_DATE, 9 | ZDB_API_BUFFER, 10 | ZDB_API_NOT_FOUND, 11 | ZDB_API_DELETED, 12 | ZDB_API_INTERNAL_ERROR, 13 | ZDB_API_TRUE, 14 | ZDB_API_FALSE, 15 | ZDB_API_INSERT_DENIED, 16 | 17 | ZDB_API_ITEMS_TOTAL // last element 18 | 19 | } zdb_api_type_t; 20 | 21 | typedef struct zdb_api_t { 22 | zdb_api_type_t status; 23 | void *payload; 24 | 25 | } zdb_api_t; 26 | 27 | typedef struct zdb_api_buffer_t { 28 | uint8_t *payload; 29 | size_t size; 30 | 31 | } zdb_api_buffer_t; 32 | 33 | typedef struct zdb_api_entry_t { 34 | zdb_api_buffer_t key; 35 | zdb_api_buffer_t payload; 36 | 37 | } zdb_api_entry_t; 38 | 39 | zdb_api_t *zdb_api_set(namespace_t *ns, void *key, size_t ksize, void *payload, size_t psize); 40 | zdb_api_t *zdb_api_get(namespace_t *ns, void *key, size_t ksize); 41 | zdb_api_t *zdb_api_exists(namespace_t *ns, void *key, size_t ksize); 42 | zdb_api_t *zdb_api_check(namespace_t *ns, void *key, size_t ksize); 43 | zdb_api_t *zdb_api_del(namespace_t *ns, void *key, size_t ksize); 44 | 45 | char *zdb_api_debug_type(zdb_api_type_t type); 46 | void zdb_api_reply_free(zdb_api_t *reply); 47 | 48 | // index helper 49 | char *zdb_index_date(uint32_t epoch, char *target, size_t length); 50 | 51 | // index loader 52 | void zdb_index_set_id(index_root_t *root, uint64_t fileid); 53 | 54 | int zdb_index_open_readonly(index_root_t *root, fileid_t fileid); 55 | int zdb_index_open_readwrite(index_root_t *root, fileid_t fileid); 56 | void zdb_index_close(index_root_t *zdbindex); 57 | 58 | index_root_t *zdb_index_init_lazy(zdb_settings_t *settings, char *indexdir, void *namespace); 59 | index_root_t *zdb_index_init(zdb_settings_t *settings, char *indexdir, void *namespace, index_branch_t **branches); 60 | uint64_t zdb_index_availity_check(index_root_t *root); 61 | 62 | // index header validity 63 | index_header_t *zdb_index_descriptor_load(index_root_t *root); 64 | index_header_t *zdb_index_descriptor_validate(index_header_t *header, index_root_t *root); 65 | 66 | // low level index 67 | index_item_t *zdb_index_raw_fetch_entry(index_root_t *root); 68 | off_t zdb_index_raw_offset(index_root_t *root); 69 | uint64_t zdb_index_next_id(index_root_t *root); 70 | 71 | // internal checksum 72 | uint32_t zdb_checksum_crc32(const uint8_t *bytes, ssize_t length); 73 | 74 | // data loader 75 | data_root_t *zdb_data_init_lazy(zdb_settings_t *settings, char *datapath, fileid_t dataid); 76 | int zdb_data_open_readonly(data_root_t *root); 77 | 78 | data_header_t *zdb_data_descriptor_load(data_root_t *root); 79 | data_header_t *zdb_data_descriptor_validate(data_header_t *header, data_root_t *root); 80 | #endif 81 | -------------------------------------------------------------------------------- /libzdb/bootstrap.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "libzdb.h" 13 | #include "libzdb_private.h" 14 | 15 | static uint32_t zdb_instanceid_generate() { 16 | struct timeval tv; 17 | 18 | // generating 'random id', greater than zero 19 | gettimeofday(&tv, NULL); 20 | srand((time_t) tv.tv_usec); 21 | 22 | return (uint32_t) ((rand() % (1 << 30)) + 1); 23 | } 24 | 25 | // 26 | // main settings initializer 27 | // 28 | zdb_settings_t *zdb_initialize() { 29 | zdb_settings_t *s = &zdb_rootsettings; 30 | 31 | if(s->initialized == 1) 32 | return NULL; 33 | 34 | // apply default settings 35 | s->datapath = ZDB_DEFAULT_DATAPATH; 36 | s->indexpath = ZDB_DEFAULT_INDEXPATH; 37 | s->datasize = ZDB_DEFAULT_DATA_MAXSIZE; 38 | 39 | // running 0-db in mixed mode by default 40 | // 41 | // this flag can be used on runtime to specify 42 | // if mixed mode is allowed or not, if a specific 43 | // mode is set here, you could restrict instance to 44 | // a single mode, but this is up to caller to enable 45 | // restriction, library doesn't restrict anything 46 | s->mode = ZDB_MODE_MIX; 47 | 48 | // resetting values 49 | s->verbose = 0; 50 | s->dump = 0; 51 | s->sync = 0; 52 | s->synctime = 0; 53 | s->hook = NULL; 54 | s->maxsize = 0; 55 | 56 | // initialize stats and init time 57 | memset(&s->stats, 0x00, sizeof(zdb_stats_t)); 58 | gettimeofday(&s->stats.inittime, NULL); 59 | 60 | // initialize hooks list 61 | hook_initialize(&s->hooks); 62 | 63 | // initialize instance id 64 | s->iid = zdb_instanceid_generate(); 65 | 66 | // set a global lock, already initialized 67 | s->initialized = 1; 68 | 69 | return s; 70 | } 71 | 72 | static int zdb_lock(char *lockpath) { 73 | int fd; 74 | 75 | zdb_debug("[+] bootstrap: checking lockfile: %s\n", lockpath); 76 | 77 | if((fd = open(lockpath, O_CREAT | O_RDONLY, 0664)) < 0) { 78 | zdb_warnp(lockpath); 79 | return -1; 80 | } 81 | 82 | if(flock(fd, LOCK_EX | LOCK_NB) < 0) { 83 | if(errno != EWOULDBLOCK) { 84 | zdb_warnp(lockpath); 85 | return 1; 86 | } 87 | 88 | zdb_danger("[-] bootstrap: an instance of zdb is already using: %s", lockpath); 89 | return -1; 90 | } 91 | 92 | return fd; 93 | } 94 | 95 | zdb_settings_t *zdb_open(zdb_settings_t *zdb_settings) { 96 | char lockpath[512]; 97 | 98 | // 99 | // ensure default directories exists 100 | // for a fresh start if this is a new instance 101 | // 102 | if(zdb_dir_exists(zdb_settings->datapath) != ZDB_DIRECTORY_EXISTS) { 103 | zdb_verbose("[+] system: creating datapath: %s\n", zdb_settings->datapath); 104 | zdb_dir_create(zdb_settings->datapath); 105 | } 106 | 107 | if(zdb_dir_exists(zdb_settings->indexpath) != ZDB_DIRECTORY_EXISTS) { 108 | zdb_verbose("[+] system: creating indexpath: %s\n", zdb_settings->indexpath); 109 | zdb_dir_create(zdb_settings->indexpath); 110 | } 111 | 112 | // ensure that data and index does not points to the 113 | // same directory 114 | char *datapath = realpath(zdb_settings->datapath, NULL); 115 | char *indexpath = realpath(zdb_settings->indexpath, NULL); 116 | 117 | if(strcmp(datapath, indexpath) == 0) { 118 | zdb_danger("[-] system: cannot use the same directory for index and data path"); 119 | return NULL; 120 | } 121 | 122 | free(datapath); 123 | free(indexpath); 124 | 125 | // check/install a lock on the index and data directory to avoid 126 | // multiple instance of zdb running on theses directories 127 | snprintf(lockpath, sizeof(lockpath), "%s/.lockfile", zdb_settings->indexpath); 128 | if((zdb_settings->indexlock = zdb_lock(lockpath)) < 0) 129 | return NULL; 130 | 131 | snprintf(lockpath, sizeof(lockpath), "%s/.lockfile", zdb_settings->datapath); 132 | if((zdb_settings->datalock = zdb_lock(lockpath)) < 0) 133 | return NULL; 134 | 135 | // namespace is the root of the whole index/data system 136 | // anything related to data is always attached to at least 137 | // one namespace (the default), and all the others 138 | // are based on a fork of namespace 139 | // 140 | // the namespace system will take care about all the loading 141 | // and the destruction 142 | namespaces_init(zdb_settings); 143 | 144 | return zdb_settings; 145 | } 146 | 147 | void zdb_close(zdb_settings_t *zdb_settings) { 148 | zdb_debug("[+] bootstrap: closing database\n"); 149 | namespaces_destroy(zdb_settings); 150 | 151 | // cleanup hook subsystem 152 | hook_destroy(&zdb_settings->hooks); 153 | 154 | zdb_debug("[+] bootstrap: cleaning library\n"); 155 | free(zdb_settings->zdbid); 156 | zdb_settings->zdbid = NULL; 157 | zdb_settings->initialized = 0; 158 | 159 | zdb_debug("[+] bootstrap: releasing locks\n"); 160 | close(zdb_settings->indexlock); 161 | close(zdb_settings->datalock); 162 | } 163 | 164 | -------------------------------------------------------------------------------- /libzdb/bootstrap.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_BOOTSTRAP_H 2 | #define __ZDB_BOOTSTRAP_H 3 | 4 | zdb_settings_t *zdb_initialize(); 5 | zdb_settings_t *zdb_open(zdb_settings_t *zdb_settings); 6 | void zdb_close(zdb_settings_t *zdb_settings); 7 | #endif 8 | -------------------------------------------------------------------------------- /libzdb/crc32.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #ifdef __SSE4_2__ 6 | // x86_64 SSE4.2 CRC32C SIMD 7 | #include 8 | #endif 9 | 10 | #ifdef __ARM_FEATURE_CRC32 11 | // ARMv8 CRC32C SIMD 12 | #include 13 | #endif 14 | 15 | // default software crc32c table 16 | static const uint32_t crc32c_table[256] = { 17 | 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, 18 | 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, 19 | 0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, 20 | 0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L, 21 | 0x105EC76FL, 0xE235446CL, 0xF165B798L, 0x030E349BL, 22 | 0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L, 23 | 0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L, 0x89D76C54L, 24 | 0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL, 25 | 0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL, 26 | 0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L, 27 | 0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L, 28 | 0x6DFE410EL, 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL, 29 | 0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L, 30 | 0xF779DEAEL, 0x05125DADL, 0x1642AE59L, 0xE4292D5AL, 31 | 0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL, 32 | 0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L, 0x6EF07595L, 33 | 0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L, 34 | 0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L, 35 | 0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L, 36 | 0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L, 37 | 0x5125DAD3L, 0xA34E59D0L, 0xB01EAA24L, 0x42752927L, 38 | 0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L, 39 | 0xDBFC821CL, 0x2997011FL, 0x3AC7F2EBL, 0xC8AC71E8L, 40 | 0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L, 41 | 0x61C69362L, 0x93AD1061L, 0x80FDE395L, 0x72966096L, 42 | 0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L, 43 | 0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L, 44 | 0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L, 45 | 0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L, 46 | 0xB602C312L, 0x44694011L, 0x5739B3E5L, 0xA55230E6L, 47 | 0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L, 48 | 0x3CDB9BDDL, 0xCEB018DEL, 0xDDE0EB2AL, 0x2F8B6829L, 49 | 0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL, 50 | 0x456CAC67L, 0xB7072F64L, 0xA457DC90L, 0x563C5F93L, 51 | 0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L, 52 | 0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL, 53 | 0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L, 54 | 0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL, 55 | 0x1871A4D8L, 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL, 56 | 0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L, 57 | 0xA24BB5A6L, 0x502036A5L, 0x4370C551L, 0xB11B4652L, 58 | 0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL, 59 | 0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL, 0x3BC21E9DL, 60 | 0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L, 61 | 0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL, 62 | 0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L, 63 | 0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L, 64 | 0xFF56BD19L, 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL, 65 | 0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L, 66 | 0x0417B1DBL, 0xF67C32D8L, 0xE52CC12CL, 0x1747422FL, 67 | 0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL, 68 | 0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L, 0x9D9E1AE0L, 69 | 0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL, 70 | 0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L, 71 | 0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L, 72 | 0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL, 73 | 0xE330A81AL, 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL, 74 | 0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L, 75 | 0x69E9F0D5L, 0x9B8273D6L, 0x88D28022L, 0x7AB90321L, 76 | 0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL, 77 | 0xF36E6F75L, 0x0105EC76L, 0x12551F82L, 0xE03E9C81L, 78 | 0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL, 79 | 0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL, 80 | 0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L 81 | }; 82 | 83 | // default crc32c software implementation 84 | static uint32_t crc32c_software(const uint8_t *data, unsigned int length) { 85 | uint32_t crc = 0; 86 | 87 | while(length--) 88 | crc = crc32c_table[(crc ^ *data++) & 0xFFL] ^ (crc >> 8); 89 | 90 | return crc; 91 | } 92 | 93 | // x86_64 sse4.2 hardware optimized implementation 94 | #ifdef __SSE4_2__ 95 | static uint32_t crc32c_sse42(const uint8_t *bytes, size_t len) { 96 | uint64_t *input = (uint64_t *) bytes; 97 | uint32_t hash = 0; 98 | size_t i = 0; 99 | 100 | if(len >= 8) { 101 | // compute 64 bits chunks 102 | for(i = 0; i < len - 8; i += 8) 103 | hash = _mm_crc32_u64(hash, *input++); 104 | } 105 | 106 | for(; i < len; i++) 107 | hash = _mm_crc32_u8(hash, bytes[i]); 108 | 109 | return hash; 110 | } 111 | #endif 112 | 113 | // armv8 hardware optimized implementation 114 | #ifdef __ARM_FEATURE_CRC32 115 | static uint32_t crc32c_armv8(const uint8_t *bytes, size_t len) { 116 | uint64_t *input = (uint64_t *) bytes; 117 | uint32_t hash = 0; 118 | size_t i = 0; 119 | 120 | if(len >= 8) { 121 | // compute 64 bits chunks 122 | for(i = 0; i < len - 8; i += 8) 123 | hash = __crc32cd(hash, *input++); 124 | } 125 | 126 | for(; i < len; i++) 127 | hash = __crc32cb(hash, bytes[i]); 128 | 129 | return hash; 130 | } 131 | #endif 132 | 133 | char *zdb_crc32_engine_value() { 134 | #ifdef __SSE4_2__ 135 | return "sse42"; 136 | #endif 137 | 138 | #ifdef __ARM_FEATURE_CRC32 139 | return "armv8"; 140 | #endif 141 | 142 | return "software"; 143 | } 144 | 145 | uint32_t zdb_crc32(const uint8_t *bytes, ssize_t length) { 146 | #ifdef __SSE4_2__ 147 | return crc32c_sse42(bytes, length); 148 | #endif 149 | 150 | #ifdef __ARM_FEATURE_CRC32 151 | return crc32c_armv8(bytes, length); 152 | #endif 153 | 154 | return crc32c_software(bytes, length); 155 | } 156 | 157 | -------------------------------------------------------------------------------- /libzdb/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_CRC32_H 2 | #define __ZDB_CRC32_H 3 | 4 | // return a string with engine in-use for crc32 5 | char *zdb_crc32_engine_value(); 6 | 7 | // libzdb crc32 with auto-selection of best engine 8 | uint32_t zdb_crc32(const uint8_t *bytes, ssize_t length); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /libzdb/filesystem.c: -------------------------------------------------------------------------------- 1 | #ifndef __APPLE__ 2 | #define _XOPEN_SOURCE 500 3 | #endif 4 | #define _DEFAULT_SOURCE 5 | #define _BSD_SOURCE 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "libzdb.h" 17 | #include "libzdb_private.h" 18 | 19 | // 20 | // system directory management 21 | // 22 | int zdb_dir_exists(char *path) { 23 | struct stat sb; 24 | 25 | if(stat(path, &sb) != 0) 26 | return ZDB_PATH_NOT_AVAILABLE; 27 | 28 | if(!S_ISDIR(sb.st_mode)) 29 | return ZDB_PATH_IS_NOT_DIRECTORY; 30 | 31 | return ZDB_DIRECTORY_EXISTS; 32 | } 33 | 34 | int zdb_dir_create(char *path) { 35 | char tmp[ZDB_PATH_MAX], *p = NULL; 36 | size_t len; 37 | 38 | snprintf(tmp, sizeof(tmp), "%s", path); 39 | len = strlen(tmp); 40 | if(tmp[len - 1] == '/') 41 | tmp[len - 1] = 0; 42 | 43 | for(p = tmp + 1; *p; p++) { 44 | if(*p == '/') { 45 | *p = 0; 46 | mkdir(tmp, S_IRWXU); 47 | *p = '/'; 48 | } 49 | } 50 | 51 | return mkdir(tmp, S_IRWXU); 52 | } 53 | 54 | static int dir_remove_cb(const char *fpath, const struct stat *sb, int tflag, struct FTW *ftwbuf) { 55 | (void) sb; 56 | (void) ftwbuf; 57 | (void) tflag; 58 | char *fullpath = (char *) fpath; 59 | int value; 60 | 61 | zdb_debug("[+] filesystem: remove: %s\n", fullpath); 62 | 63 | if((value = remove(fullpath))) 64 | zdb_warnp(fullpath); 65 | 66 | return 0; 67 | } 68 | 69 | int zdb_dir_remove(char *path) { 70 | return nftw(path, dir_remove_cb, 64, FTW_DEPTH | FTW_PHYS); 71 | } 72 | 73 | static int dir_clean_cb(const char *fpath, const struct stat *sb, int tflag, struct FTW *ftwbuf) { 74 | (void) sb; 75 | (void) ftwbuf; 76 | (void) tflag; 77 | char *fullpath = (char *) fpath; 78 | size_t length = strlen(fullpath); 79 | 80 | if(strncmp(fullpath + length - 14, "zdb-data-", 9) == 0) { 81 | zdb_debug("[+] filesystem: removing datafile: %s\n", fullpath); 82 | unlink(fullpath); 83 | } 84 | 85 | if(strncmp(fullpath + length - 15, "zdb-index-", 10) == 0) { 86 | zdb_debug("[+] filesystem: removing indexfile: %s\n", fullpath); 87 | unlink(fullpath); 88 | } 89 | 90 | return 0; 91 | } 92 | 93 | int zdb_dir_clean_payload(char *path) { 94 | return nftw(path, dir_clean_cb, 64, FTW_DEPTH | FTW_PHYS); 95 | } 96 | 97 | int zdb_file_exists(char *path) { 98 | struct stat sb; 99 | 100 | if(stat(path, &sb) != 0) 101 | return ZDB_PATH_NOT_AVAILABLE; 102 | 103 | if(S_ISDIR(sb.st_mode)) 104 | return ZDB_PATH_IS_DIRECTORY; 105 | 106 | return ZDB_FILE_EXISTS; 107 | } 108 | -------------------------------------------------------------------------------- /libzdb/filesystem.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_FILESYSTEM_H 2 | #define __ZDB_FILESYSTEM_H 3 | 4 | int zdb_dir_exists(char *path); 5 | int zdb_dir_create(char *path); 6 | int zdb_dir_remove(char *path); 7 | int zdb_dir_clean_payload(char *path); 8 | int zdb_file_exists(char *path); 9 | 10 | #define ZDB_FILE_EXISTS 0 11 | #define ZDB_DIRECTORY_EXISTS 1 12 | #define ZDB_PATH_NOT_AVAILABLE 2 13 | #define ZDB_PATH_IS_DIRECTORY 3 14 | #define ZDB_PATH_IS_NOT_DIRECTORY 4 15 | #endif 16 | -------------------------------------------------------------------------------- /libzdb/hook.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_HOOK_H 2 | #define __ZDB_HOOK_H 3 | 4 | #define ZDB_HOOKS_INITIAL_LENGTH 8 5 | #define ZDB_HOOKS_EXPIRE_SECONDS 300 6 | 7 | // a hook is called when an external application/script 8 | // is provided at runtime, this hook executable 9 | // will be executed in background for different actions 10 | typedef struct hook_t { 11 | pid_t pid; // process id when started 12 | size_t argc; // length or arguments 13 | char **argv; // arguments 14 | time_t created; // creation timestamp 15 | time_t finished; // exit timestamp 16 | int status; // exit status code 17 | 18 | size_t argidx; // current argument index (used for fillin) 19 | 20 | } hook_t; 21 | 22 | typedef struct zdb_hooks_t { 23 | size_t length; 24 | size_t active; 25 | hook_t **hooks; 26 | 27 | } zdb_hooks_t; 28 | 29 | // initialize hook subsystem 30 | size_t hook_initialize(zdb_hooks_t *hooks); 31 | 32 | // cleanup hook subsystem 33 | void hook_destroy(zdb_hooks_t *hooks); 34 | 35 | hook_t *hook_new(char *name, size_t argc); 36 | int hook_append(hook_t *hook, char *argument); 37 | pid_t hook_execute(hook_t *hook); 38 | int hook_execute_wait(hook_t *hook); 39 | 40 | // need to be called periodicly to cleanup zombies 41 | void libzdb_hooks_cleanup(); 42 | #endif 43 | -------------------------------------------------------------------------------- /libzdb/index_branch.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "libzdb.h" 7 | #include "libzdb_private.h" 8 | 9 | // maximum allowed branch in memory 10 | // 11 | // this settings is mainly the most important to 12 | // determine the keys lookup time 13 | // 14 | // the more bits you allows here, the more buckets 15 | // can be used for lookup without collision 16 | // 17 | // the index works like a hash-table and uses crc32 'hash' 18 | // algorithm, the result of the crc32 is used to point to 19 | // the bucket, but using a full 32-bits hashlist would 20 | // consume more than (2^32 * 8) bytes of memory (on 64-bits) 21 | // 22 | // the default settings sets this to 24 bits, which allows 23 | // 16 millions direct entries, collisions uses linked-list 24 | // 25 | // makes sur mask and amount of branch are always in relation 26 | // use 'index_set_buckets_bits' to be sure 27 | uint32_t buckets_branches = (1 << 24); 28 | uint32_t buckets_mask = (1 << 24) - 1; 29 | 30 | // WARNING: this doesn't resize anything, you should calls this 31 | // only before initialization 32 | int index_set_buckets_bits(uint8_t bits) { 33 | buckets_branches = 1 << bits; 34 | buckets_mask = (1 << bits) - 1; 35 | 36 | return buckets_branches; 37 | } 38 | 39 | // 40 | // index branch 41 | // this implementation uses a lazy load of branches 42 | // this allows us to use a lot of branches (buckets_branches) in this case) 43 | // without consuming all the memory if we don't need it 44 | // 45 | index_branch_t **index_buckets_init() { 46 | return (index_branch_t **) calloc(sizeof(index_branch_t *), buckets_branches); 47 | } 48 | 49 | index_branch_t *index_branch_init(index_branch_t **branches, uint32_t branchid) { 50 | // zdb_debug("[+] initializing branch id 0x%x\n", branchid); 51 | 52 | branches[branchid] = malloc(sizeof(index_branch_t)); 53 | index_branch_t *branch = branches[branchid]; 54 | 55 | branch->length = 0; 56 | branch->last = NULL; 57 | branch->list = NULL; 58 | 59 | return branch; 60 | } 61 | 62 | void index_branch_free(index_branch_t **branches, uint32_t branchid) { 63 | // this branch was not allocated 64 | if(!branches[branchid]) 65 | return; 66 | 67 | index_entry_t *entry = branches[branchid]->list; 68 | index_entry_t *next = NULL; 69 | 70 | // deleting branch content by 71 | // iterate over the linked-list 72 | for(; entry; entry = next) { 73 | next = entry->next; 74 | free(entry); 75 | } 76 | 77 | // deleting branch 78 | free(branches[branchid]); 79 | } 80 | 81 | // returns branch from rootindex, if branch is not allocated yet, returns NULL 82 | // useful for any read on the index in memory 83 | index_branch_t *index_branch_get(index_branch_t **branches, uint32_t branchid) { 84 | if(!branches) 85 | return NULL; 86 | 87 | return branches[branchid]; 88 | } 89 | 90 | // returns branch from rootindex, if branch doesn't exists, it will be allocated 91 | // (useful for any write in the index in memory) 92 | index_branch_t *index_branch_get_allocate(index_branch_t **branches, uint32_t branchid) { 93 | if(!branches[branchid]) 94 | return index_branch_init(branches, branchid); 95 | 96 | // zdb_debug("[+] branch: exists: %lu entries\n", branches[branchid]->length); 97 | return branches[branchid]; 98 | } 99 | 100 | // append an entry (item) to the memory list 101 | // since we use a linked-list, the logic of appending 102 | // only occures here 103 | // 104 | // if there is no index, we just skip the appending 105 | index_entry_t *index_branch_append(index_branch_t **branches, uint32_t branchid, index_entry_t *entry) { 106 | index_branch_t *branch; 107 | 108 | if(!branches) 109 | return NULL; 110 | 111 | // grabbing the branch 112 | branch = index_branch_get_allocate(branches, branchid); 113 | branch->length += 1; 114 | 115 | // adding this item and pointing previous last one 116 | // to this new one 117 | if(!branch->list) 118 | branch->list = entry; 119 | 120 | if(branch->last) 121 | branch->last->next = entry; 122 | 123 | branch->last = entry; 124 | entry->next = NULL; 125 | 126 | return entry; 127 | } 128 | 129 | // remove one entry on this branch 130 | // since it's a linked-list, we need to know which entry was the previous one 131 | // we use a single-direction linked-list 132 | // 133 | // removing an entry from the list don't free this entry, is just re-order 134 | // list to keep it coherent 135 | index_entry_t *index_branch_remove(index_branch_t *branch, index_entry_t *entry, index_entry_t *previous) { 136 | // removing the first entry 137 | if(branch->list == entry) 138 | branch->list = entry->next; 139 | 140 | // skipping this entry, linking next from previous 141 | // to our next one 142 | if(previous) 143 | previous->next = entry->next; 144 | 145 | // if our entry was the last one 146 | // the new last one is the previous one 147 | if(branch->last == entry) 148 | branch->last = previous; 149 | 150 | branch->length -= 1; 151 | 152 | return entry; 153 | } 154 | 155 | // iterate over a branch and try to find the previous entry of the given entry 156 | // if by mystake, the entry was not found on the branch, we returns the entry itself 157 | // if entry was the first entry, previous will also be NULL 158 | index_entry_t *index_branch_get_previous(index_branch_t *branch, index_entry_t *entry) { 159 | index_entry_t *previous = NULL; 160 | index_entry_t *iterator = branch->list; 161 | 162 | while(iterator && iterator != entry) { 163 | previous = iterator; 164 | iterator = iterator->next; 165 | } 166 | 167 | // we reached the end of the list, without finding 168 | // a matching entry, this is mostly a mistake from caller 169 | // let's notify it by replying with it's own object 170 | if(!iterator) 171 | return entry; 172 | 173 | return previous; 174 | } 175 | -------------------------------------------------------------------------------- /libzdb/index_branch.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_INDEX_BRANCH_H 2 | #define __ZDB_INDEX_BRANCH_H 3 | 4 | // buckets 5 | extern uint32_t buckets_branches; 6 | extern uint32_t buckets_mask; 7 | 8 | int index_set_buckets_bits(uint8_t bits); 9 | index_branch_t **index_buckets_init(); 10 | 11 | // initializers 12 | index_branch_t *index_branch_init(index_branch_t **branches, uint32_t branchid); 13 | void index_branch_free(index_branch_t **branches, uint32_t branchid); 14 | 15 | // accessors 16 | index_branch_t *index_branch_get(index_branch_t **branches, uint32_t branchid); 17 | index_branch_t *index_branch_get_allocate(index_branch_t **branches, uint32_t branchid); 18 | index_entry_t *index_branch_append(index_branch_t **branches, uint32_t branchid, index_entry_t *entry); 19 | index_entry_t *index_branch_remove(index_branch_t *branch, index_entry_t *entry, index_entry_t *previous); 20 | index_entry_t *index_branch_get_previous(index_branch_t *branch, index_entry_t *entry); 21 | #endif 22 | -------------------------------------------------------------------------------- /libzdb/index_get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "libzdb.h" 10 | #include "libzdb_private.h" 11 | 12 | static index_entry_t *index_get_handler_memkey(index_root_t *index, void *id, uint8_t idlength) { 13 | return index_entry_get(index, id, idlength); 14 | } 15 | 16 | static index_entry_t *index_get_handler_sequential(index_root_t *index, void *id, uint8_t idlength) { 17 | if(idlength != sizeof(seqid_t)) { 18 | zdb_debug("[-] index: sequential get: invalid key length (%u <> %ld)\n", idlength, sizeof(seqid_t)); 19 | return NULL; 20 | } 21 | 22 | // converting key into binary format 23 | seqid_t key; 24 | memcpy(&key, id, sizeof(seqid_t)); 25 | // key = be64toh(key); 26 | 27 | // resolving key into file id 28 | index_seqmap_t *seqmap = index_fileid_from_seq(index, key); 29 | 30 | // resolving relative offset 31 | uint32_t relative = key - seqmap->seqid; 32 | uint32_t offset = index_seq_offset(relative); 33 | 34 | // reading index on disk 35 | index_item_t *item; 36 | 37 | if(!(item = index_item_get_disk(index, seqmap->fileid, offset, sizeof(seqid_t)))) 38 | return NULL; 39 | 40 | memcpy(index_reusable_entry->id, item->id, item->idlength); 41 | index_reusable_entry->idlength = item->idlength; 42 | index_reusable_entry->offset = item->offset; 43 | index_reusable_entry->dataid = item->dataid; 44 | index_reusable_entry->indexid = seqmap->fileid; 45 | index_reusable_entry->flags = item->flags; 46 | index_reusable_entry->idxoffset = offset; 47 | index_reusable_entry->crc = item->crc; 48 | index_reusable_entry->parentid = item->parentid; 49 | index_reusable_entry->parentoff = item->parentoff; 50 | index_reusable_entry->timestamp = item->timestamp; 51 | index_reusable_entry->length = item->length; 52 | 53 | // index_entry_dump(index_reusable_entry); 54 | 55 | // cleaning intermediate object 56 | free(item); 57 | 58 | return index_reusable_entry; 59 | } 60 | 61 | static index_entry_t * (*index_get_handlers[])(index_root_t *root, void *id, uint8_t idlength) = { 62 | index_get_handler_memkey, // key-value mode 63 | index_get_handler_sequential, // incremental mode 64 | index_get_handler_sequential, // direct-key mode (not used anymore) 65 | index_get_handler_sequential, // fixed block mode (not implemented yet) 66 | }; 67 | 68 | index_entry_t *index_get(index_root_t *index, void *id, uint8_t idlength) { 69 | index_entry_t *entry; 70 | 71 | zdb_debug("[+] index: get: lookup key: "); 72 | zdb_debughex(id, idlength); 73 | 74 | if(!(entry = index_get_handlers[index->mode](index, id, idlength))) { 75 | zdb_debug("[-] index: get: key not found\n"); 76 | return NULL; 77 | } 78 | 79 | // key found but deleted 80 | if(entry->flags & INDEX_ENTRY_DELETED) { 81 | zdb_debug("[-] index: get: key requested deleted\n"); 82 | return NULL; 83 | } 84 | 85 | return entry; 86 | } 87 | 88 | // 89 | // low level helper 90 | // 91 | index_item_t *index_raw_fetch_entry(index_root_t *root) { 92 | uint8_t idlength; 93 | index_item_t *entry = NULL; 94 | 95 | if(read(root->indexfd, &idlength, sizeof(idlength)) != sizeof(idlength)) 96 | return NULL; 97 | 98 | // we have the length of the key 99 | ssize_t entrylength = sizeof(index_item_t) + idlength; 100 | if(!(entry = malloc(entrylength))) 101 | zdb_diep("index_raw_fetch_entry: malloc"); 102 | 103 | // rollback the 1 byte read for the id length 104 | lseek(root->indexfd, -1, SEEK_CUR); 105 | 106 | if(read(root->indexfd, entry, entrylength) != entrylength) 107 | zdb_diep("index header read failed"); 108 | 109 | return entry; 110 | } 111 | -------------------------------------------------------------------------------- /libzdb/index_get.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_INDEX_GET_H 2 | #define ZDB_INDEX_GET_H 3 | 4 | index_entry_t *index_get(index_root_t *index, void *id, uint8_t idlength); 5 | index_item_t *index_raw_fetch_entry(index_root_t *root); 6 | #endif 7 | -------------------------------------------------------------------------------- /libzdb/index_loader.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_INDEX_LOADER_H 2 | #define __ZDB_INDEX_LOADER_H 3 | 4 | // initialize index header file 5 | index_header_t index_initialize(int fd, fileid_t indexid, index_root_t *root); 6 | 7 | // initialize the whole index system 8 | index_root_t *index_init(zdb_settings_t *settings, char *indexdir, void *namespace, index_branch_t **branches); 9 | index_root_t *index_init_lazy(zdb_settings_t *settings, char *indexdir, void *namespace); 10 | 11 | // internal functions 12 | index_root_t *index_rehash(index_root_t *root); 13 | void index_internal_load(index_root_t *root); 14 | void index_internal_allocate_single(); 15 | 16 | // sanity check 17 | uint64_t index_availity_check(index_root_t *root); 18 | index_header_t *index_descriptor_load(index_root_t *root); 19 | index_header_t *index_descriptor_validate(index_header_t *header, index_root_t *root); 20 | int index_switch_mode(index_root_t *root); 21 | 22 | // gracefully clean everything 23 | void index_delete_files(char *indexdir); 24 | void index_destroy(index_root_t *root); 25 | void index_destroy_global(); 26 | #endif 27 | -------------------------------------------------------------------------------- /libzdb/index_scan.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_INDEX_SCAN_H 2 | #define __ZDB_INDEX_SCAN_H 3 | 4 | // scan internal representation 5 | // we use a status and a pointer to the header 6 | // in order to know what to do 7 | typedef enum index_scan_status_t { 8 | INDEX_SCAN_SUCCESS, // requested index entry found 9 | INDEX_SCAN_REQUEST_PREVIOUS, // offset requested found in the previous file 10 | INDEX_SCAN_EOF_REACHED, // end of file reached, last key of next file requested 11 | INDEX_SCAN_UNEXPECTED, // unexpected (memory, ...) error 12 | INDEX_SCAN_NO_MORE_DATA, // last item requested, nothing more 13 | INDEX_SCAN_DELETED, // entry was deleted, scan is updated to go further 14 | 15 | } index_scan_status_t; 16 | 17 | typedef struct index_scan_t { 18 | int fd; // file descriptor 19 | size_t original; // offset of the original key requested 20 | size_t target; // offset of the target key (read from the original) 21 | // target will be 0 on the first call 22 | // target will be updated if the offset is in another file 23 | index_item_t *header; // target header, set when found 24 | index_scan_status_t status; // status code 25 | fileid_t fileid; // index file id 26 | 27 | } index_scan_t; 28 | 29 | index_scan_t index_previous_header(index_root_t *root, fileid_t fileid, size_t offset); 30 | index_scan_t index_next_header(index_root_t *root, fileid_t fileid, size_t offset); 31 | index_scan_t index_first_header(index_root_t *root); 32 | index_scan_t index_last_header(index_root_t *root); 33 | #endif 34 | -------------------------------------------------------------------------------- /libzdb/index_seq.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "libzdb.h" 7 | #include "libzdb_private.h" 8 | 9 | // perform a binary search on the seqmap to get 10 | // the fileid back based on the index mapped with fileid 11 | static index_seqmap_t *index_seqmap_from_seq(index_seqid_t *seqid, seqid_t id) { 12 | seqid_t lower = 0; 13 | seqid_t higher = seqid->length; 14 | 15 | // binary search 16 | while(lower < higher) { 17 | seqid_t mid = (lower + higher) / 2; 18 | 19 | // exact match 20 | if(id == seqid->seqmap[mid].seqid) 21 | return &seqid->seqmap[mid]; 22 | 23 | // lower side 24 | if(id < seqid->seqmap[mid].seqid) { 25 | higher = mid; 26 | continue; 27 | } 28 | 29 | // are we on the right interval 30 | if(lower + 1 >= seqid->length) 31 | return &seqid->seqmap[lower]; 32 | 33 | if(seqid->seqmap[lower].seqid < id && seqid->seqmap[lower + 1].seqid > id) 34 | return &seqid->seqmap[lower]; 35 | 36 | // it's on the higher side 37 | lower = mid; 38 | } 39 | 40 | return &seqid->seqmap[lower]; 41 | } 42 | 43 | index_seqmap_t *index_fileid_from_seq(index_root_t *root, seqid_t seqid) { 44 | index_seqmap_t *seqmap = index_seqmap_from_seq(root->seqid, seqid); 45 | zdb_debug("[+] index: seqmap: resolving %" PRIu64 " -> file %u\n", seqid, seqmap->fileid); 46 | 47 | return seqmap; 48 | } 49 | 50 | void index_seqid_push(index_root_t *root, seqid_t id, fileid_t indexid) { 51 | zdb_debug("[+] index seq: mapping id %" PRIu64 " to file %u\n", id, indexid); 52 | 53 | if(root->seqid->length + 1 == root->seqid->allocated) { 54 | // growing up the vector 55 | root->seqid->allocated += 1024; 56 | zdb_debug("[+] index seq: growing up vector of files (%u entries)\n", root->seqid->allocated); 57 | 58 | if(!(root->seqid->seqmap = realloc(root->seqid->seqmap, sizeof(index_seqmap_t) * root->seqid->allocated))) 59 | zdb_diep("index seqmap: realloc"); 60 | } 61 | 62 | root->seqid->seqmap[root->seqid->length].fileid = indexid; 63 | root->seqid->seqmap[root->seqid->length].seqid = id; 64 | root->seqid->length += 1; 65 | } 66 | 67 | size_t index_seq_offset(seqid_t relative) { 68 | // skip index header 69 | size_t offset = sizeof(index_header_t); 70 | 71 | // index is linear like this 72 | // [header][obj-1][obj-2][obj-3][...] 73 | // 74 | // object X offset can be found by computing 75 | // size of each entry, in direct mode, keys are 76 | // always fixed-length 77 | // 78 | // each entry fixed-key-length 79 | offset += (relative * (sizeof(index_item_t) + sizeof(seqid_t))); 80 | 81 | return offset; 82 | } 83 | 84 | void index_seqid_dump(index_root_t *root) { 85 | for(fileid_t i = 0; i < root->seqid->length; i++) { 86 | index_seqmap_t *item = &root->seqid->seqmap[i]; 87 | zdb_log("[+] index seq: seqid %" PRIu64 " -> file %u\n", item->seqid, item->fileid); 88 | } 89 | } 90 | 91 | -------------------------------------------------------------------------------- /libzdb/index_seq.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_INDEX_SEQ_H 2 | #define __ZDB_INDEX_SEQ_H 3 | 4 | index_seqmap_t *index_fileid_from_seq(index_root_t *root, seqid_t seqid); 5 | void index_seqid_push(index_root_t *root, seqid_t id, fileid_t indexid); 6 | size_t index_seq_offset(seqid_t relative); 7 | 8 | void index_seqid_dump(index_root_t *root); 9 | #endif 10 | -------------------------------------------------------------------------------- /libzdb/index_set.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_INDEX_SET_H 2 | #define ZDB_INDEX_SET_H 3 | 4 | typedef struct index_set_t { 5 | index_entry_t *entry; 6 | void *id; 7 | 8 | } index_set_t; 9 | 10 | index_entry_t *index_set(index_root_t *root, index_set_t *new, index_entry_t *existing); 11 | index_entry_t *index_set_memory(index_root_t *root, void *id, index_entry_t *entry); 12 | 13 | index_item_t *index_item_from_set(index_root_t *root, index_set_t *set); 14 | 15 | // internal index append functions 16 | int index_append_entry_on_disk(index_root_t *root, index_set_t *set); 17 | #endif 18 | -------------------------------------------------------------------------------- /libzdb/libzdb.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "libzdb.h" 13 | #include "libzdb_private.h" 14 | 15 | // 16 | // global system settings 17 | // 18 | zdb_settings_t zdb_rootsettings = { 19 | .datapath = ZDB_DEFAULT_DATAPATH, 20 | .indexpath = ZDB_DEFAULT_INDEXPATH, 21 | .verbose = 0, 22 | .dump = 0, 23 | .sync = 0, 24 | .synctime = 0, 25 | .mode = ZDB_MODE_KEY_VALUE, 26 | .hook = NULL, 27 | .datasize = ZDB_DEFAULT_DATA_MAXSIZE, 28 | .maxsize = 0, 29 | .initialized = 0, 30 | }; 31 | 32 | 33 | // debug tools 34 | static char __hex[] = "0123456789abcdef"; 35 | 36 | void zdb_fulldump(void *_data, size_t len) { 37 | uint8_t *data = _data; 38 | unsigned int i, j; 39 | 40 | printf("[*] data fulldump [%p -> %p] (%lu bytes)\n", data, data + len, len); 41 | printf("[*] 0x0000: "); 42 | 43 | for(i = 0; i < len; ) { 44 | printf("%02x ", data[i++]); 45 | 46 | if(i % 16 == 0) { 47 | printf("|"); 48 | 49 | for(j = i - 16; j < i; j++) 50 | printf("%c", ((isprint(data[j]) ? data[j] : '.'))); 51 | 52 | printf("|\n[*] 0x%04x: ", i); 53 | } 54 | } 55 | 56 | if(i % 16) { 57 | printf("%-*s |", 5 * (16 - (i % 16)), " "); 58 | 59 | for(j = i - (i % 16); j < len; j++) 60 | printf("%c", ((isprint(data[j]) ? data[j] : '.'))); 61 | 62 | printf("%-*s|\n", 16 - ((int) len % 16), " "); 63 | } 64 | 65 | printf("\n"); 66 | } 67 | 68 | // human-readable size parser 69 | static char *human_readable_suffix = "kMGT"; 70 | 71 | size_t *zdb_human_readable_parse(char *input, size_t *target) { 72 | char *endp = input; 73 | char *match = NULL; 74 | size_t shift = 0; 75 | errno = 0; 76 | 77 | long double value = strtold(input, &endp); 78 | if(errno || endp == input || value < 0) 79 | return NULL; 80 | 81 | if(!(match = strchr(human_readable_suffix, *endp))) 82 | return NULL; 83 | 84 | if(*match) 85 | shift = (match - human_readable_suffix + 1) * 10; 86 | 87 | *target = value * (1LU << shift); 88 | 89 | return target; 90 | } 91 | 92 | // public wrapper 93 | void zdb_tools_fulldump(void *_data, size_t len) { 94 | return zdb_fulldump(_data, len); 95 | } 96 | 97 | void zdb_hexdump(void *input, size_t length) { 98 | unsigned char *buffer = (unsigned char *) input; 99 | char *output = calloc((length * 2) + 1, 1); 100 | char *writer = output; 101 | 102 | for(unsigned int i = 0, j = 0; i < length; i++, j += 2) { 103 | *writer++ = __hex[(buffer[i] & 0xF0) >> 4]; 104 | *writer++ = __hex[buffer[i] & 0x0F]; 105 | } 106 | 107 | printf("0x%s\n", output); 108 | free(output); 109 | } 110 | 111 | // public wrapper 112 | void zdb_tools_hexdump(void *input, size_t length) { 113 | return zdb_hexdump(input, length); 114 | } 115 | 116 | // 117 | // global warning and fatal message 118 | // 119 | void *zdb_warnp(char *str) { 120 | zdb_timelog(stderr); 121 | fprintf(stderr, COLOR_YELLOW "[-] %s: %s" COLOR_RESET "\n", str, strerror(errno)); 122 | return NULL; 123 | } 124 | 125 | void zdb_verbosep(char *prefix, char *str) { 126 | #ifdef RELEASE 127 | // only match on verbose flag if we are 128 | // in release mode, otherwise do always the 129 | // print, we are in debug mode anyway 130 | if(!zdb_rootsettings.verbose) 131 | return; 132 | #endif 133 | 134 | zdb_timelog(stderr); 135 | fprintf(stderr, "[-] %s: %s: %s\n", prefix, str, strerror(errno)); 136 | } 137 | 138 | void zdb_diep(char *str) { 139 | zdb_warnp(str); 140 | exit(EXIT_FAILURE); 141 | } 142 | 143 | void zdb_timelog(FILE *fp) { 144 | struct timeval n, *b; 145 | double value = 0; 146 | 147 | // boot time 148 | b = &zdb_rootsettings.stats.inittime; 149 | 150 | gettimeofday(&n, NULL); 151 | value = (double)(n.tv_usec - b->tv_usec) / 1000000 + (double)(n.tv_sec - b->tv_sec); 152 | 153 | fprintf(fp, "[% 15.6f]", value); 154 | } 155 | 156 | char *zdb_header_date(uint32_t epoch, char *target, size_t length) { 157 | struct tm *timeval; 158 | time_t unixtime; 159 | 160 | unixtime = epoch; 161 | 162 | timeval = localtime(&unixtime); 163 | strftime(target, length, "%F %T", timeval); 164 | 165 | return target; 166 | } 167 | 168 | -------------------------------------------------------------------------------- /libzdb/libzdb_private.h: -------------------------------------------------------------------------------- 1 | #ifndef __LIBZDB_PRIVATE_H 2 | #define __LIBZDB_PRIVATE_H 3 | 4 | void zdb_hexdump(void *buffer, size_t length); 5 | void zdb_fulldump(void *data, size_t len); 6 | 7 | #ifndef RELEASE 8 | #define zdb_verbose(...) { zdb_timelog(stdout); printf(__VA_ARGS__); } 9 | #define zdb_debug(...) { zdb_timelog(stdout); printf(__VA_ARGS__); } 10 | #define zdb_debughex(...) { zdb_hexdump(__VA_ARGS__); } 11 | #else 12 | #define zdb_verbose(...) { if(zdb_rootsettings.verbose) { zdb_timelog(stdout); printf(__VA_ARGS__); } } 13 | #define zdb_debug(...) ((void)0) 14 | #define zdb_debughex(...) ((void)0) 15 | #endif 16 | 17 | extern zdb_settings_t zdb_rootsettings; 18 | 19 | void zdb_diep(char *str); 20 | void *zdb_warnp(char *str); 21 | void zdb_verbosep(char *prefix, char *str); 22 | #endif 23 | -------------------------------------------------------------------------------- /libzdb/namespace.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_NAMESPACE_H 2 | #define __ZDB_NAMESPACE_H 3 | 4 | // default namespace name for new clients 5 | // this namespace will be used for any unauthentificated clients 6 | // or any action without namespace specified 7 | #define NAMESPACE_DEFAULT "default" 8 | 9 | // current expected version of header 10 | #define NAMESPACE_CURRENT_VERSION 1 11 | 12 | #define NAMESPACE_MAX_LENGTH 128 13 | 14 | typedef enum ns_flags_t { 15 | NS_FLAGS_PUBLIC = 1, // public read-only namespace 16 | NS_FLAGS_WORM = 2, // worm mode enabled or not 17 | NS_FLAGS_EXTENDED = 4, // extended header is present (legacy, mandatory) 18 | 19 | } ns_flags_t; 20 | 21 | typedef enum ns_lock_t { 22 | NS_LOCK_UNLOCKED = 0, 23 | NS_LOCK_READ_ONLY = 1, 24 | NS_LOCK_READ_WRITE = 2, 25 | 26 | } ns_lock_t; 27 | 28 | // ns_header_t contains the header of a specific namespace 29 | // this header will be the only content of the namespace descriptor file 30 | typedef struct ns_header_t { 31 | uint32_t version; // keep track of this version 32 | uint8_t namelength; // length of the namespace name 33 | uint8_t passlength; // length of the password 34 | uint64_t maxsize; // maximum namespace size (if defined) 35 | uint8_t flags; // some flags (see define above) 36 | 37 | } __attribute__((packed)) ns_header_t; 38 | 39 | typedef struct namespace_t { 40 | char *name; // namespace string-name 41 | char *password; // optional password 42 | char *indexpath; // index root directory 43 | char *datapath; // data root directory 44 | index_root_t *index; // index structure pointer 45 | data_root_t *data; // data structure pointer 46 | char public; // publicly readable (read without password) 47 | size_t maxsize; // maximum size allowed 48 | size_t idlist; // nsroot list index 49 | size_t version; // internal version used 50 | ns_lock_t locked; // set namespace read/write temporary status 51 | char worm; // worm mode (write only read multiple) 52 | // this mode disable overwrite/deletion 53 | 54 | } namespace_t; 55 | 56 | typedef struct ns_root_t { 57 | size_t length; // amount of namespaces allocated 58 | size_t effective; // amount of namespaces currently loaded 59 | namespace_t **namespaces; // pointers to namespaces 60 | zdb_settings_t *settings; // global settings reminder 61 | index_branch_t **branches; // unique global branches list 62 | 63 | // as explained in namespace.c, we keep a single big 64 | // index which that contains everything (all namespaces together) 65 | // 66 | // for each index structure, we will point the branches to the 67 | // same big index branches all the time, this is why we keep 68 | // this one here, as the 'original one' 69 | 70 | } ns_root_t; 71 | 72 | size_t namespace_length(); 73 | int namespace_valid_name(char *name); 74 | void namespace_descriptor_update(namespace_t *namespace, int fd); 75 | 76 | namespace_t *namespace_iter(); 77 | namespace_t *namespace_iter_next(namespace_t *namespace); 78 | 79 | int namespace_create(char *name); 80 | int namespace_delete(namespace_t *namespace); 81 | int namespace_commit(namespace_t *namespace); 82 | int namespace_flush(namespace_t *namespace); 83 | int namespace_reload(namespace_t *namespace); 84 | int namespace_is_fresh(namespace_t *namespace); 85 | int namespace_lock(namespace_t *namespace); 86 | int namespace_unlock(namespace_t *namespace); 87 | int namespace_is_locked(namespace_t *namespace); 88 | int namespace_freeze(namespace_t *namespace); 89 | int namespace_unfreeze(namespace_t *namespace); 90 | int namespace_is_frozen(namespace_t *namespace); 91 | void namespace_free(namespace_t *namespace); 92 | namespace_t *namespace_get(char *name); 93 | 94 | int namespaces_init(zdb_settings_t *settings); 95 | ns_root_t *namespaces_allocate(zdb_settings_t *settings); 96 | int namespaces_destroy(); 97 | int namespaces_emergency(); 98 | 99 | namespace_t *namespace_load(ns_root_t *nsroot, char *name); 100 | namespace_t *namespace_load_light(ns_root_t *nsroot, char *name, int ensure); 101 | 102 | namespace_t *namespace_get_default(); 103 | #endif 104 | -------------------------------------------------------------------------------- /libzdb/security.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "libzdb.h" 9 | #include "libzdb_private.h" 10 | 11 | // zdb_challenge generate a random string 12 | // which can be used for cryptographic random 13 | // this can be used to salt stuff and generate nonce 14 | char *zdb_challenge() { 15 | char buffer[8]; 16 | char *string; 17 | 18 | if(getentropy(buffer, sizeof(buffer)) < 0) { 19 | zdb_warnp("getentropy"); 20 | return NULL; 21 | } 22 | 23 | if(!(string = malloc((sizeof(buffer) * 2) + 1))) { 24 | zdb_warnp("challenge: malloc"); 25 | return NULL; 26 | } 27 | 28 | for(unsigned int i = 0; i < sizeof(buffer); i++) 29 | sprintf(string + (i * 2), "%02x", buffer[i] & 0xff); 30 | 31 | zdb_debug("[+] security: challenge generated: %s\n", string); 32 | 33 | return string; 34 | } 35 | 36 | // generate an allocated string, which contains hexahash of 37 | // sha1 concatenated with password with colon 38 | // sha1(salt:password) 39 | // string needs to be free'd after use 40 | char *zdb_hash_password(char *salt, char *password) { 41 | char *hashmatch; 42 | 43 | if(asprintf(&hashmatch, "%s:%s", salt, password) < 0) { 44 | zdb_warnp("asprintf"); 45 | return NULL; 46 | } 47 | 48 | char buffer[ZDB_SHA1_DIGEST_LENGTH + 1]; 49 | char bufferstr[ZDB_SHA1_DIGEST_STR_LENGTH + 1]; 50 | 51 | memset(buffer, 0, sizeof(buffer)); 52 | memset(bufferstr, 0, sizeof(bufferstr)); 53 | 54 | // compute sha1 and build hex-string 55 | zdb_sha1(buffer, hashmatch, strlen(hashmatch)); 56 | 57 | for(int i = 0; i < ZDB_SHA1_DIGEST_LENGTH; i++) 58 | sprintf(bufferstr + (i * 2), "%02x", buffer[i] & 0xff); 59 | 60 | free(hashmatch); 61 | 62 | return strdup(bufferstr); 63 | } 64 | -------------------------------------------------------------------------------- /libzdb/security.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_SECURITY_H 2 | #define __ZDB_SECURITY_H 3 | 4 | char *zdb_challenge(); 5 | char *zdb_hash_password(char *salt, char *password); 6 | #endif 7 | -------------------------------------------------------------------------------- /libzdb/settings.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "libzdb.h" 8 | #include "libzdb_private.h" 9 | 10 | static char *zdb_modes[] = { 11 | "default key-value", 12 | "sequential keys", 13 | "direct key position", 14 | "direct key fixed block length", 15 | "mixed mode", 16 | }; 17 | 18 | // 19 | // public settings accessor 20 | // 21 | zdb_settings_t *zdb_settings_get() { 22 | return &zdb_rootsettings; 23 | } 24 | 25 | // 26 | // tools 27 | // 28 | 29 | // returns running mode in readable string 30 | char *zdb_running_mode(index_mode_t mode) { 31 | if(mode > (sizeof(zdb_modes) / sizeof(char *)) - 1) 32 | return "unsupported mode"; 33 | 34 | return zdb_modes[mode]; 35 | } 36 | 37 | // returns zdb string id 38 | char *zdb_id() { 39 | if(!zdb_rootsettings.zdbid) 40 | return "unknown-id"; 41 | 42 | return zdb_rootsettings.zdbid; 43 | } 44 | 45 | // set zdb id from string // FIXME 46 | char *zdb_id_set(char *id) { 47 | zdb_rootsettings.zdbid = strdup(id); 48 | return zdb_rootsettings.zdbid; 49 | } 50 | 51 | // returns the instance id (generated during init) 52 | uint32_t zdb_instanceid_get() { 53 | return zdb_rootsettings.iid; 54 | } 55 | 56 | // returns plain text version 57 | char *zdb_version() { 58 | return ZDB_VERSION; 59 | } 60 | 61 | // returns plain text revision (git revision) 62 | char *zdb_revision() { 63 | return ZDB_REVISION; 64 | } 65 | -------------------------------------------------------------------------------- /libzdb/settings.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_SETTINGS_H 2 | #define __ZDB_SETTINGS_H 3 | 4 | zdb_settings_t *zdb_settings_get(); 5 | 6 | char *zdb_version(); 7 | char *zdb_revision(); 8 | 9 | char *zdb_running_mode(index_mode_t mode); 10 | 11 | char *zdb_id(); 12 | char *zdb_id_set(char *id); 13 | 14 | uint32_t zdb_instanceid_get(); 15 | #endif 16 | -------------------------------------------------------------------------------- /libzdb/sha1.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_SHA1_H 2 | #define __ZDB_SHA1_H 3 | 4 | #define ZDB_SHA1_DIGEST_STR_LENGTH 40 5 | #define ZDB_SHA1_DIGEST_LENGTH 20 6 | 7 | void zdb_sha1(char *hash, const char *str, unsigned int len); 8 | #endif 9 | -------------------------------------------------------------------------------- /libzdb/sockets.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_SOCKETS_H 2 | #define ZDB_SOCKETS_H 3 | 4 | // is this still needed ? 5 | #endif 6 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = zdbtests 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -pg -coverage -g -std=gnu99 -O0 -W -Wall -Wextra -msse4.2 -Wno-implicit-fallthrough -I../libzdb 6 | LDFLAGS += -coverage -lhiredis -lpthread ../libzdb/libzdb.a 7 | 8 | CFLAGS += $(shell pkg-config --cflags hiredis) 9 | 10 | all: $(EXEC) 11 | 12 | $(EXEC): $(OBJ) 13 | $(CC) -o $@ $^ $(LDFLAGS) 14 | 15 | %.o: %.c 16 | $(CC) $(CFLAGS) -c $< 17 | 18 | clean: 19 | $(RM) *.o 20 | 21 | mrproper: clean 22 | $(RM) $(EXEC) 23 | -------------------------------------------------------------------------------- /tests/pipeline/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = pipeline 2 | SRC=$(wildcard *.c) 3 | OBJ=$(SRC:.c=.o) 4 | 5 | CFLAGS=-g -std=gnu99 -W -Wall 6 | LDFLAGS=-lhiredis 7 | 8 | all: $(EXEC) 9 | 10 | release: CFLAGS+=-DRELEASE 11 | release: $(EXEC) 12 | 13 | $(EXEC): $(OBJ) 14 | $(CC) -o $@ $^ $(LDFLAGS) 15 | 16 | %.o: %.c 17 | $(CC) $(CFLAGS) -c $< 18 | 19 | clean: 20 | $(RM) *.o 21 | 22 | mrproper: clean 23 | $(RM) $(EXEC) 24 | -------------------------------------------------------------------------------- /tests/pipeline/pipeline.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | void fatal(redisContext *context) { 7 | fprintf(stderr, "[-] %s\n", context->errstr); 8 | exit(EXIT_FAILURE); 9 | } 10 | 11 | int pipeline(redisContext *context) { 12 | redisReply *reply; 13 | size_t sets = 125; 14 | 15 | char *commands[] = {"PING", "INFO", "PING", "NSLIST", "DBSIZE", "TIME"}; 16 | 17 | for(size_t i = 0; i < sizeof(commands) / sizeof(char *); i++) { 18 | printf("[+] executing: %s\n", commands[i]); 19 | redisAppendCommand(context, commands[i]); 20 | } 21 | 22 | for(size_t i = 0; i < sizeof(commands) / sizeof(char *); i++) { 23 | if(redisGetReply(context, (void **) &reply) != REDIS_OK) 24 | fatal(context); 25 | 26 | printf("[+] response okay\n"); 27 | freeReplyObject(reply); 28 | } 29 | 30 | for(size_t i = 0; i < sets; i++) { 31 | redisAppendCommand(context, "DEL pipeline-%d", i); 32 | redisAppendCommand(context, "SET pipeline-%d THISISMYDATAHELLOYEAHWORKS", i); 33 | } 34 | 35 | for(size_t i = 0; i < sets * 2; i++) { 36 | if(redisGetReply(context, (void **) &reply) != REDIS_OK) 37 | fatal(context); 38 | 39 | printf("[+] response okay: %s\n", reply->str); 40 | freeReplyObject(reply); 41 | } 42 | 43 | return 0; 44 | } 45 | 46 | redisContext *initialize(char *hostname, int port) { 47 | struct timeval timeout = {5, 0}; 48 | redisContext *context; 49 | 50 | printf("[+] connecting: %s, port: %d\n", hostname, port); 51 | 52 | if(!(context = redisConnectWithTimeout(hostname, port, timeout))) 53 | return NULL; 54 | 55 | if(context->err) { 56 | fprintf(stderr, "[-] redis error: %s\n", context->errstr); 57 | return NULL; 58 | } 59 | 60 | return context; 61 | } 62 | 63 | int main() { 64 | char *inhost = "localhost"; 65 | int inport = 9900; 66 | redisContext *context; 67 | 68 | printf("[+] initializing hosts\n"); 69 | 70 | if(!(context = initialize(inhost, inport))) 71 | exit(EXIT_FAILURE); 72 | 73 | pipeline(context); 74 | printf("[+] pipeline done\n"); 75 | 76 | redisFree(context); 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | if [ ! -d zdbd ]; then 5 | echo "Please run this script from root project directory" 6 | exit 1 7 | fi 8 | 9 | rm -rf /tmp/zdbtest 10 | 11 | # test arguments 12 | ./zdbd/zdb --help || true 13 | ./zdbd/zdb --blabla || true 14 | ./zdbd/zdb --datasize $((8 * 1024 * 1024 * 1024)) || true 15 | ./zdbd/zdb --datasize 32 || true 16 | 17 | # test load with slashes 18 | ./zdbd/zdb --verbose --dump --data /tmp/zdbtest-data --index /tmp/zdbtest-index 19 | ./zdbd/zdb --verbose --dump --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ 20 | 21 | # first real test suite 22 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --hook /bin/true --datasize $((128 * 1024 * 1024)) 23 | ./tests/zdbtests 24 | sleep 1 25 | 26 | # reopen existing data 27 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --dump 28 | 29 | # simulate a segmentation fault 30 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --hook /bin/true 31 | pkill -SEGV zdb 32 | sleep 1 33 | 34 | # simulate a SIGINT (ctrl+c) 35 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --hook /bin/true 36 | pkill -INT zdb 37 | sleep 1 38 | 39 | # cleaning stuff 40 | rm -rf /tmp/zdbtest 41 | 42 | # starting test suite with small datasize, generating lot of file jump 43 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --hook /bin/true --datasize $((518 * 1024)) 44 | ./tests/zdbtests 45 | sleep 1 46 | 47 | # cleaning stuff again 48 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 49 | 50 | # starting with authentification 51 | ./zdbd/zdb --background --verbose --socket /tmp/zdb.sock --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ \ 52 | --admin protect \ 53 | --synctime 10 \ 54 | --mode user 55 | 56 | ./tests/zdbtests 57 | sleep 1 58 | 59 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 60 | 61 | # launch tcp testsuite 62 | ./zdbd/zdb --background --verbose --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --admin root \ 63 | --logfile /tmp/zdb.logs \ 64 | --listen 127.0.0.1 --port 9900 \ 65 | --sync 66 | 67 | ./tests/zdbtests 68 | 69 | # open in sequential (will fails because created in another mode) 70 | ./zdbd/zdb --verbose --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --dump --mode seq || true 71 | 72 | # truncate index 73 | echo nopenopenope > /tmp/zdbtest-index/default/zdb-index-00000 74 | ./zdbd/zdb --verbose --data /tmp/zdbtest-data/ --index /tmp/zdbtest-index/ --dump || true 75 | 76 | # clean 77 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 78 | 79 | # test protected mode 80 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --protect || true 81 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --protect --admin helloworld 82 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --maxsize 131072 83 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 84 | 85 | # create empty dataset in direct mode 86 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --mode direct 87 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 88 | 89 | # create empty dataset in sequential mode 90 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --mode seq 91 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 92 | 93 | # trying non existing mode 94 | ./zdbd/zdb --data /tmp/zdbtest-data --index /tmp/zdbtest-index --dump --mode nonexist || true 95 | rm -rf /tmp/zdbtest-data /tmp/zdbtest-index 96 | 97 | # run tests in sequential mode 98 | ./zdbd/zdb --background --socket /tmp/zdb.sock --data /tmp/zdbtest-data --index /tmp/zdbtest-index --mode seq 99 | ./tests/zdbtests 100 | sleep 1 101 | 102 | # reload sequential database 103 | ./zdbd/zdb --socket /tmp/zdb.sock --data /tmp/zdbtest-data --index /tmp/zdbtest-index --mode seq --dump 104 | 105 | echo "All tests done." 106 | -------------------------------------------------------------------------------- /tests/tests.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "tests_user.h" 6 | #include "tests.h" 7 | 8 | static char *project = "0-db"; 9 | 10 | static registered_tests_t tests = { 11 | .length = 0, 12 | .longest = 0, 13 | .list = {}, 14 | 15 | .success = 0, 16 | .failed = 0, 17 | .failed_fatal = 0, 18 | .warning = 0, 19 | .skipped = 0, 20 | }; 21 | 22 | // register a function as runtest 23 | // insert the function in the run list 24 | void tests_register(char *name, int (*func)(test_t *)) { 25 | // printf("[+] registering: %s (%p)\n", name, func); 26 | 27 | tests.list[tests.length].name = name; 28 | tests.list[tests.length].test = func; 29 | 30 | if(strlen(name) > tests.longest) 31 | tests.longest = strlen(name); 32 | 33 | tests.length += 1; 34 | } 35 | 36 | void testsuite(test_t *maintest) { 37 | for(unsigned int i = 0; i < tests.length; i++) { 38 | runtest_t *test = &tests.list[i]; 39 | 40 | printf("[+] >> " CYAN("%s") ": running\n", test->name); 41 | test->result = test->test(maintest); 42 | 43 | switch(test->result) { 44 | case TEST_SUCCESS: 45 | printf("[+] >> " CYAN("%s") ": " GREEN("success") "\n", test->name); 46 | tests.success += 1; 47 | break; 48 | 49 | case TEST_FAILED_FATAL: 50 | printf("[-] >> " CYAN("%s") ": " RED("failed (fatal)") "\n", test->name); 51 | tests.failed += 1; 52 | tests.failed_fatal += 1; 53 | break; 54 | 55 | case TEST_FAILED: 56 | printf("[-] >> " CYAN("%s") ": " RED("failed") "\n", test->name); 57 | tests.failed += 1; 58 | break; 59 | 60 | case TEST_WARNING: 61 | printf("[-] >> " CYAN("%s") ": " YELLOW("warning") "\n", test->name); 62 | tests.warning += 1; 63 | break; 64 | 65 | case TEST_SKIPPED: 66 | printf("[-] >> " CYAN("%s") ": " GREY("skipped") "\n", test->name); 67 | tests.skipped += 1; 68 | break; 69 | } 70 | } 71 | 72 | } 73 | 74 | 75 | int initialize_tcp(test_t *settings) { 76 | settings->host = "localhost"; 77 | settings->port = 9900; 78 | 79 | settings->zdb = redisConnect(settings->host, settings->port); 80 | settings->type = CONNECTION_TYPE_TCP; 81 | 82 | if(!settings->zdb || settings->zdb->err) { 83 | const char *error = (settings->zdb->err) ? settings->zdb->errstr : "memory error"; 84 | log("%s:%d: %s\n", settings->host, settings->port, error); 85 | return 1; 86 | } 87 | 88 | return 0; 89 | } 90 | 91 | void initialize(test_t *settings) { 92 | char *socket = "/tmp/zdb.sock"; 93 | 94 | settings->zdb = redisConnectUnix(socket); 95 | settings->type = CONNECTION_TYPE_UNIX; 96 | 97 | if(!settings->zdb || settings->zdb->err) { 98 | const char *error = (settings->zdb->err) ? settings->zdb->errstr : "memory error"; 99 | log("%s: %s\n", socket, error); 100 | 101 | if(initialize_tcp(settings)) 102 | exit(EXIT_FAILURE); 103 | } 104 | 105 | signal(SIGPIPE, SIG_IGN); 106 | } 107 | 108 | int main(int argc, char *argv[]) { 109 | (void) argc; 110 | (void) argv; 111 | static test_t settings; 112 | 113 | printf("[+] initializing " CYAN("%s") " tests suite\n", project); 114 | printf("[+] tests registered: %u\n", tests.length); 115 | printf("[+] \n"); 116 | 117 | for(unsigned int i = 0; i < tests.length; i++) { 118 | runtest_t *test = &tests.list[i]; 119 | printf("[+] % 4d) %-*s [%p]\n", i + 1, tests.longest + 2, test->name, test->test); 120 | } 121 | 122 | printf("[+] \n"); 123 | printf("[+] preparing tests\n"); 124 | initialize(&settings); 125 | 126 | printf("[+] running tests\n"); 127 | printf("[+]\n"); 128 | testsuite(&settings); 129 | 130 | printf("[+]\n"); 131 | printf("[+] all tests done, summary:\n"); 132 | printf("[+]\n"); 133 | printf("[+] " GREEN("success") ": %u\n", tests.success); 134 | printf("[+] " RED("failed") " : %u (%u fatal)\n", tests.failed, tests.failed_fatal); 135 | printf("[+] " YELLOW("warning") ": %u\n", tests.warning); 136 | printf("[+] " GREY("skipped") ": %u\n", tests.skipped); 137 | printf("[+]\n"); 138 | 139 | return 0; 140 | } 141 | -------------------------------------------------------------------------------- /tests/tests.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_TESTS_H 2 | #define ZDB_TESTS_H 3 | 4 | typedef struct runtest_t { 5 | char *name; // function name 6 | int (*test)(test_t *); // function pointer 7 | int result; // return code 8 | 9 | } runtest_t; 10 | 11 | typedef struct registered_tests_t { 12 | unsigned int length; 13 | unsigned int longest; 14 | runtest_t list[1024]; 15 | 16 | unsigned int success; 17 | unsigned int failed; 18 | unsigned int failed_fatal; 19 | unsigned int warning; 20 | unsigned int skipped; 21 | 22 | } registered_tests_t; 23 | 24 | void tests_register(char *name, int (*func)(test_t *)); 25 | 26 | // tests return code 27 | #define TEST_SUCCESS (0) 28 | #define TEST_FAILED_FATAL (-1) 29 | #define TEST_FAILED (-2) 30 | #define TEST_WARNING (-3) 31 | #define TEST_SKIPPED (-4) 32 | 33 | // test logger 34 | #define log(...) { printf("[ ] " __VA_ARGS__); } 35 | 36 | #define __executor(name) name 37 | #define __constructor(name) __construct_##name 38 | 39 | // macro definition: 40 | // 41 | // int name(test_t *test); -> declare prototype 42 | // __attribute__ ((constructor)) void __constructor(name)() { -> create a constructor 43 | // tests_register(#name, name); -> call the global register 44 | // } -> end of the register 45 | // int __executor(name)(test_t *test) -> declare the real function 46 | 47 | #define runtest_prio(prio, name) \ 48 | int name(test_t *test); \ 49 | __attribute__ ((constructor (prio))) void __constructor(name)() { \ 50 | tests_register(#name, name); \ 51 | } \ 52 | int __executor(name)(test_t *test) 53 | 54 | #define runtest(name) runtest_prio(1000, name) 55 | 56 | // 57 | // here is a more readable expanded version 58 | // 59 | // ----------------------------------- 60 | // runtest(test1) { 61 | // printf(">> I'm test 1\n"); 62 | // return 0; 63 | // } 64 | // ----------------------------------- 65 | // 66 | // -> is translated to: 67 | // 68 | // ----------------------------------- 69 | // int test1(test_t *test); 70 | // 71 | // __attribute__ ((constructor)) void __constructor_test1() { 72 | // tests_register("test1", test1); 73 | // } 74 | // 75 | // int test1(test_t *test) { 76 | // printf(">> I'm test 1\n"); 77 | // return 0; 78 | // } 79 | // ----------------------------------- 80 | // 81 | 82 | #define COLOR_RED "\033[31;1m" 83 | #define COLOR_YELLOW "\033[33;1m" 84 | #define COLOR_GREEN "\033[32;1m" 85 | #define COLOR_CYAN "\033[36;1m" 86 | #define COLOR_GREY "\033[30;1m" 87 | #define COLOR_RESET "\033[0m" 88 | 89 | #define RED(x) COLOR_RED x COLOR_RESET 90 | #define YELLOW(x) COLOR_YELLOW x COLOR_RESET 91 | #define GREEN(x) COLOR_GREEN x COLOR_RESET 92 | #define CYAN(x) COLOR_CYAN x COLOR_RESET 93 | #define GREY(x) COLOR_GREY x COLOR_RESET 94 | 95 | void initialize(test_t *settings); 96 | int initialize_tcp(test_t *settings); 97 | #endif 98 | -------------------------------------------------------------------------------- /tests/tests_user.h: -------------------------------------------------------------------------------- 1 | #ifndef USERTESTS_H 2 | #define USERTESTS_H 3 | 4 | #include 5 | 6 | #define CONNECTION_TYPE_UNIX 0 7 | #define CONNECTION_TYPE_TCP 1 8 | 9 | typedef enum rmode_t { 10 | USERKEY, 11 | SEQUENTIAL, 12 | 13 | } rmode_t; 14 | 15 | typedef struct test_t { 16 | redisContext *zdb; 17 | int type; 18 | char *host; 19 | int port; 20 | rmode_t mode; 21 | 22 | } test_t; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /tests/zdb_history.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "tests_user.h" 6 | #include "zdb_utils.h" 7 | #include "tests.h" 8 | 9 | // sequential priority 10 | #define sp 175 11 | 12 | static char *namespace_history = "test_history"; 13 | 14 | static int history_check(test_t *test, int argc, const char *argv[], char *expected) { 15 | redisReply *reply; 16 | 17 | if(!(reply = zdb_response_history(test, argc, argv))) 18 | return zdb_result(reply, TEST_FAILED_FATAL); 19 | 20 | if(strcmp(reply->element[2]->str, expected) == 0) 21 | return zdb_result(reply, TEST_SUCCESS); 22 | 23 | log("%s\n", reply->str); 24 | 25 | return zdb_result(reply, TEST_FAILED); 26 | } 27 | 28 | // create a new namespace 29 | runtest_prio(sp, history_init) { 30 | return zdb_nsnew(test, namespace_history); 31 | } 32 | 33 | // select this new namespace 34 | runtest_prio(sp, history_select) { 35 | const char *argv[] = {"SELECT", namespace_history}; 36 | return zdb_command(test, argvsz(argv), argv); 37 | } 38 | 39 | 40 | runtest_prio(sp, history_init_chain1) { 41 | return zdb_set(test, "changeme", "value 1"); 42 | } 43 | 44 | runtest_prio(sp, history_init_chain2) { 45 | return zdb_set(test, "changeme", "value -- 2"); 46 | } 47 | 48 | runtest_prio(sp, history_init_chain3) { 49 | return zdb_set(test, "changeme", "val 3"); 50 | } 51 | 52 | runtest_prio(sp, history_init_chain4) { 53 | return zdb_set(test, "changeme", "new value 4"); 54 | } 55 | 56 | runtest_prio(sp, history_init_chain5) { 57 | return zdb_set(test, "changeme", "history value 5"); 58 | } 59 | 60 | runtest_prio(sp, history_init_chain6) { 61 | return zdb_set(test, "changeme", "history value 6"); 62 | } 63 | 64 | runtest_prio(sp, history_first_hit) { 65 | if(test->mode == SEQUENTIAL) 66 | return TEST_SKIPPED; 67 | 68 | const char *argv[] = {"HISTORY", "changeme"}; 69 | return history_check(test, argvsz(argv), argv, "history value 6"); 70 | } 71 | 72 | 73 | /* 74 | // start scan test 75 | runtest_prio(sp, scan_get_first_key) { 76 | const char *argv[] = {"SCAN"}; 77 | return scan_check(test, argvsz(argv), argv, "key1"); 78 | } 79 | 80 | runtest_prio(sp, scan_get_last_key) { 81 | const char *argv[] = {"RSCAN"}; 82 | return scan_check(test, argvsz(argv), argv, "key6"); 83 | } 84 | 85 | runtest_prio(sp, scan_get_second_key) { 86 | const char *argv[] = {"SCAN", "key1"}; 87 | return scan_check(test, argvsz(argv), argv, "key2"); 88 | } 89 | 90 | runtest_prio(sp, scan_get_last_minusone_key) { 91 | const char *argv[] = {"RSCAN", "key6"}; 92 | return scan_check(test, argvsz(argv), argv, "key5"); 93 | } 94 | 95 | 96 | runtest_prio(sp, scan_remove_first) { 97 | const char *argv[] = {"DEL", "key1"}; 98 | return zdb_command(test, argvsz(argv), argv); 99 | } 100 | 101 | runtest_prio(sp, scan_get_new_first_key) { 102 | const char *argv[] = {"SCAN"}; 103 | return scan_check(test, argvsz(argv), argv, "key2"); 104 | } 105 | 106 | 107 | runtest_prio(sp, scan_remove_last) { 108 | const char *argv[] = {"DEL", "key6"}; 109 | return zdb_command(test, argvsz(argv), argv); 110 | } 111 | 112 | runtest_prio(sp, scan_get_new_last_key) { 113 | const char *argv[] = {"RSCAN"}; 114 | return scan_check(test, argvsz(argv), argv, "key5"); 115 | } 116 | 117 | // scan on unknown key 118 | runtest_prio(sp, scan_non_existing) { 119 | const char *argv[] = {"SCAN", "nonexisting"}; 120 | return zdb_command_error(test, argvsz(argv), argv); 121 | } 122 | 123 | runtest_prio(sp, rscan_non_existing) { 124 | const char *argv[] = {"RSCAN", "nonexisting"}; 125 | return zdb_command_error(test, argvsz(argv), argv); 126 | } 127 | 128 | // scan on deleted key 129 | runtest_prio(sp, scan_deleted_key) { 130 | const char *argv[] = {"SCAN", "key1"}; 131 | return zdb_command_error(test, argvsz(argv), argv); 132 | } 133 | 134 | runtest_prio(sp, rscan_deleted_key) { 135 | const char *argv[] = {"RSCAN", "key1"}; 136 | return zdb_command_error(test, argvsz(argv), argv); 137 | } 138 | 139 | // scan from last key 140 | runtest_prio(sp, scan_ask_after_last) { 141 | const char *argv[] = {"SCAN", "key5"}; 142 | return zdb_command_error(test, argvsz(argv), argv); 143 | } 144 | 145 | // rscan from first key 146 | runtest_prio(sp, scan_ask_before_first) { 147 | const char *argv[] = {"RSCAN", "key2"}; 148 | return zdb_command_error(test, argvsz(argv), argv); 149 | } 150 | */ 151 | -------------------------------------------------------------------------------- /tests/zdb_lowlevel.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "tests_user.h" 7 | #include "tests.h" 8 | #include "zdb_utils.h" 9 | 10 | #define sp 700 11 | 12 | int lowlevel_send_invalid(test_t *test, char *buffer, size_t buflen) { 13 | if(write(test->zdb->fd, buffer, strlen(buffer)) != (ssize_t) strlen(buffer)) { 14 | perror("write"); 15 | return TEST_FAILED_FATAL; 16 | } 17 | 18 | if(read(test->zdb->fd, buffer, buflen) < 0) { 19 | perror("read"); 20 | return TEST_FAILED_FATAL; 21 | } 22 | 23 | if(buffer[0] == '-') { 24 | // connection was closed, reopen it 25 | // redisReconnect(test->zdb); 26 | initialize_tcp(test); 27 | return TEST_SUCCESS; 28 | } 29 | 30 | return TEST_FAILED; 31 | } 32 | 33 | runtest_prio(sp, lowlevel_slow_ping) { 34 | char buffer[512]; 35 | 36 | if(test->type != CONNECTION_TYPE_TCP) 37 | return TEST_SKIPPED; 38 | 39 | strcpy(buffer, "*1\r\n$4\r\nPING\r\n"); 40 | 41 | for(unsigned int i = 0; i < strlen(buffer); i++) { 42 | if(write(test->zdb->fd, buffer + i, 1) != 1) 43 | perror("write"); 44 | 45 | usleep(10000); 46 | } 47 | 48 | if(recv(test->zdb->fd, buffer, sizeof(buffer), MSG_NOSIGNAL) < 0) 49 | perror("read"); 50 | 51 | if(buffer[0] == '+') 52 | return TEST_SUCCESS; 53 | 54 | return TEST_FAILED; 55 | } 56 | 57 | runtest_prio(sp, lowlevel_not_an_array) { 58 | char buffer[512]; 59 | 60 | if(test->type != CONNECTION_TYPE_TCP) 61 | return TEST_SKIPPED; 62 | 63 | // invalid request, not an array 64 | strcpy(buffer, "+1\r\n$4\r\nPING\r\n"); 65 | 66 | return lowlevel_send_invalid(test, buffer, sizeof(buffer)); 67 | } 68 | 69 | runtest_prio(sp, lowlevel_no_argc) { 70 | char buffer[512]; 71 | 72 | if(test->type != CONNECTION_TYPE_TCP) 73 | return TEST_SKIPPED; 74 | 75 | // empty array 76 | strcpy(buffer, "*0\r\n"); 77 | 78 | return lowlevel_send_invalid(test, buffer, sizeof(buffer)); 79 | } 80 | 81 | runtest_prio(sp, lowlevel_too_many_arguments) { 82 | char buffer[512]; 83 | 84 | if(test->type != CONNECTION_TYPE_TCP) 85 | return TEST_SKIPPED; 86 | 87 | // too many argument 88 | strcpy(buffer, "*1025\r\n"); 89 | 90 | return lowlevel_send_invalid(test, buffer, sizeof(buffer)); 91 | } 92 | 93 | runtest_prio(sp, lowlevel_not_string_argument) { 94 | char buffer[512]; 95 | 96 | if(test->type != CONNECTION_TYPE_TCP) 97 | return TEST_SKIPPED; 98 | 99 | // sending array as first argument (neasted array) which is 100 | // not supported 101 | strcpy(buffer, "*1\r\n*1\r\n$1\r\nX\r\n"); 102 | 103 | return lowlevel_send_invalid(test, buffer, sizeof(buffer)); 104 | } 105 | 106 | // testing tricky case when the payload is just 107 | // enough to make the final \r\n between two buffer 108 | // read server side (by default buffer is 8192 bytes) 109 | runtest_prio(sp, lowlevel_tricky_buffer_limit) { 110 | if(test->mode == SEQUENTIAL) 111 | return TEST_SKIPPED; 112 | 113 | char buffer[8195]; 114 | ssize_t length; 115 | 116 | memcpy(buffer, "*3\r\n$3\r\nSET\r\n$3\r\nXXX\r\n$8164\r\n", 29); 117 | memset(buffer + 29, 'K', 8193 - 29); 118 | memcpy(buffer + 8193, "\r\n", 2); 119 | 120 | if(write(test->zdb->fd, buffer, sizeof(buffer)) < 0) 121 | perror("write"); 122 | 123 | if((length = read(test->zdb->fd, buffer, sizeof(buffer))) < 0) 124 | perror("read"); 125 | 126 | if(memcmp(buffer, "$3\r\nXXX\r\n", 9) == 0) 127 | return TEST_SUCCESS; 128 | 129 | return TEST_FAILED; 130 | } 131 | 132 | // testing tricky case when the header of a field is just 133 | // between two buffer read server side 134 | // (by default buffer is 8192 bytes) 135 | runtest_prio(sp, lowlevel_tricky_buffer_header_limit) { 136 | if(test->mode == SEQUENTIAL) 137 | return TEST_SKIPPED; 138 | 139 | char buffer[8201]; 140 | ssize_t length; 141 | 142 | memcpy(buffer, "*2\r\n$8178\r\n", 11); 143 | memset(buffer + 11, 'W', 8178); 144 | memcpy(buffer + 8189, "\r\n$4\r\nXXXX\r\n", 12); 145 | 146 | if(write(test->zdb->fd, buffer, sizeof(buffer)) < 0) 147 | perror("write"); 148 | 149 | if((length = read(test->zdb->fd, buffer, sizeof(buffer))) < 0) 150 | perror("read"); 151 | 152 | if(memcmp(buffer, "-Command not supported", 22) == 0) 153 | return TEST_SUCCESS; 154 | 155 | return TEST_FAILED; 156 | } 157 | 158 | // testing tricky case when the header of a field is just 159 | // at the end of one packet buffer 160 | // (by default buffer is 8192 bytes) 161 | runtest_prio(sp, lowlevel_tricky_buffer_header_split) { 162 | if(test->mode == SEQUENTIAL) 163 | return TEST_SKIPPED; 164 | 165 | char buffer[8198]; 166 | ssize_t length; 167 | 168 | memcpy(buffer, "*2\r\n$8175\r\n", 11); 169 | memset(buffer + 11, 'W', 8175); 170 | memcpy(buffer + 8186, "\r\n$4\r\nXXXX\r\n", 12); 171 | 172 | if(write(test->zdb->fd, buffer, sizeof(buffer)) < 0) 173 | perror("write"); 174 | 175 | if((length = read(test->zdb->fd, buffer, sizeof(buffer))) < 0) 176 | perror("read"); 177 | 178 | if(memcmp(buffer, "-Command not supported", 22) == 0) 179 | return TEST_SUCCESS; 180 | 181 | return TEST_FAILED; 182 | } 183 | 184 | 185 | 186 | 187 | #define MAX_CONNECTIONS 128 188 | runtest_prio(sp, lowlevel_open_many_connection) { 189 | redisContext *ctx[MAX_CONNECTIONS] = {NULL}; 190 | int response = TEST_SUCCESS; 191 | 192 | if(test->type != CONNECTION_TYPE_TCP) 193 | return TEST_SKIPPED; 194 | 195 | for(unsigned int i = 0; i < MAX_CONNECTIONS; i++) { 196 | ctx[i] = redisConnect(test->host, test->port); 197 | if(!ctx[i] || ctx[i]->err) { 198 | response = TEST_FAILED; 199 | break; 200 | } 201 | } 202 | 203 | for(unsigned int i = 0; i < MAX_CONNECTIONS; i++) { 204 | if(ctx[i]) 205 | redisFree(ctx[i]); 206 | } 207 | 208 | return response; 209 | } 210 | 211 | runtest_prio(sp, lowlevel_mirror) { 212 | const char *argv[] = {"MIRROR"}; 213 | int value = zdb_command(test, argvsz(argv), argv); 214 | 215 | if(test->type != CONNECTION_TYPE_TCP) 216 | return TEST_SKIPPED; 217 | 218 | if(value != TEST_SUCCESS) 219 | return value; 220 | 221 | // reopen the connection 222 | initialize_tcp(test); 223 | return TEST_SUCCESS; 224 | } 225 | 226 | 227 | -------------------------------------------------------------------------------- /tests/zdb_misc.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "tests_user.h" 8 | #include "zdb_utils.h" 9 | #include "tests.h" 10 | 11 | // sequential priority 12 | #define sp 800 13 | 14 | static char *namespace_misc = "default"; 15 | 16 | // select this new namespace 17 | runtest_prio(sp, misc_select) { 18 | const char *argv[] = {"SELECT", namespace_misc}; 19 | return zdb_command(test, argvsz(argv), argv); 20 | } 21 | 22 | 23 | runtest_prio(sp, misc_time) { 24 | redisReply *reply; 25 | 26 | if(!(reply = redisCommand(test->zdb, "TIME"))) 27 | return zdb_result(reply, TEST_FAILED_FATAL); 28 | 29 | if(reply->type != REDIS_REPLY_ARRAY) { 30 | log("Not an array: %s\n", reply->str); 31 | return zdb_result(reply, TEST_FAILED_FATAL); 32 | } 33 | 34 | if(reply->elements != 2) { 35 | log("Wrong argument response: %lu\n", reply->elements); 36 | return zdb_result(reply, TEST_FAILED); 37 | } 38 | 39 | return TEST_SUCCESS; 40 | } 41 | 42 | runtest_prio(sp, misc_info) { 43 | const char *argv[] = {"INFO"}; 44 | return zdb_command_str(test, argvsz(argv), argv); 45 | } 46 | 47 | runtest_prio(sp, misc_wait_missing_args) { 48 | const char *argv[] = {"WAIT"}; 49 | return zdb_command_error(test, argvsz(argv), argv); 50 | } 51 | 52 | runtest_prio(sp, misc_wait_cmd_not_exists) { 53 | const char *argv[] = {"WAIT", "nonexisting"}; 54 | return zdb_command_error(test, argvsz(argv), argv); 55 | } 56 | 57 | static void *misc_wait_send_ping(void *args) { 58 | usleep(500000); 59 | 60 | const char *argv[] = {"PING"}; 61 | zdb_command(args, argvsz(argv), argv); 62 | 63 | return NULL; 64 | } 65 | 66 | runtest_prio(sp, misc_wait_real) { 67 | // cloning settings and duplicating connection 68 | test_t newconn = *test; 69 | initialize(&newconn); 70 | 71 | // creating waiting thread 72 | pthread_t thread; 73 | pthread_create(&thread, NULL, misc_wait_send_ping, &newconn); 74 | 75 | // executing wait command 76 | const char *argv[] = {"WAIT", "PING"}; 77 | int value = zdb_command(test, argvsz(argv), argv); 78 | 79 | // get back from normal 80 | pthread_join(thread, NULL); 81 | 82 | return value; 83 | } 84 | 85 | 86 | -------------------------------------------------------------------------------- /tests/zdb_payload.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "tests_user.h" 6 | #include "zdb_utils.h" 7 | #include "tests.h" 8 | 9 | // sequential priority 10 | #define sp 170 11 | 12 | static char *namespace_payload = "test_payload"; 13 | static size_t sizes_payload[] = { 14 | 512, // 512 bytes 15 | 1024, // 1 KB 16 | 4 * 1024, // 4 KB 17 | 64 * 1024, // 64 KB 18 | 512 * 1024, // 512 KB 19 | 1024 * 1024, // 1 MB 20 | 2 * 1024 * 1024, // 2 MB 21 | 4 * 1024 * 1024, // 4 MB 22 | 8 * 1024 * 1024, // 8 MB 23 | }; 24 | 25 | #define cmdptr int (*command)(test_t *, void *, size_t, void *, size_t) 26 | 27 | int payload_execute(test_t *test, char *key, size_t keylen, size_t length, cmdptr) { 28 | char *payload; 29 | 30 | if(!(payload = malloc(length))) 31 | return TEST_FAILED_FATAL; 32 | 33 | memset(payload, 0x42, length); 34 | 35 | int response = command(test, key, keylen, payload, length); 36 | free(payload); 37 | 38 | return response; 39 | 40 | } 41 | 42 | int set_fixed_payload(test_t *test, size_t index) { 43 | char key[64]; 44 | size_t keylen; 45 | 46 | if(test->mode == USERKEY) { 47 | sprintf(key, "data-%lu", sizes_payload[index]); 48 | keylen = strlen(key); 49 | } 50 | 51 | if(test->mode == SEQUENTIAL) { 52 | memset(key, 0x00, sizeof(key)); 53 | keylen = 0; 54 | } 55 | 56 | return payload_execute(test, key, keylen, sizes_payload[index], zdb_bset); 57 | } 58 | 59 | int get_fixed_payload(test_t *test, uint64_t index) { 60 | char key[64]; 61 | size_t keylen; 62 | 63 | if(test->mode == USERKEY) { 64 | sprintf(key, "data-%lu", sizes_payload[index]); 65 | keylen = strlen(key); 66 | } 67 | 68 | if(test->mode == SEQUENTIAL) { 69 | memcpy(key, &index, sizeof(uint64_t)); 70 | keylen = sizeof(uint64_t); 71 | } 72 | 73 | return payload_execute(test, key, keylen, sizes_payload[index], zdb_bcheck); 74 | } 75 | 76 | 77 | // create a new namespace 78 | runtest_prio(sp, payload_init) { 79 | return zdb_nsnew(test, namespace_payload); 80 | } 81 | 82 | // select this new namespace 83 | runtest_prio(sp, payload_select) { 84 | const char *argv[] = {"SELECT", namespace_payload}; 85 | return zdb_command(test, argvsz(argv), argv); 86 | } 87 | 88 | 89 | // set different datasize, know payload 90 | runtest_prio(sp, payload_set_512b) { 91 | return set_fixed_payload(test, 0); 92 | } 93 | 94 | runtest_prio(sp, payload_set_1k) { 95 | return set_fixed_payload(test, 1); 96 | } 97 | 98 | runtest_prio(sp, payload_set_4k) { 99 | return set_fixed_payload(test, 2); 100 | } 101 | 102 | runtest_prio(sp, payload_set_64k) { 103 | return set_fixed_payload(test, 3); 104 | } 105 | 106 | runtest_prio(sp, payload_set_512k) { 107 | return set_fixed_payload(test, 4); 108 | } 109 | 110 | runtest_prio(sp, payload_set_1m) { 111 | return set_fixed_payload(test, 5); 112 | } 113 | 114 | runtest_prio(sp, payload_set_2m) { 115 | return set_fixed_payload(test, 6); 116 | } 117 | 118 | runtest_prio(sp, payload_set_4m) { 119 | return set_fixed_payload(test, 7); 120 | } 121 | 122 | runtest_prio(sp, payload_set_8m) { 123 | return set_fixed_payload(test, 8); 124 | } 125 | 126 | /* 127 | // client is disconnected if payload is too big 128 | // 129 | // this test should fail (limit is set to 8 MB) 130 | runtest_prio(sp, payload_set_10m_fail) { 131 | int response = set_fixed_payload(test, 10 * 1024 * 1024); 132 | if(response == TEST_FAILED_FATAL) 133 | return TEST_SUCCESS; 134 | 135 | return TEST_FAILED; 136 | } 137 | */ 138 | 139 | 140 | // read the differents key sets and ensure response 141 | runtest_prio(sp, payload_get_512b) { 142 | return get_fixed_payload(test, 0); 143 | } 144 | 145 | runtest_prio(sp, payload_get_1k) { 146 | return get_fixed_payload(test, 1); 147 | } 148 | 149 | runtest_prio(sp, payload_get_4k) { 150 | return get_fixed_payload(test, 2); 151 | } 152 | 153 | runtest_prio(sp, payload_get_64k) { 154 | return get_fixed_payload(test, 3); 155 | } 156 | 157 | runtest_prio(sp, payload_get_512k) { 158 | return get_fixed_payload(test, 4); 159 | } 160 | 161 | runtest_prio(sp, payload_get_1m) { 162 | return get_fixed_payload(test, 5); 163 | } 164 | 165 | runtest_prio(sp, payload_get_2m) { 166 | return get_fixed_payload(test, 6); 167 | } 168 | 169 | runtest_prio(sp, payload_get_4m) { 170 | return get_fixed_payload(test, 7); 171 | } 172 | 173 | runtest_prio(sp, payload_get_8m) { 174 | return get_fixed_payload(test, 8); 175 | } 176 | 177 | 178 | -------------------------------------------------------------------------------- /tests/zdb_scan.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "tests_user.h" 6 | #include "zdb_utils.h" 7 | #include "tests.h" 8 | 9 | // sequential priority 10 | #define sp 175 11 | 12 | static char *namespace_scan = "test_scan"; 13 | 14 | static int scan_check(test_t *test, int argc, const char *argv[], char *expected) { 15 | if(test->mode == SEQUENTIAL) 16 | return TEST_SKIPPED; 17 | 18 | redisReply *reply; 19 | 20 | if(!(reply = zdb_response_scan(test, argc, argv))) 21 | return zdb_result(reply, TEST_FAILED_FATAL); 22 | 23 | redisReply *list = reply->element[1]; 24 | 25 | if(strcmp(list->element[0]->element[0]->str, expected) == 0) 26 | return zdb_result(reply, TEST_SUCCESS); 27 | 28 | log("%s\n", reply->str); 29 | 30 | return zdb_result(reply, TEST_FAILED); 31 | } 32 | 33 | // create a new namespace 34 | runtest_prio(sp, scan_init) { 35 | return zdb_nsnew(test, namespace_scan); 36 | } 37 | 38 | // select this new namespace 39 | runtest_prio(sp, scan_select) { 40 | const char *argv[] = {"SELECT", namespace_scan}; 41 | return zdb_command(test, argvsz(argv), argv); 42 | } 43 | 44 | 45 | runtest_prio(sp, scan_init_chain1) { 46 | return zdb_set(test, "key1", "aaaa"); 47 | } 48 | 49 | runtest_prio(sp, scan_init_chain2) { 50 | return zdb_set(test, "key2", "bbbb"); 51 | } 52 | 53 | runtest_prio(sp, scan_init_chain3) { 54 | return zdb_set(test, "key3", "cccc"); 55 | } 56 | 57 | runtest_prio(sp, scan_init_chain4) { 58 | return zdb_set(test, "key4", "dddd"); 59 | } 60 | 61 | runtest_prio(sp, scan_init_chain5) { 62 | return zdb_set(test, "key5", "eeee"); 63 | } 64 | 65 | runtest_prio(sp, scan_init_chain6) { 66 | return zdb_set(test, "key6", "ffff"); 67 | } 68 | 69 | // start scan test 70 | runtest_prio(sp, scan_get_first_key) { 71 | const char *argv[] = {"SCAN"}; 72 | return scan_check(test, argvsz(argv), argv, "key1"); 73 | } 74 | 75 | runtest_prio(sp, scan_get_last_key) { 76 | const char *argv[] = {"RSCAN"}; 77 | return scan_check(test, argvsz(argv), argv, "key6"); 78 | } 79 | 80 | /* 81 | runtest_prio(sp, scan_get_second_key) { 82 | const char *argv[] = {"SCAN", "key1"}; 83 | return scan_check(test, argvsz(argv), argv, "key2"); 84 | } 85 | 86 | runtest_prio(sp, scan_get_last_minusone_key) { 87 | const char *argv[] = {"RSCAN", "key6"}; 88 | return scan_check(test, argvsz(argv), argv, "key5"); 89 | } 90 | */ 91 | 92 | 93 | runtest_prio(sp, scan_remove_first) { 94 | if(test->mode == SEQUENTIAL) 95 | return TEST_SKIPPED; 96 | 97 | const char *argv[] = {"DEL", "key1"}; 98 | return zdb_command(test, argvsz(argv), argv); 99 | } 100 | 101 | runtest_prio(sp, scan_get_new_first_key) { 102 | const char *argv[] = {"SCAN"}; 103 | return scan_check(test, argvsz(argv), argv, "key2"); 104 | } 105 | 106 | 107 | runtest_prio(sp, scan_remove_last) { 108 | if(test->mode == SEQUENTIAL) 109 | return TEST_SKIPPED; 110 | 111 | const char *argv[] = {"DEL", "key6"}; 112 | return zdb_command(test, argvsz(argv), argv); 113 | } 114 | 115 | runtest_prio(sp, scan_get_new_last_key) { 116 | const char *argv[] = {"RSCAN"}; 117 | return scan_check(test, argvsz(argv), argv, "key5"); 118 | } 119 | 120 | // scan on unknown key 121 | runtest_prio(sp, scan_non_existing) { 122 | const char *argv[] = {"SCAN", "nonexisting"}; 123 | return zdb_command_error(test, argvsz(argv), argv); 124 | } 125 | 126 | runtest_prio(sp, rscan_non_existing) { 127 | const char *argv[] = {"RSCAN", "nonexisting"}; 128 | return zdb_command_error(test, argvsz(argv), argv); 129 | } 130 | 131 | // scan on deleted key 132 | runtest_prio(sp, scan_deleted_key) { 133 | const char *argv[] = {"SCAN", "key1"}; 134 | return zdb_command_error(test, argvsz(argv), argv); 135 | } 136 | 137 | runtest_prio(sp, rscan_deleted_key) { 138 | const char *argv[] = {"RSCAN", "key1"}; 139 | return zdb_command_error(test, argvsz(argv), argv); 140 | } 141 | 142 | // scan from last key 143 | runtest_prio(sp, scan_ask_after_last) { 144 | const char *argv[] = {"SCAN", "key5"}; 145 | return zdb_command_error(test, argvsz(argv), argv); 146 | } 147 | 148 | // rscan from first key 149 | runtest_prio(sp, scan_ask_before_first) { 150 | const char *argv[] = {"RSCAN", "key2"}; 151 | return zdb_command_error(test, argvsz(argv), argv); 152 | } 153 | 154 | runtest_prio(sp, scan_keycur_not_exists) { 155 | const char *argv[] = {"KEYCUR", "not-exists"}; 156 | return zdb_command_error(test, argvsz(argv), argv); 157 | } 158 | 159 | runtest_prio(sp, scan_keycur_exists) { 160 | if(test->mode == SEQUENTIAL) 161 | return TEST_SKIPPED; 162 | 163 | const char *argv[] = {"KEYCUR", "key4"}; 164 | return zdb_command_str(test, argvsz(argv), argv); 165 | } 166 | 167 | runtest_prio(sp, scan_kscan_invalid) { 168 | if(test->mode == SEQUENTIAL) 169 | return TEST_SKIPPED; 170 | 171 | const char *argv[] = {"KSCAN"}; 172 | return zdb_command_error(test, argvsz(argv), argv); 173 | } 174 | 175 | runtest_prio(sp, scan_kscan_switch_default) { 176 | const char *argv[] = {"SELECT", "default"}; 177 | return zdb_command(test, argvsz(argv), argv); 178 | } 179 | 180 | runtest_prio(sp, scan_kscan) { 181 | if(test->mode == SEQUENTIAL) 182 | return TEST_SKIPPED; 183 | 184 | const char *argv[] = {"KSCAN", "h"}; 185 | redisReply *reply = zdb_response_scan(test, argvsz(argv), argv); 186 | 187 | if(reply) { 188 | freeReplyObject(reply); 189 | return TEST_SUCCESS; 190 | } 191 | 192 | return TEST_FAILED; 193 | } 194 | 195 | -------------------------------------------------------------------------------- /tests/zdb_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_TESTS_UTILS_H 2 | #define ZDB_TESTS_UTILS_H 3 | 4 | int zdb_result(redisReply *reply, int value); 5 | int zdb_command(test_t *test, int argc, const char *argv[]); 6 | int zdb_command_str(test_t *test, int argc, const char *argv[]); 7 | int zdb_command_error(test_t *test, int argc, const char *argv[]); 8 | int zdb_set(test_t *test, char *key, char *value); 9 | int zdb_set_seq(test_t *test, uint64_t key, char *value, uint64_t *response); 10 | int zdb_bset(test_t *test, void *key, size_t keylen, void *payload, size_t paylen); 11 | int zdb_check(test_t *test, char *key, char *value); 12 | int zdb_bcheck(test_t *test, void *key, size_t keylen, void *payload, size_t paylen); 13 | int zdb_nsnew(test_t *test, char *nsname); 14 | 15 | int zdb_basic_check(test_t *test, const char *command); 16 | long long zdb_command_integer(test_t *test, int argc, const char *argv[]); 17 | redisReply *zdb_response_scan(test_t *test, int argc, const char *argv[]); 18 | redisReply *zdb_response_history(test_t *test, int argc, const char *argv[]); 19 | 20 | char *zdb_auth_challenge(test_t *test); 21 | 22 | #define SEQNEW 1337 23 | #define argvsz(x) (sizeof(x) / sizeof(char *)) 24 | #endif 25 | -------------------------------------------------------------------------------- /tools/Makefile: -------------------------------------------------------------------------------- 1 | all release clean mrproper: 2 | $(MAKE) -C index-dump $@ 3 | $(MAKE) -C integrity-check $@ 4 | # $(MAKE) -C compaction $@ 5 | $(MAKE) -C index-rebuild $@ 6 | $(MAKE) -C namespace-editor $@ 7 | $(MAKE) -C namespace-dump $@ 8 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # Zero-DB Tools 2 | 3 | ## Compaction 4 | Parse whole `datafiles` of a namespace, and discard data not needed anymore 5 | 6 | ## Index Dump 7 | Debug tool, dumping the contents of a specific `indexfile` 8 | 9 | ## Index Rebuild 10 | Rebuild a whole index directory based on data directory 11 | 12 | ## Integrity Check 13 | Check integrity of a datafile (offline integrity check) 14 | 15 | ## Namespace Editor 16 | Create or edit a namespace descriptor file 17 | -------------------------------------------------------------------------------- /tools/compaction/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = compaction 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../src 6 | LDFLAGS += -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | # all: $(EXEC) 15 | all: 16 | 17 | release: CFLAGS += -DRELEASE 18 | release: $(EXEC) 19 | 20 | $(EXEC): $(OBJ) 21 | $(CC) -o $@ $^ $(LDFLAGS) 22 | 23 | %.o: %.c 24 | $(CC) $(CFLAGS) -c $< 25 | 26 | clean: 27 | $(RM) *.o 28 | 29 | mrproper: clean 30 | $(RM) $(EXEC) 31 | -------------------------------------------------------------------------------- /tools/compaction/branches.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "zerodb.h" 8 | #include "index.h" 9 | #include "index_branch.h" 10 | 11 | // maximum allowed branch in memory 12 | // 13 | // this settings is mainly the most important to 14 | // determine the keys lookup time 15 | // 16 | // the more bits you allows here, the more buckets 17 | // can be used for lookup without collision 18 | // 19 | // the index works like a hash-table and uses crc32 'hash' 20 | // algorithm, the result of the crc32 is used to point to 21 | // the bucket, but using a full 32-bits hashlist would 22 | // consume more than (2^32 * 8) bytes of memory (on 64-bits) 23 | // 24 | // the default settings sets this to 24 bits, which allows 25 | // 16 millions direct entries, collisions uses linked-list 26 | // 27 | // makes sur mask and amount of branch are always in relation 28 | // use 'index_set_buckets_bits' to be sure 29 | uint32_t buckets_branches = (1 << 24); 30 | uint32_t buckets_mask = (1 << 24) - 1; 31 | 32 | // WARNING: this doesn't resize anything, you should calls this 33 | // only before initialization 34 | int index_set_buckets_bits(uint8_t bits) { 35 | buckets_branches = 1 << bits; 36 | buckets_mask = (1 << bits) - 1; 37 | 38 | return buckets_branches; 39 | } 40 | 41 | // perform the basic "hashing" (crc based) used to point to the expected branch 42 | // we only keep partial amount of the result to not fill the memory too fast 43 | uint32_t index_key_hash(unsigned char *id, uint8_t idlength) { 44 | uint64_t *input = (uint64_t *) id; 45 | uint32_t hash = 0; 46 | ssize_t i = 0; 47 | 48 | for(i = 0; i < idlength - 8; i += 8) 49 | hash = _mm_crc32_u64(hash, *input++); 50 | 51 | for(; i < idlength; i++) 52 | hash = _mm_crc32_u8(hash, id[i]); 53 | 54 | return hash & buckets_mask; 55 | } 56 | 57 | index_entry_t *index_entry_get(index_root_t *root, unsigned char *id, uint8_t idlength) { 58 | uint32_t branchkey = index_key_hash(id, idlength); 59 | index_branch_t *branch = index_branch_get(root->branches, branchkey); 60 | index_entry_t *entry; 61 | 62 | // branch not exists 63 | if(!branch) 64 | return NULL; 65 | 66 | for(entry = branch->list; entry; entry = entry->next) { 67 | if(entry->idlength != idlength) 68 | continue; 69 | 70 | if(entry->namespace != root->namespace) 71 | continue; 72 | 73 | if(memcmp(entry->id, id, idlength) == 0) 74 | return entry; 75 | } 76 | 77 | return NULL; 78 | } 79 | 80 | 81 | // 82 | // index branch 83 | // this implementation use a lazy load of branches 84 | // this allows us to use lot of branch (buckets_branches) in this case) 85 | // without consuming all the memory if we don't need it 86 | // 87 | index_branch_t **index_buckets_init() { 88 | return (index_branch_t **) calloc(sizeof(index_branch_t *), buckets_branches); 89 | } 90 | 91 | index_branch_t *index_branch_init(index_branch_t **branches, uint32_t branchid) { 92 | // debug("[+] initializing branch id 0x%x\n", branchid); 93 | 94 | branches[branchid] = malloc(sizeof(index_branch_t)); 95 | index_branch_t *branch = branches[branchid]; 96 | 97 | branch->length = 0; 98 | branch->last = NULL; 99 | branch->list = NULL; 100 | 101 | return branch; 102 | } 103 | 104 | void index_branch_free(index_branch_t **branches, uint32_t branchid) { 105 | // this branch was not allocated 106 | if(!branches[branchid]) 107 | return; 108 | 109 | index_entry_t *entry = branches[branchid]->list; 110 | index_entry_t *next = NULL; 111 | 112 | // deleting branch content by 113 | // iterate over the linked-list 114 | for(; entry; entry = next) { 115 | next = entry->next; 116 | free(entry); 117 | } 118 | 119 | // deleting branch 120 | free(branches[branchid]); 121 | } 122 | 123 | // returns branch from rootindex, if branch is not allocated yet, returns NULL 124 | // useful for any read on the index in memory 125 | index_branch_t *index_branch_get(index_branch_t **branches, uint32_t branchid) { 126 | if(!branches) 127 | return NULL; 128 | 129 | return branches[branchid]; 130 | } 131 | 132 | // returns branch from rootindex, if branch doesn't exists, it will be allocated 133 | // (useful for any write in the index in memory) 134 | index_branch_t *index_branch_get_allocate(index_branch_t **branches, uint32_t branchid) { 135 | if(!branches[branchid]) 136 | return index_branch_init(branches, branchid); 137 | 138 | // debug("[+] branch: exists: %lu entries\n", branches[branchid]->length); 139 | return branches[branchid]; 140 | } 141 | 142 | // append an entry (item) to the memory list 143 | // since we use a linked-list, the logic of appending 144 | // only occures here 145 | // 146 | // if there is no index, we just skip the appending 147 | index_entry_t *index_branch_append(index_branch_t **branches, uint32_t branchid, index_entry_t *entry) { 148 | index_branch_t *branch; 149 | 150 | if(!branches) 151 | return NULL; 152 | 153 | // grabbing the branch 154 | branch = index_branch_get_allocate(branches, branchid); 155 | branch->length += 1; 156 | 157 | // adding this item and pointing previous last one 158 | // to this new one 159 | if(!branch->list) 160 | branch->list = entry; 161 | 162 | if(branch->last) 163 | branch->last->next = entry; 164 | 165 | branch->last = entry; 166 | entry->next = NULL; 167 | 168 | return entry; 169 | } 170 | 171 | // remove one entry on this branch 172 | // since it's a linked-list, we need to know which entry was the previous one 173 | // we use a single-direction linked-list 174 | // 175 | // removing an entry from the list don't free this entry, is just re-order 176 | // list to keep it coherent 177 | index_entry_t *index_branch_remove(index_branch_t *branch, index_entry_t *entry, index_entry_t *previous) { 178 | // removing the first entry 179 | if(branch->list == entry) 180 | branch->list = entry->next; 181 | 182 | // skipping this entry, linking next from previous 183 | // to our next one 184 | if(previous) 185 | previous->next = entry->next; 186 | 187 | // if our entry was the last one 188 | // the new last one is the previous one 189 | if(branch->last == entry) 190 | branch->last = previous; 191 | 192 | branch->length -= 1; 193 | 194 | return entry; 195 | } 196 | -------------------------------------------------------------------------------- /tools/compaction/branches.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDB_INDEX_BRANCH_H 2 | #define __ZDB_INDEX_BRANCH_H 3 | 4 | // buckets 5 | extern uint32_t buckets_branches; 6 | extern uint32_t buckets_mask; 7 | 8 | int index_set_buckets_bits(uint8_t bits); 9 | index_branch_t **index_buckets_init(); 10 | 11 | // initializers 12 | index_branch_t *index_branch_init(index_branch_t **branches, uint32_t branchid); 13 | void index_branch_free(index_branch_t **branches, uint32_t branchid); 14 | 15 | // accessors 16 | index_branch_t *index_branch_get(index_branch_t **branches, uint32_t branchid); 17 | index_branch_t *index_branch_get_allocate(index_branch_t **branches, uint32_t branchid); 18 | index_entry_t *index_branch_append(index_branch_t **branches, uint32_t branchid, index_entry_t *entry); 19 | index_entry_t *index_branch_remove(index_branch_t *branch, index_entry_t *entry, index_entry_t *previous); 20 | 21 | uint32_t index_key_hash(unsigned char *id, uint8_t idlength); 22 | #endif 23 | -------------------------------------------------------------------------------- /tools/compaction/compaction.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_TOOLS_COMPACTION_H 2 | #define ZDB_TOOLS_COMPACTION_H 3 | 4 | 5 | typedef struct datamap_entry_t { 6 | off_t offset; // offset on source file 7 | size_t length; // full entry length (header + id + payload) 8 | char keep; // do we need to keep this entry or not 9 | 10 | } datamap_entry_t; 11 | 12 | typedef struct datamap_t { 13 | size_t length; 14 | size_t allocated; 15 | fileid_t fileid; 16 | datamap_entry_t *entries; 17 | 18 | } datamap_t; 19 | 20 | typedef struct compaction_t { 21 | char *datapath; 22 | char *targetpath; 23 | char *namespace; 24 | 25 | datamap_t **filesmap; 26 | 27 | } compaction_t; 28 | 29 | void *warnp(char *str); 30 | void diep(char *str); 31 | void dies(char *str); 32 | #endif 33 | -------------------------------------------------------------------------------- /tools/compaction/namespace.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "zerodb.h" 13 | #include "index.h" 14 | #include "index_branch.h" 15 | #include "index_loader.h" 16 | #include "data.h" 17 | #include "namespace.h" 18 | #include "filesystem.h" 19 | #include "redis.h" 20 | #include "hook.h" 21 | 22 | static int namespace_descriptor_open(namespace_t *namespace) { 23 | char pathname[ZDB_PATH_MAX]; 24 | int fd; 25 | 26 | snprintf(pathname, ZDB_PATH_MAX, "%s/zdb-namespace", namespace->indexpath); 27 | 28 | if((fd = open(pathname, O_CREAT | O_RDWR, 0600)) < 0) { 29 | warning("[-] cannot create or open in read-write the namespace file\n"); 30 | return -1; 31 | } 32 | 33 | return fd; 34 | } 35 | 36 | // read (or create) a namespace descriptor 37 | // namespace descriptor is a binary file containing namespace 38 | // specification such password, maxsize, etc. (see header) 39 | static void namespace_descriptor_load(namespace_t *namespace) { 40 | ns_header_t header; 41 | int fd; 42 | 43 | if((fd = namespace_descriptor_open(namespace)) < 0) 44 | return; 45 | 46 | if(read(fd, &header, sizeof(ns_header_t)) != sizeof(ns_header_t)) { 47 | // probably new file, let's write initial namespace information 48 | close(fd); 49 | return; 50 | } 51 | 52 | namespace->maxsize = header.maxsize; 53 | namespace->public = (header.flags & NS_FLAGS_PUBLIC); 54 | 55 | if(header.passlength) { 56 | if(!(namespace->password = calloc(sizeof(char), header.passlength + 1))) { 57 | warnp("namespace password malloc"); 58 | return; 59 | } 60 | 61 | // skipping the namespace name, jumping to password 62 | lseek(fd, header.namelength, SEEK_CUR); 63 | 64 | if(read(fd, namespace->password, header.passlength) != (ssize_t) header.passlength) 65 | warnp("namespace password read"); 66 | } 67 | 68 | debug("[+] namespace '%s': maxsize: %lu\n", namespace->name, namespace->maxsize); 69 | debug("[+] -> password protection: %s\n", namespace->password ? "yes" : "no"); 70 | debug("[+] -> public access: %s\n", namespace->public ? "yes" : "no"); 71 | 72 | close(fd); 73 | } 74 | 75 | static char *namespace_path(char *prefix, char *name) { 76 | char pathname[ZDB_PATH_MAX]; 77 | snprintf(pathname, ZDB_PATH_MAX, "%s/%s", prefix, name); 78 | 79 | return strdup(pathname); 80 | } 81 | 82 | 83 | // load (or create if it doesn't exists) a namespace 84 | namespace_t *namespace_load_light(ns_root_t *nsroot, char *name) { 85 | namespace_t *namespace; 86 | 87 | debug("[+] namespaces: loading '%s'\n", name); 88 | 89 | if(!(namespace = malloc(sizeof(namespace_t)))) { 90 | warnp("namespace malloc"); 91 | return NULL; 92 | } 93 | 94 | namespace->name = strdup(name); 95 | namespace->password = NULL; // no password by default, need to be set later 96 | namespace->indexpath = namespace_path(nsroot->settings->indexpath, name); 97 | namespace->datapath = namespace_path(nsroot->settings->datapath, name); 98 | namespace->public = 1; // by default, namespace are public (no password) 99 | namespace->maxsize = 0; // by default, there is no limits 100 | namespace->idlist = 0; // by default, no list set 101 | 102 | // load descriptor from disk 103 | namespace_descriptor_load(namespace); 104 | 105 | return namespace; 106 | } 107 | 108 | ns_root_t *namespaces_allocate(settings_t *settings) { 109 | ns_root_t *root; 110 | 111 | // we start by the default namespace 112 | if(!(root = (ns_root_t *) malloc(sizeof(ns_root_t)))) 113 | diep("namespaces malloc"); 114 | 115 | root->length = 1; // we start with the default one, only 116 | root->effective = 1; // no namespace really loaded yet 117 | root->settings = settings; // keep reference to the settings, needed for paths 118 | root->branches = NULL; // maybe we don't need the branches, see below 119 | 120 | if(!(root->namespaces = (namespace_t **) malloc(sizeof(namespace_t *) * root->length))) 121 | diep("namespace malloc"); 122 | 123 | // allocating (if needed, only some modes needs it) the big (single) index branches 124 | if(settings->mode == KEYVALUE || settings->mode == SEQUENTIAL) { 125 | debug("[+] namespaces: pre-allocating index (%d lazy branches)\n", buckets_branches); 126 | 127 | // allocating minimal branches array 128 | if(!(root->branches = (index_branch_t **) calloc(sizeof(index_branch_t *), buckets_branches))) 129 | diep("calloc"); 130 | } 131 | 132 | return root; 133 | } 134 | 135 | -------------------------------------------------------------------------------- /tools/compaction/validity.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "compaction.h" 9 | 10 | int directory_check(char *target) { 11 | struct stat sb; 12 | 13 | if(stat(target, &sb) != 0) { 14 | warnp(target); 15 | return 1; 16 | } 17 | 18 | if(!S_ISDIR(sb.st_mode)) 19 | return 1; 20 | 21 | return 0; 22 | } 23 | 24 | int validity_check(compaction_t *compaction) { 25 | char filename[256]; 26 | 27 | // preliminary check 28 | // does the data directory exists 29 | if(directory_check(compaction->datapath)) { 30 | fprintf(stderr, "[-] %s: target is not a directory\n", compaction->datapath); 31 | return 1; 32 | } 33 | 34 | // does data namespace directory exists 35 | snprintf(filename, sizeof(filename), "%s/%s", compaction->datapath, compaction->namespace); 36 | if(directory_check(filename)) { 37 | fprintf(stderr, "[-] %s: target is not a directory\n", filename); 38 | return 1; 39 | } 40 | 41 | // does the target directory exists 42 | if(directory_check(compaction->targetpath)) { 43 | fprintf(stderr, "[-] %s: target is not a directory\n", compaction->targetpath); 44 | return 1; 45 | } 46 | 47 | // does the target namespace directory 48 | // **doesn't** exists (we want to create it, to start fresh) 49 | snprintf(filename, sizeof(filename), "%s/%s", compaction->targetpath, compaction->namespace); 50 | if(!directory_check(filename)) { 51 | fprintf(stderr, "[-] %s: target already exists\n", compaction->targetpath); 52 | dies("the namespace on the target directory should not already exists"); 53 | } 54 | 55 | if(mkdir(filename, 0775)) 56 | diep(filename); 57 | 58 | return 0; 59 | } 60 | 61 | 62 | -------------------------------------------------------------------------------- /tools/compaction/validity.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_TOOLS_COMPACTION_VALIDITY_H 2 | #define ZDB_TOOLS_COMPACTION_VALIDITY_H 3 | 4 | int validity_check(compaction_t *compaction); 5 | #endif 6 | -------------------------------------------------------------------------------- /tools/hooks-monitor/zdb-hooks-monitor.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import redis 3 | import time 4 | from datetime import datetime 5 | 6 | class ZDBHooksMonitor: 7 | def __init__(self, host, port=9900): 8 | self.host = host 9 | self.port = port 10 | 11 | self.green = "\033[32;1m" 12 | self.yellow = "\033[33;1m" 13 | self.reset = "\033[0m" 14 | 15 | self.redis = redis.Redis(host, port) 16 | 17 | def live(self): 18 | while True: 19 | hooks = self.redis.execute_command("HOOKS") 20 | 21 | print("\033[2J\033[H") 22 | print(" Hook | Status | Started | Ended | Argument") 23 | print("----------------------+--------------+---------------+-----------------+---------------") 24 | 25 | for hook in hooks: 26 | name = hook[0].decode('utf-8') 27 | 28 | status = self.green + "finished" + self.reset + (" [%d]" % hook[5]) 29 | 30 | if hook[4] == 0: 31 | status = self.yellow + "running" + self.reset 32 | 33 | started = datetime.fromtimestamp(hook[3]) 34 | ended = datetime.fromtimestamp(hook[4]) 35 | command = hook[1][0].decode('utf-8') if len(hook[1]) > 0 else "" 36 | 37 | startstr = started.strftime("%m-%d %H:%M:%S") 38 | endstr = ended.strftime("%m-%d %H:%M:%S") if hook[4] != 0 else "..." 39 | 40 | print(" %-20s | %-23s | %-13s | %-14s | %s" % (name, status, startstr, endstr, command)) 41 | 42 | print("") 43 | time.sleep(1) 44 | 45 | if __name__ == '__main__': 46 | host = "localhost" 47 | port = 9900 48 | 49 | if len(sys.argv) > 1: 50 | host = sys.argv[1] 51 | 52 | if len(sys.argv) > 2: 53 | port = int(sys.argv[2]) 54 | 55 | zls = ZDBHooksMonitor(host, port) 56 | zls.live() 57 | -------------------------------------------------------------------------------- /tools/incremental-update/incremental.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import redis 3 | import time 4 | import hashlib 5 | 6 | class ZDBIncremental: 7 | def __init__(self, master, mport, slave, sport): 8 | minfo = {"host": master, "port": mport, "namespace": "default", "name": "master"} 9 | sinfo = {"host": slave, "port": sport, "namespace": "default", "name": "slave"} 10 | 11 | self.master = redis.Redis(host=master, port=mport) 12 | self.master.__data = minfo 13 | 14 | self.slave = redis.Redis(host=slave, port=sport) 15 | self.slave.__data = sinfo 16 | 17 | # disable defaults callbacks 18 | for target in [self.master, self.slave]: 19 | target.set_response_callback("NSINFO", target.response_callbacks['INFO']) 20 | target.set_response_callback("DEL", target.response_callbacks['RENAME']) # rename uses bool_ok like we need 21 | target.set_response_callback("SET", bytes) 22 | target.set_response_callback("AUTH", bytes) 23 | 24 | # locking needs some review without authentication 25 | self.slave_lock = False 26 | 27 | def authenticate(self, target, password): 28 | print(f"[+] authenticating: {target.__data['name']}") 29 | 30 | request = target.execute_command("AUTH", "SECURE", "CHALLENGE") 31 | challenge = request.decode('utf-8') 32 | 33 | encoded = f"{challenge}:{password}" 34 | response = hashlib.sha1(encoded.encode("utf-8")).hexdigest() 35 | 36 | # authenticate 37 | status = target.execute_command("AUTH", "SECURE", response) 38 | if status != b"OK": 39 | return False 40 | 41 | # authenticate on namespace as well 42 | status = target.execute_command("SELECT", target.__data["namespace"], password) 43 | 44 | return status == b"OK" 45 | 46 | def sync(self, master, slave): 47 | try: 48 | raw = self.master.execute_command("DATA", "RAW", slave['dataid'], slave['offset']) 49 | 50 | except redis.exceptions.ResponseError as e: 51 | if str(e) == "EOF": 52 | self.slave.execute_command("NSJUMP") 53 | return None 54 | 55 | raise e 56 | 57 | # print(raw) 58 | 59 | if raw[3] == 0: 60 | # SET 61 | response = self.slave.execute_command("SET", raw[0], raw[5], raw[4]) 62 | if response != raw[0]: 63 | raise RuntimeError(f"incorrect set {response}") 64 | 65 | else: 66 | # DEL 67 | self.slave.execute_command("DEL", raw[0], raw[4]) 68 | 69 | 70 | def run(self): 71 | print(f"[+] master host: {self.master.__data['host']}, port: {self.master.__data['port']}") 72 | print(f"[+] slave host: {self.slave.__data['host']}, port: {self.slave.__data['port']}") 73 | print(f"[+] syncing namespaces: {self.master.__data['namespace']} -> {self.slave.__data['namespace']}") 74 | 75 | if self.slave_lock: 76 | print("[+] locking slave namespace") 77 | self.slave.execute_command("NSSET", self.slave.__data['namespace'], "lock", "1") 78 | 79 | while True: 80 | master = {} 81 | slave = {} 82 | 83 | nsmaster = self.master.execute_command("NSINFO", self.master.__data['namespace']) 84 | master['dataid'] = int(nsmaster['data_current_id']) 85 | master['offset'] = int(nsmaster['data_current_offset']) 86 | master['size'] = int(nsmaster['data_size_bytes']) 87 | 88 | nsslave = self.slave.execute_command("NSINFO", self.slave.__data['namespace']) 89 | slave['dataid'] = int(nsslave['data_current_id']) 90 | slave['offset'] = int(nsslave['data_current_offset']) 91 | slave['size'] = int(nsslave['data_size_bytes']) 92 | 93 | if slave['dataid'] > master['dataid']: 94 | raise RuntimeError("slave ahead from master") 95 | 96 | if master['dataid'] == slave['dataid']: 97 | if master['offset'] == slave['offset']: 98 | if self.slave_lock: 99 | # unlocking namespace 100 | self.slave.execute_command("NSSET", self.slave.__data['namespace'], "lock", "0") 101 | 102 | sys.stdout.write("\r[+] syncing: %.2f / %.2f MB (%.1f %%), waiting changes \033[K" % (ssize, msize, progress)) 103 | time.sleep(10) 104 | 105 | if self.slave_lock: 106 | # locking again 107 | self.slave.execute_command("NSSET", self.slave.__data['namespace'], "lock", "1") 108 | 109 | continue 110 | 111 | msize = master['size'] / 1024 / 1024 112 | ssize = slave['size'] / 1024 / 1024 113 | progress = (slave['size'] / master['size']) * 100 114 | dataid = slave['dataid'] 115 | offset = slave['offset'] 116 | 117 | sys.stdout.write("\r[+] syncing: %.2f / %.2f MB (%.1f %%) [request %d:%d] \033[K" % (ssize, msize, progress, dataid, offset)) 118 | sys.stdout.flush() 119 | 120 | self.sync(master, slave) 121 | 122 | 123 | if __name__ == '__main__': 124 | incremental = ZDBIncremental("hub.grid.tf", 9900, "127.0.0.1", 9900) 125 | incremental.authenticate(incremental.master, "master-password") 126 | incremental.authenticate(incremental.slave, "slave-password") 127 | incremental.run() 128 | -------------------------------------------------------------------------------- /tools/index-dump/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = index-dump 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | $(EXEC): $(OBJ) 25 | $(CC) -o $@ $^ $(LDFLAGS) 26 | 27 | %.o: %.c 28 | $(CC) $(CFLAGS) -c $< 29 | 30 | clean: 31 | $(RM) *.o 32 | 33 | mrproper: clean 34 | $(RM) $(EXEC) 35 | -------------------------------------------------------------------------------- /tools/index-dump/index-dump.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "libzdb.h" 6 | 7 | int index_dump_files(index_root_t *zdbindex, uint64_t maxfile) { 8 | index_header_t *header; 9 | char datestr[64]; 10 | size_t totalentries = 0; 11 | 12 | for(fileid_t fileid = 0; fileid < maxfile; fileid += 1) { 13 | // 14 | // loading index file 15 | // 16 | zdb_index_open_readonly(zdbindex, fileid); 17 | 18 | if(!(header = zdb_index_descriptor_load(zdbindex))) 19 | return 1; 20 | 21 | if(!zdb_index_descriptor_validate(header, zdbindex)) 22 | return 1; 23 | 24 | printf("[+] index-dump: header seems correct\n"); 25 | printf("[+] index-dump: created at: %s\n", zdb_header_date(header->created, datestr, sizeof(datestr))); 26 | printf("[+] index-dump: last open: %s\n", zdb_header_date(header->opened, datestr, sizeof(datestr))); 27 | printf("[+] index-dump: index mode: %s\n", zdb_running_mode(header->mode)); 28 | 29 | // 30 | // dumping contents 31 | // 32 | index_item_t *entry = NULL; 33 | size_t entrycount = 0; 34 | off_t curoff; 35 | 36 | curoff = zdb_index_raw_offset(zdbindex); 37 | 38 | while((entry = zdb_index_raw_fetch_entry(zdbindex))) { 39 | entrycount += 1; 40 | totalentries += 1; 41 | 42 | zdb_header_date(entry->timestamp, datestr, sizeof(datestr)); 43 | 44 | printf("[+] index entry: %lu, offset: %" PRId64 "\n", entrycount, (int64_t) curoff); 45 | printf("[+] id length : %" PRIu8 "\n", entry->idlength); 46 | printf("[+] data length: %" PRIu32 "\n", entry->length); 47 | printf("[+] data offset: %" PRIu32 "\n", entry->offset); 48 | printf("[+] data fileid: %" PRIu16 "\n", entry->dataid); 49 | printf("[+] entry flags: 0x%X\n", entry->flags); 50 | printf("[+] entry date : %s\n", datestr); 51 | printf("[+] previous : %" PRIu32 "\n", entry->previous); 52 | printf("[+] data crc : %08x\n", entry->crc); 53 | printf("[+] parent id : %" PRIu16 "\n", entry->parentid); 54 | printf("[+] parent offs: %" PRIu32 "\n", entry->parentoff); 55 | printf("[+] entry key : "); 56 | zdb_tools_hexdump(entry->id, entry->idlength); 57 | printf("\n"); 58 | 59 | // saving current offset 60 | curoff = zdb_index_raw_offset(zdbindex); 61 | free(entry); 62 | } 63 | 64 | printf("[+] ---------------------------\n"); 65 | printf("[+] file done, file entries found: %lu\n", entrycount); 66 | 67 | zdb_index_close(zdbindex); 68 | } 69 | 70 | printf("[+] ---------------------------\n"); 71 | printf("[+] all done, entries found: %lu\n", totalentries); 72 | 73 | return 0; 74 | } 75 | 76 | int main(int argc, char *argv[]) { 77 | char *dirname = NULL; 78 | 79 | if(argc < 2) { 80 | fprintf(stderr, "Usage: %s index-path\n", argv[0]); 81 | exit(EXIT_FAILURE); 82 | } 83 | 84 | // loading zdb 85 | printf("[*] 0-db engine v%s\n", zdb_version()); 86 | 87 | // fetching directory path 88 | dirname = argv[1]; 89 | printf("[+] index-dump: loading: %s\n", dirname); 90 | 91 | if(zdb_dir_exists(dirname) != ZDB_DIRECTORY_EXISTS) { 92 | fprintf(stderr, "[-] index-dump: could not reach index directory\n"); 93 | exit(EXIT_FAILURE); 94 | } 95 | 96 | // initializing database 97 | zdb_settings_t *zdb_settings = zdb_initialize(); 98 | zdb_id_set("index-dump"); 99 | 100 | // WARNING: we *don't* open the database, this would populate 101 | // memory, reading and loading everything... we just want 102 | // to initialize structs but not loads anything 103 | // 104 | // we will shortcut the database loading and directly 105 | // call the low-level index lazy loader (lazy loader won't 106 | // load anything except structs) 107 | // 108 | // zdb_open(zdb_settings); 109 | index_root_t *zdbindex; 110 | 111 | if(!(zdbindex = zdb_index_init_lazy(zdb_settings, dirname, NULL))) { 112 | fprintf(stderr, "[-] index-dump: cannot load index\n"); 113 | exit(EXIT_FAILURE); 114 | } 115 | 116 | uint64_t maxfile = zdb_index_availity_check(zdbindex); 117 | if(maxfile == 0) { 118 | fprintf(stderr, "[-] index-dump: no index files found\n"); 119 | exit(EXIT_FAILURE); 120 | } 121 | 122 | // dumping index 123 | return index_dump_files(zdbindex, maxfile); 124 | } 125 | -------------------------------------------------------------------------------- /tools/index-rebuild/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = index-rebuild 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | $(EXEC): $(OBJ) 25 | $(CC) -o $@ $^ $(LDFLAGS) 26 | 27 | %.o: %.c 28 | $(CC) $(CFLAGS) -c $< 29 | 30 | clean: 31 | $(RM) *.o 32 | 33 | mrproper: clean 34 | $(RM) $(EXEC) 35 | -------------------------------------------------------------------------------- /tools/integrity-check/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = integrity-check 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | shadump: CFLAGS += -DSHADUMP 25 | shadump: LDFLAGS += -lcrypto 26 | shadump: $(EXEC) 27 | 28 | $(EXEC): $(OBJ) 29 | $(CC) -o $@ $^ $(LDFLAGS) 30 | 31 | %.o: %.c 32 | $(CC) $(CFLAGS) -c $< 33 | 34 | clean: 35 | $(RM) *.o 36 | 37 | mrproper: clean 38 | $(RM) $(EXEC) 39 | -------------------------------------------------------------------------------- /tools/integrity-check/integrity-check.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #ifdef SHADUMP 12 | #include 13 | #endif 14 | #include "libzdb.h" 15 | #include "data.h" 16 | 17 | #ifdef SHADUMP 18 | void sha256dump(char *input, size_t length) { 19 | unsigned char hash[SHA256_DIGEST_LENGTH]; 20 | SHA256_CTX sha256; 21 | 22 | SHA256_Init(&sha256); 23 | SHA256_Update(&sha256, input, length); 24 | SHA256_Final(hash, &sha256); 25 | 26 | hexdump(hash, SHA256_DIGEST_LENGTH); 27 | } 28 | #endif 29 | 30 | int data_integrity(data_root_t *zdbdata) { 31 | data_header_t *header; 32 | char datestr[64]; 33 | int errors = 0; 34 | 35 | // opening data file 36 | zdbdata->datafd = zdb_data_open_readonly(zdbdata); 37 | 38 | // validating header 39 | if(!(header = zdb_data_descriptor_load(zdbdata))) 40 | return 1; 41 | 42 | if(!zdb_data_descriptor_validate(header, zdbdata)) 43 | return 1; 44 | 45 | printf("[+] integrity-check: header seems correct\n"); 46 | printf("[+] integrity-check: created at: %s\n", zdb_header_date(header->created, datestr, sizeof(datestr))); 47 | printf("[+] integrity-check: last open: %s\n", zdb_header_date(header->opened, datestr, sizeof(datestr))); 48 | 49 | // now it's time to read each entries 50 | // each time, one entry starts by the entry-header 51 | // then entry payload. 52 | // the entry headers starts with the amount of bytes 53 | // of the key, which is needed to read the full header 54 | uint8_t idlength; 55 | data_entry_header_t *entry = NULL; 56 | size_t entrycount = 0; 57 | char *buffer = NULL; 58 | 59 | while(read(zdbdata->datafd, &idlength, sizeof(idlength)) == sizeof(idlength)) { 60 | // we have the length of the key 61 | ssize_t entrylength = sizeof(data_entry_header_t) + idlength; 62 | if(!(entry = realloc(entry, entrylength))) 63 | zdb_diep("realloc"); 64 | 65 | // rollback the 1 byte read for the id length 66 | off_t current = lseek(zdbdata->datafd, -1, SEEK_CUR); 67 | 68 | if(read(zdbdata->datafd, entry, entrylength) != entrylength) 69 | zdb_diep("data header read failed"); 70 | 71 | entrycount += 1; 72 | 73 | printf("[+] data entry: %lu, id length: %d\n", entrycount, entry->idlength); 74 | printf("[+] expected length: %u, current offset: %" PRId64 "\n", entry->datalength, (int64_t) current); 75 | printf("[+] previous offset: %u\n", entry->previous); 76 | printf("[+] expected crc : %08x\n", entry->integrity); 77 | printf("[+] entry flags : 0x%x\n", entry->flags); 78 | printf("[+] entry key : "); 79 | zdb_tools_hexdump(entry->id, entry->idlength); 80 | 81 | if(entry->datalength == 0) 82 | continue; 83 | 84 | if(!(buffer = realloc(buffer, entry->datalength))) 85 | zdb_diep("realloc"); 86 | 87 | if(read(zdbdata->datafd, buffer, entry->datalength) != entry->datalength) 88 | zdb_diep("payload entry read failed"); 89 | 90 | uint32_t crc = zdb_checksum_crc32((uint8_t *) buffer, entry->datalength); 91 | 92 | if(crc != entry->integrity) { 93 | fprintf(stderr, "[-] integrity check failed: %08x <> %08x\n", crc, entry->integrity); 94 | errors += 1; 95 | 96 | } else { 97 | printf("[+] data crc : ok, match\n"); 98 | } 99 | 100 | #ifdef SHADUMP 101 | printf("[+] data sha256 : "); 102 | sha256dump(buffer, entry->datalength); 103 | printf("\n"); 104 | #endif 105 | } 106 | 107 | free(entry); 108 | 109 | return errors; 110 | } 111 | 112 | int main(int argc, char *argv[]) { 113 | if(argc < 2) { 114 | fprintf(stderr, "Usage: %s data-filename\n", argv[0]); 115 | exit(EXIT_FAILURE); 116 | } 117 | 118 | char *filename = argv[1]; 119 | printf("[+] checking data integrity of: %s\n", filename); 120 | 121 | if(zdb_file_exists(filename) != ZDB_FILE_EXISTS) { 122 | fprintf(stderr, "[-] %s: invalid target file\n", filename); 123 | exit(EXIT_FAILURE); 124 | } 125 | 126 | // splitting directory and file from argument 127 | char *filedir = dirname(strdup(filename)); 128 | char *datafile = basename(filename); 129 | 130 | if(strncmp(datafile, "zdb-data-", 9)) { 131 | fprintf(stderr, "[-] integrity-check: data filename seems wrong\n"); 132 | exit(EXIT_FAILURE); 133 | } 134 | 135 | fileid_t dataid = atoi(datafile + 9); 136 | 137 | printf("[+] datafile path: %s\n", filedir); 138 | printf("[+] datafile name: %s\n", datafile); 139 | printf("[+] datafile id : %d\n", dataid); 140 | 141 | zdb_settings_t *zdb_settings = zdb_initialize(); 142 | zdb_id_set("integrity-checker"); 143 | 144 | // WARNING: we *don't* open the database, this would populate 145 | // memory, reading and loading everything... we just want 146 | // to initialize structs but not loads anything 147 | // 148 | // we will shortcut the database loading and directly 149 | // call the low-level data lazy loader (lazy loader won't 150 | // load anything except structs) 151 | // 152 | // zdb_open(zdb_settings); 153 | data_root_t *zdbdata; 154 | 155 | if(!(zdbdata = zdb_data_init_lazy(zdb_settings, filedir, dataid))) { 156 | fprintf(stderr, "[-] integrity-check: cannot load data\n"); 157 | exit(EXIT_FAILURE); 158 | } 159 | 160 | int val = 0; 161 | 162 | if((val = data_integrity(zdbdata))) 163 | fprintf(stderr, "[-] data file inconsistency: %d error(s)\n", val); 164 | 165 | return val; 166 | } 167 | 168 | -------------------------------------------------------------------------------- /tools/live-stats/zdb-live-stats.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import redis 3 | import time 4 | import pprint 5 | import curses 6 | 7 | class ZDBLiveStats: 8 | def __init__(self, host, port=9900): 9 | self.host = host 10 | self.port = port 11 | 12 | self.redis = redis.Redis(host, port) 13 | self.screen = curses.initscr() 14 | 15 | self.wininit() 16 | 17 | def window(self, title, y, x, height=9, width=32): 18 | win = curses.newwin(height, width, y, x) 19 | win.box() 20 | self.write(win, 0, " %s " % title) 21 | win.refresh() 22 | 23 | return win 24 | 25 | 26 | def wininit(self): 27 | self.screen.border(0) 28 | curses.curs_set(0) 29 | 30 | self.write(self.screen, 1, "0-db live monitor") 31 | self.write(self.screen, 3, "Host: %s [port: %d]" % (self.host, self.port)) 32 | 33 | self.screen.refresh() 34 | 35 | self.system = self.window("System", 5, 1, 7, 64) 36 | self.net = self.window("Network", 12, 1) 37 | self.index = self.window("Index", 12, 33) 38 | self.data = self.window("Data", 12, 65) 39 | 40 | def write(self, win, line, text): 41 | win.addstr(line, 2, text) 42 | 43 | def finish(self): 44 | curses.endwin() 45 | 46 | def diff(self, current, previous, key): 47 | return current[key] - previous[key] 48 | 49 | def kb(self, value): 50 | return value / 1024 51 | 52 | def size(self, value): 53 | units = ['KB', 'MB', 'GB', 'TB'] 54 | uid = 0 55 | value = value / 1024 56 | 57 | while value / 1024 > 1024: 58 | value = value / 1024 59 | uid += 1 60 | 61 | return "%10.2f %s" % (value, units[uid]) 62 | 63 | def uptime(self, value): 64 | if value < 600: 65 | return "%.0f minutes" % (value / 60) 66 | 67 | if value < 86400: 68 | return "%.1f hours" % (value / 3600) 69 | 70 | return "%d days, %.1f hours" % ((value / 86400), (value % 86400) / 3600) 71 | 72 | def stats(self): 73 | info = self.redis.info() 74 | # pprint.pprint(info) 75 | return info 76 | 77 | def live(self): 78 | previous = self.stats() 79 | 80 | while True: 81 | current = self.stats() 82 | 83 | stats = { 84 | 'data-read': self.diff(current, previous, 'data_disk_read_bytes'), 85 | 'data-write': self.diff(current, previous, 'data_disk_write_bytes'), 86 | 'index-read': self.diff(current, previous, 'index_disk_read_bytes'), 87 | 'index-write': self.diff(current, previous, 'index_disk_write_bytes'), 88 | 'net-rx': self.diff(current, previous, 'network_rx_bytes'), 89 | 'net-tx': self.diff(current, previous, 'network_tx_bytes'), 90 | 'commands': self.diff(current, previous, 'commands_executed'), 91 | 'clients': self.diff(current, previous, 'clients_lifetime'), 92 | } 93 | 94 | # pprint.pprint(stats) 95 | 96 | self.system.refresh() 97 | self.write(self.system, 2, " Uptime : %s" % self.uptime(current['uptime'])) 98 | self.write(self.system, 3, " Clients : %-10d [%d]" % (stats['clients'], current['clients_lifetime'])) 99 | self.write(self.system, 4, " Commands : %-10d [%d]" % (stats['commands'], current['commands_executed'])) 100 | self.system.refresh() 101 | 102 | self.write(self.net, 2, " Download %10.2f KB/s" % self.kb(stats['net-rx'])) 103 | self.write(self.net, 3, " %s" % self.size(current['network_rx_bytes'])) 104 | self.write(self.net, 5, " Upload %10.2f KB/s" % self.kb(stats['net-tx'])) 105 | self.write(self.net, 6, " %s" % self.size(current['network_tx_bytes'])) 106 | self.net.refresh() 107 | 108 | self.write(self.index, 2, " Read %10.2f KB/s" % self.kb(stats['index-read'])) 109 | self.write(self.index, 3, " %s" % self.size(current['index_disk_read_bytes'])) 110 | self.write(self.index, 5, " Write %10.2f KB/s" % self.kb(stats['index-write'])) 111 | self.write(self.index, 6, " %s" % self.size(current['index_disk_write_bytes'])) 112 | self.index.refresh() 113 | 114 | self.write(self.data, 2, " Read %10.2f KB/s" % self.kb(stats['data-read'])) 115 | self.write(self.data, 3, " %s" % self.size(current['data_disk_read_bytes'])) 116 | self.write(self.data, 5, " Write %10.2f KB/s" % self.kb(stats['data-write'])) 117 | self.write(self.data, 6, " %s" % self.size(current['data_disk_write_bytes'])) 118 | self.data.refresh() 119 | 120 | self.screen.refresh() 121 | 122 | previous = current 123 | time.sleep(1) 124 | 125 | return True 126 | 127 | if __name__ == '__main__': 128 | host = "localhost" 129 | port = 9900 130 | 131 | if len(sys.argv) > 1: 132 | host = sys.argv[1] 133 | 134 | if len(sys.argv) > 2: 135 | port = int(sys.argv[2]) 136 | 137 | zls = ZDBLiveStats(host, port) 138 | 139 | try: 140 | zls.live() 141 | 142 | finally: 143 | zls.finish() 144 | -------------------------------------------------------------------------------- /tools/namespace-dump/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = namespace-dump 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | json: CFLAGS += -DJSON_DUMP 25 | json: $(EXEC) 26 | 27 | $(EXEC): $(OBJ) 28 | $(CC) -o $@ $^ $(LDFLAGS) 29 | 30 | %.o: %.c 31 | $(CC) $(CFLAGS) -c $< 32 | 33 | clean: 34 | $(RM) *.o 35 | 36 | mrproper: clean 37 | $(RM) $(EXEC) 38 | -------------------------------------------------------------------------------- /tools/namespace-dump/namespace-dump.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "libzdb.h" 6 | 7 | int main(int argc, char *argv[]) { 8 | char *dirname = NULL; 9 | char *nsname = NULL; 10 | 11 | if(argc < 3) { 12 | fprintf(stderr, "Usage: %s index-rootpath namespace-name\n", argv[0]); 13 | exit(EXIT_FAILURE); 14 | } 15 | 16 | // loading zdb 17 | printf("[*] 0-db engine v%s\n", zdb_version()); 18 | 19 | // fetching directory path 20 | dirname = argv[1]; 21 | nsname = argv[2]; 22 | 23 | printf("[+] namespace-dump: loading [%s] from: %s\n", nsname, dirname); 24 | 25 | if(zdb_dir_exists(dirname) != ZDB_DIRECTORY_EXISTS) { 26 | fprintf(stderr, "[-] namespace-dump: could not reach index directory\n"); 27 | exit(EXIT_FAILURE); 28 | } 29 | 30 | // initializing database 31 | zdb_settings_t *zdb_settings = zdb_initialize(); 32 | zdb_settings->indexpath = dirname; 33 | 34 | zdb_id_set("namespace-dump"); 35 | 36 | // lazy load namespace 37 | ns_root_t *nsroot = namespaces_allocate(zdb_settings); 38 | namespace_t *ns = namespace_load_light(nsroot, nsname, 0); 39 | 40 | printf("[+] ----------------------------------------\n"); 41 | printf("[+] name : %s\n", ns->name); 42 | printf("[+] password : %s\n", ns->password ? ns->password : ""); 43 | printf("[+] public flag : %d\n", ns->public); 44 | printf("[+] mode worm : %d\n", ns->worm); 45 | printf("[+] maximum size: %lu bytes (%.2f MB)\n", ns->maxsize, MB(ns->maxsize)); 46 | printf("[+] -----------------------------------------\n"); 47 | 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /tools/namespace-editor/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = namespace-editor 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | $(EXEC): $(OBJ) 25 | $(CC) -o $@ $^ $(LDFLAGS) 26 | 27 | %.o: %.c 28 | $(CC) $(CFLAGS) -c $< 29 | 30 | clean: 31 | $(RM) *.o 32 | 33 | mrproper: clean 34 | $(RM) $(EXEC) 35 | -------------------------------------------------------------------------------- /tools/namespace-editor/namespace-editor.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "libzdb.h" 8 | 9 | static struct option long_options[] = { 10 | {"index", required_argument, 0, 'f'}, 11 | {"namespace", required_argument, 0, 'n'}, 12 | {"password", required_argument, 0, 'p'}, 13 | {"maxsize", required_argument, 0, 'm'}, 14 | {"public", required_argument, 0, 'P'}, 15 | {"worm" , required_argument, 0, 'w'}, 16 | {"help", no_argument, 0, 'h'}, 17 | {0, 0, 0, 0} 18 | }; 19 | 20 | int namespace_edit(char *dirname, char *nsname, namespace_t *namespace) { 21 | if(zdb_dir_exists(dirname) != ZDB_DIRECTORY_EXISTS) { 22 | fprintf(stderr, "[-] namespace-dump: could not reach index directory\n"); 23 | exit(EXIT_FAILURE); 24 | } 25 | 26 | // initializing database 27 | zdb_settings_t *zdb_settings = zdb_initialize(); 28 | zdb_settings->indexpath = dirname; 29 | 30 | zdb_id_set("namespace-editor"); 31 | 32 | // lazy load namespace 33 | ns_root_t *nsroot = namespaces_allocate(zdb_settings); 34 | namespace_t *ns = namespace_load_light(nsroot, nsname, 0); 35 | 36 | if(!ns) { 37 | fprintf(stderr, "[-] could not load namespace\n"); 38 | return 1; 39 | } 40 | 41 | printf("[+] ----------------------------------------\n"); 42 | printf("[+] current values ------------------\n"); 43 | printf("[+] ----------------------------------------\n"); 44 | printf("[+] name : %s\n", ns->name); 45 | printf("[+] password : %s\n", ns->password ? ns->password : ""); 46 | printf("[+] public flag : %d\n", ns->public); 47 | printf("[+] worm mode : %d\n", ns->worm); 48 | printf("[+] maximum size: %lu bytes (%.2f MB)\n", ns->maxsize, MB(ns->maxsize)); 49 | printf("[+] -----------------------------------------\n"); 50 | printf("[+]\n"); 51 | printf("[+] writing new values\n"); 52 | 53 | ns->name = nsname; 54 | ns->password = namespace->password; 55 | ns->public = namespace->public; 56 | ns->worm = namespace->worm; 57 | ns->maxsize = namespace->maxsize; 58 | 59 | // writing changes 60 | namespace_commit(ns); 61 | 62 | printf("[+]\n"); 63 | printf("[+] ----------------------------------------\n"); 64 | printf("[+] updated values ------------------\n"); 65 | printf("[+] ----------------------------------------\n"); 66 | printf("[+] name : %s\n", ns->name); 67 | printf("[+] password : %s\n", ns->password ? ns->password : ""); 68 | printf("[+] public flag : %d\n", ns->public); 69 | printf("[+] worm mode : %d\n", ns->worm); 70 | printf("[+] maximum size: %lu bytes (%.2f MB)\n", ns->maxsize, MB(ns->maxsize)); 71 | 72 | return 0; 73 | } 74 | 75 | void usage() { 76 | printf("Namespace editor tool arguments:\n\n"); 77 | 78 | printf(" --index index root directory path (required)\n"); 79 | printf(" --namespace name of the namespace (required)\n"); 80 | printf(" --password password (default, empty)\n"); 81 | printf(" --maxsize maximum size in bytes (default, no limit)\n"); 82 | printf(" --public does the namespace is public or not (yes or no, default yes)\n"); 83 | printf(" --worm does the namespace use WORM mode (yes or no, default no)\n"); 84 | printf(" --help print this message\n"); 85 | 86 | exit(EXIT_FAILURE); 87 | } 88 | 89 | int main(int argc, char *argv[]) { 90 | int option_index = 0; 91 | namespace_t namespace; 92 | char *dirname = NULL; 93 | char *nsname = NULL; 94 | 95 | // initializing everything to zero 96 | memset(&namespace, 0x00, sizeof(namespace_t)); 97 | 98 | // set namespace to be public by default 99 | namespace.public = 1; 100 | 101 | while(1) { 102 | // int i = getopt_long_only(argc, argv, "d:i:l:p:vxh", long_options, &option_index); 103 | int i = getopt_long_only(argc, argv, "", long_options, &option_index); 104 | 105 | if(i == -1) 106 | break; 107 | 108 | switch(i) { 109 | case 'f': 110 | dirname = optarg; 111 | break; 112 | 113 | case 'n': 114 | nsname = optarg; 115 | break; 116 | 117 | case 'p': 118 | namespace.password = optarg; 119 | break; 120 | 121 | case 'm': 122 | namespace.maxsize = atoll(optarg); 123 | break; 124 | 125 | case 'P': 126 | if(strcmp(optarg, "yes") == 0) { 127 | namespace.public = 1; 128 | 129 | } else if(strcmp(optarg, "no") == 0) { 130 | namespace.public = 0; 131 | 132 | } else { 133 | fprintf(stderr, "[-] invalid value for --public (yes or no expected)\n"); 134 | usage(); 135 | } 136 | 137 | break; 138 | 139 | case 'w': 140 | if(strcmp(optarg, "yes") == 0) { 141 | namespace.worm = 1; 142 | 143 | } else if(strcmp(optarg, "no") == 0) { 144 | namespace.worm = 0; 145 | 146 | } else { 147 | fprintf(stderr, "[-] invalid value for --worm (yes or no expected)\n"); 148 | usage(); 149 | } 150 | 151 | break; 152 | 153 | 154 | case 'h': 155 | usage(); 156 | break; 157 | 158 | case '?': 159 | default: 160 | exit(EXIT_FAILURE); 161 | } 162 | } 163 | 164 | if(!dirname) { 165 | fprintf(stderr, "[-] missing index root directory\n"); 166 | usage(); 167 | } 168 | 169 | if(!nsname) { 170 | fprintf(stderr, "[-] missing namespace name, you need to specify a name\n"); 171 | usage(); 172 | } 173 | 174 | printf("[+] zdb namespace editor\n"); 175 | 176 | return namespace_edit(dirname, nsname, &namespace); 177 | } 178 | -------------------------------------------------------------------------------- /tools/quick-compaction/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = quick-compact 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall -O2 -I../../libzdb 6 | LDFLAGS += ../../libzdb/libzdb.a -rdynamic 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | ifeq ($(COVERAGE),1) 15 | CFLAGS += -coverage -fprofile-arcs -ftest-coverage 16 | LDFLAGS += -lgcov --coverage 17 | endif 18 | 19 | all: $(EXEC) 20 | 21 | release: CFLAGS += -DRELEASE 22 | release: $(EXEC) 23 | 24 | $(EXEC): $(OBJ) 25 | $(CC) -o $@ $^ $(LDFLAGS) 26 | 27 | %.o: %.c 28 | $(CC) $(CFLAGS) -c $< 29 | 30 | clean: 31 | $(RM) *.o 32 | 33 | mrproper: clean 34 | $(RM) $(EXEC) 35 | -------------------------------------------------------------------------------- /utilities/README.md: -------------------------------------------------------------------------------- 1 | # 0-db Utilities 2 | Here comes some utilities you can use with 0-db 3 | 4 | ## Replications 5 | Simple replication is supported by 0-db. 6 | Theses utilities are not yet fully stable, but should works for basic setup and main use cases. 7 | They will be updated and improved like 0-db. 8 | 9 | Since how 0-db was thinked, the replication doesn't works like usual master-slave works. In addition, you have 10 | multiple replication choice. 11 | 12 | ### Namespace replication (db-replicate) 13 | One method to replicate your database is per namespace replication listener. You start a process which will 14 | monitor changes on a namespace in the 0-db master and replay new key changes in a slave 0-db database. 15 | 16 | This utility doesn't support deleting key it only watch and replicate `SET` key. 17 | 18 | To use this utility, you just have to run: 19 | ``` 20 | ./db-replicate --source-host/port master-host/port --remote-host/port slave-host/port --namespace namespace-to-watch 21 | ``` 22 | 23 | This is still a PoC and doesn't support password authentication, etc. for now. 24 | 25 | ### Full database replication (db-mirror) 26 | This utility is more advanced then `db-replicate` and provide an accurate replication between two full-set 0-db (all namespaces). 27 | 28 | All 0-db instance can accept a **kind-of slave clients** which will receive an exact copy of received commands, in order 29 | to replay them on a slave instance, with some extra flags to know on which namespace and timestamp to opperate. 30 | 31 | This comes with slower performance, since the master needs to send a copy of everything to all slaves, but there is no 32 | acknowledgment. If a slave is in a slow endpoint, connection speed will slow down master. 33 | 34 | To use the replication, you need administrator password/right: 35 | ``` 36 | ./db-mirror --source host[:port[,password]] --remote host[:port[,password]] 37 | ``` 38 | 39 | ### On-demand Namespace replication (db-sync) 40 | This works like `db-replicate` except if just do a single-shot replication of two namespace, and doesn't watch for changes. 41 | This is useful if you want to copy the namespace from one source to a destination 0-db. 42 | -------------------------------------------------------------------------------- /utilities/db-compacted/mkdb-quick.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import random 3 | import string 4 | 5 | class ZeroDBCompactQuick: 6 | def __init__(self, host, port): 7 | self.r = redis.Redis(host, port) 8 | self.namespace = "quick" 9 | 10 | def ex(self, command): 11 | print(command) 12 | 13 | val = self.r.execute_command(command) 14 | print(val) 15 | 16 | return val 17 | 18 | def generate(self, length): 19 | return ''.join(random.choice(string.ascii_uppercase) for _ in range(length)) 20 | 21 | def prepare(self): 22 | try: 23 | self.ex("NSNEW %s" % self.namespace) 24 | 25 | except redis.exceptions.ResponseError: 26 | self.ex("NSDEL %s" % self.namespace) 27 | self.ex("NSNEW %s" % self.namespace) 28 | 29 | self.ex("SELECT %s" % self.namespace) 30 | 31 | def run(self): 32 | self.prepare() 33 | 34 | # Fill datafile 00000 -> 00006 35 | for f in ['a', 'b', 'c', 'd', 'e', 'f', 'g']: 36 | for i in range(1, 128): 37 | self.ex("SET %c%d %s" % (f, i, self.generate(4096))) 38 | 39 | # Overwrite all keys on 00005 40 | for i in range(1, 128): 41 | self.ex("SET f%d %s" % (i, self.generate(64))) 42 | 43 | # Overwrite g1 and g6 44 | self.ex("SET g1 OVERWRITE-1") 45 | self.ex("SET g127 OVERWRITE-2") 46 | 47 | # Remove first and last entries 48 | self.ex("DEL a1") 49 | self.ex("DEL e127") 50 | 51 | # Remove full datafile 00002 52 | for i in range(1, 128): 53 | self.ex("DEL c%d" % i) 54 | 55 | # Remove some keys from 00003 56 | self.ex("DEL d1") 57 | self.ex("DEL d67") 58 | self.ex("DEL d94") 59 | self.ex("DEL d100") 60 | self.ex("DEL d108") 61 | 62 | 63 | if __name__ == '__main__': 64 | db = ZeroDBCompactQuick("localhost", 9900) 65 | db.run() 66 | -------------------------------------------------------------------------------- /utilities/db-compacted/mkdb-small.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import random 3 | import string 4 | 5 | """ 6 | Please run 0-db using '--datasize 512' in order to 7 | make this working correctly 8 | 9 | This test is made to fill database with well know edge and tricky 10 | case which needs to be handled correctly on compaction process 11 | and avoid using a huge database to try all of this 12 | 13 | Compaction is cleaning unused keys from datafile and index, 14 | but in order to keep everything clean, we just remove needed part 15 | from file, we don't reorganize files themself, this keep the 16 | "always append" idea true and improve incremental changes detection 17 | 18 | This is why even a datafile without any data left inside, 19 | should still exists after compaction, with just a header inside 20 | 21 | All keys used to flag other keys (for eg, deletion etc.) should disapears 22 | 23 | This generate a 7+ datafile db, filled with well known name 24 | Each key name is defined 'XY', X is a letter (from a to g) 25 | and Y a number, (from 1 to 6) 26 | 27 | The first file, only the first entry is removed 28 | The second file, nothing is done 29 | The third file, everything is removed 30 | The fourth file, some keys (1, 3, 4) are removed 31 | The fifth file, only the last key is removed 32 | 33 | The resulting compacted datafile should be: 34 | 00001: [a2, a3, a4, a5, a6] 35 | 00002: [b1, b2, b3, b4, b5, b6] 36 | 00003: [] # empty but should still exists 37 | 00004: [d2, d5, d6] 38 | 00005: [e1, e2, e3, e4, e5] 39 | 00006: [f1, f2, f3, f4, f5, f6] # with new values 40 | 00007: [g1, g2, g3, g4, g5, g6] # with new values for g1 and g6 41 | 0000*: should be empty, since flag should be discarded 42 | """ 43 | 44 | class ZeroDBCompactTest: 45 | def __init__(self, host, port): 46 | self.r = redis.Redis(host, port) 47 | 48 | def ex(self, command): 49 | print(command) 50 | 51 | val = self.r.execute_command(command) 52 | print(val) 53 | 54 | return val 55 | 56 | def generate(self, length): 57 | return ''.join(random.choice(string.ascii_uppercase) for _ in range(length)) 58 | 59 | def prepare(self): 60 | try: 61 | self.ex("NSNEW compactme") 62 | 63 | except redis.exceptions.ResponseError: 64 | self.ex("NSDEL compactme") 65 | self.ex("NSNEW compactme") 66 | 67 | self.ex("SELECT compactme") 68 | 69 | def run(self): 70 | self.prepare() 71 | 72 | # Fill datafile 00000 -> 00006 73 | for f in ['a', 'b', 'c', 'd', 'e', 'f', 'g']: 74 | for i in range(1, 7): 75 | self.ex("SET %c%d %s" % (f, i, self.generate(64))) 76 | 77 | # Overwrite all keys on 00005 78 | for i in range(1, 7): 79 | self.ex("SET f%d %s" % (i, self.generate(64))) 80 | 81 | # Overwrite g1 and g6 82 | self.ex("SET g1 OVERWRITE-1") 83 | self.ex("SET g6 OVERWRITE-2") 84 | 85 | # Remove first and last entries 86 | self.ex("DEL a1") 87 | self.ex("DEL e6") 88 | 89 | # Remove full datafile 00002 90 | for i in range(1, 7): 91 | self.ex("DEL c%d" % i) 92 | 93 | # Remove some keys from 00003 94 | self.ex("DEL d1") 95 | self.ex("DEL d3") 96 | self.ex("DEL d4") 97 | 98 | 99 | if __name__ == '__main__': 100 | db = ZeroDBCompactTest("localhost", 9900) 101 | db.run() 102 | -------------------------------------------------------------------------------- /utilities/db-mirror/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = db-mirror 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall 6 | LDFLAGS += -lhiredis 7 | 8 | all: $(EXEC) 9 | 10 | release: CFLAGS += -DRELEASE 11 | release: $(EXEC) 12 | 13 | $(EXEC): $(OBJ) 14 | $(CC) -o $@ $^ $(LDFLAGS) 15 | 16 | %.o: %.c 17 | $(CC) $(CFLAGS) -c $< 18 | 19 | clean: 20 | $(RM) *.o 21 | 22 | mrproper: clean 23 | $(RM) $(EXEC) 24 | -------------------------------------------------------------------------------- /utilities/db-mirror/db-mirror.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "db-mirror.h" 8 | 9 | static struct option long_options[] = { 10 | {"source", required_argument, 0, 's'}, 11 | {"remote", required_argument, 0, 'r'}, 12 | {"help", no_argument, 0, 'h'}, 13 | {0, 0, 0, 0} 14 | }; 15 | 16 | void diep(char *str) { 17 | perror(str); 18 | exit(EXIT_FAILURE); 19 | } 20 | 21 | static char __hex[] = "0123456789abcdef"; 22 | 23 | void hexdump(void *input, size_t length) { 24 | unsigned char *buffer = (unsigned char *) input; 25 | char *output = calloc((length * 2) + 1, 1); 26 | char *writer = output; 27 | 28 | for(unsigned int i = 0, j = 0; i < length; i++, j += 2) { 29 | *writer++ = __hex[(buffer[i] & 0xF0) >> 4]; 30 | *writer++ = __hex[buffer[i] & 0x0F]; 31 | } 32 | 33 | printf("0x%s", output); 34 | free(output); 35 | } 36 | 37 | redisContext *initialize(char *hostname, int port) { 38 | struct timeval timeout = {5, 0}; 39 | redisContext *context; 40 | 41 | printf("[+] connecting: %s, port: %d\n", hostname, port); 42 | 43 | if(!(context = redisConnectWithTimeout(hostname, port, timeout))) 44 | return NULL; 45 | 46 | if(context->err) { 47 | fprintf(stderr, "[-] redis error: %s\n", context->errstr); 48 | return NULL; 49 | } 50 | 51 | return context; 52 | } 53 | 54 | redisContext *mkhost(char *argument) { 55 | int port = 9900; 56 | char *hostname = NULL; 57 | char *password = NULL; 58 | char *match, *temp; 59 | 60 | // FIXME: memory leak on hostname 61 | 62 | // do we have a port specified 63 | if((match = strchr(argument, ':'))) { 64 | // does this port is before coma 65 | // otherwise the colon could be on 66 | // the password 67 | if((temp = strchr(argument, ','))) { 68 | // if coma is before colon, it's okay 69 | // it's a port specification 70 | if(temp > match) { 71 | port = atoi(match + 1); 72 | hostname = strndup(argument, match - argument); 73 | } 74 | 75 | } else { 76 | port = atoi(match + 1); 77 | hostname = strndup(argument, match - argument); 78 | } 79 | } 80 | 81 | if((match = strchr(argument, ','))) { 82 | password = match + 1; 83 | 84 | if(!hostname) 85 | hostname = strndup(argument, match - argument); 86 | } 87 | 88 | // nothing more than the hostname was specified 89 | if(!hostname) 90 | hostname = strdup(argument); 91 | 92 | // connect to the database 93 | redisContext *ctx; 94 | redisReply *auth; 95 | 96 | if(!(ctx = initialize(hostname, port))) 97 | return NULL; 98 | 99 | if(password) { 100 | if(!(auth = redisCommand(ctx, "AUTH %s", password))) { 101 | fprintf(stderr, "[-] %s:%d: could not send AUTH command to server\n", hostname, port); 102 | return NULL; 103 | } 104 | 105 | if(strcmp(auth->str, "OK")) { 106 | fprintf(stderr, "[-] %s:%d: could not authenticate: %s\n", hostname, port, auth->str); 107 | } 108 | 109 | freeReplyObject(auth); 110 | } 111 | 112 | printf("[+] sending MASTER request to database\n"); 113 | if(!(auth = redisCommand(ctx, "MASTER"))) 114 | return NULL; 115 | 116 | printf("[+] response: %s\n", auth->str); 117 | freeReplyObject(auth); 118 | 119 | printf("[+] %s:%d: database connected\n", hostname, port); 120 | return ctx; 121 | } 122 | 123 | redisContext *appendhost(sync_t *sync, char *argument) { 124 | int index = sync->remotes; 125 | sync->remotes += 1; 126 | 127 | if(!(sync->targets = realloc(sync->targets, sizeof(redisContext *) * sync->remotes))) 128 | diep("realloc"); 129 | 130 | sync->targets[index] = mkhost(argument); 131 | return sync->targets[index]; 132 | } 133 | 134 | void usage(char *program) { 135 | printf("%s: synchronize two 0-db database\n\n", program); 136 | 137 | printf("Available options:\n"); 138 | printf(" --source host[:port[,password]] host parameter for source database\n"); 139 | printf(" --remote host[:port[,password]] host parameter for target database\n\n"); 140 | printf(" --help this message (implemented)\n\n"); 141 | 142 | printf("Only one source can be provided and is required\n"); 143 | printf("Multiple target can be set, at least one is required\n"); 144 | } 145 | 146 | int main(int argc, char **argv) { 147 | int option_index = 0; 148 | sync_t sync = { 149 | .source = NULL, 150 | .remotes = 0, 151 | .targets = NULL, 152 | }; 153 | 154 | while(1) { 155 | int i = getopt_long_only(argc, argv, "", long_options, &option_index); 156 | 157 | if(i == -1) 158 | break; 159 | 160 | switch(i) { 161 | case 's': 162 | if(sync.source) { 163 | fprintf(stderr, "[-] multiple source provided\n"); 164 | exit(EXIT_FAILURE); 165 | } 166 | 167 | sync.source = mkhost(optarg); 168 | break; 169 | 170 | case 'r': 171 | appendhost(&sync, optarg); 172 | break; 173 | 174 | case 'h': 175 | usage(argv[0]); 176 | exit(EXIT_FAILURE); 177 | 178 | case '?': 179 | default: 180 | exit(EXIT_FAILURE); 181 | } 182 | } 183 | 184 | if(!sync.source) { 185 | fprintf(stderr, "[-] missing source host\n"); 186 | exit(EXIT_FAILURE); 187 | } 188 | 189 | if(sync.remotes == 0) { 190 | fprintf(stderr, "[-] missing at least one target host\n"); 191 | exit(EXIT_FAILURE); 192 | } 193 | 194 | int value = 0; 195 | 196 | // initial replication 197 | replicate(&sync); 198 | 199 | // int value = mirror(&sync); 200 | 201 | 202 | // redisFree(sync.sourcep); 203 | // redisFree(sync.source); 204 | // redisFree(sync.target); 205 | 206 | return value; 207 | } 208 | -------------------------------------------------------------------------------- /utilities/db-mirror/db-mirror.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_DB_MIRROR 2 | #define ZDB_DB_MIRROR 3 | 4 | #define MB(x) (x / (1024 * 1024.0)) 5 | 6 | // 7 | // main database context 8 | // 9 | typedef struct sync_t { 10 | redisContext *source; // source payload 11 | 12 | unsigned int remotes; 13 | redisContext **targets; 14 | 15 | } sync_t; 16 | 17 | // 18 | // payload stats 19 | // 20 | typedef struct status_t { 21 | size_t size; // total in bytes, to transfert 22 | size_t transfered; // total in bytes transfered 23 | 24 | size_t keys; // amount of keys to transfert 25 | size_t copied; // amount of keys transfered 26 | size_t requested; // amount of keys requested 27 | 28 | } status_t; 29 | 30 | // 31 | // namespace context 32 | // 33 | typedef struct namespace_t { 34 | char *name; 35 | int public; 36 | char *password; 37 | int limit; 38 | int mode; 39 | status_t status; 40 | 41 | } namespace_t; 42 | 43 | typedef struct namespaces_t { 44 | namespace_t *list; 45 | unsigned int length; 46 | 47 | } namespaces_t; 48 | 49 | 50 | // 51 | // interface 52 | // 53 | typedef struct keylist_t { 54 | size_t length; 55 | size_t allocated; 56 | size_t size; 57 | redisReply **keys; 58 | 59 | } keylist_t; 60 | 61 | int replicate(sync_t *sync); 62 | void diep(char *str); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /utilities/db-mirror/db-prepare.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "db-mirror.h" 8 | 9 | namespace_t *warmup(sync_t *sync, namespace_t *namespace) { 10 | redisReply *reply; 11 | char *match, *temp; 12 | 13 | printf("[+] fetching namespace information: %s\n", namespace->name); 14 | 15 | if(!(reply = redisCommand(sync->source, "NSINFO %s 0", namespace->name))) 16 | return NULL; 17 | 18 | if((match = strstr(reply->str, "mode: seq"))) { 19 | printf("[-] namespace running in sequential mode, replication not supported\n"); 20 | exit(EXIT_FAILURE); 21 | } 22 | 23 | if(!(match = strstr(reply->str, "data_size_bytes:"))) 24 | return NULL; 25 | 26 | namespace->status.size = atol(match + 17); 27 | 28 | if(!(match = strstr(reply->str, "entries:"))) 29 | return NULL; 30 | 31 | namespace->status.keys = atoi(match + 9); 32 | 33 | namespace->public = strstr(reply->str, "public: yes") ? 1 : 0; 34 | 35 | if(strstr(reply->str, "password: yes")) { 36 | if(!(match = strstr(reply->str, "password_raw: "))) 37 | return NULL; 38 | 39 | match += 14; 40 | temp = strchr(match, '\n'); 41 | 42 | if(!(namespace->password = strndup(match, temp - match))) 43 | diep("strdup"); 44 | } 45 | 46 | printf("[+] size: %.2f MB\n", MB(namespace->status.size)); 47 | printf("[+] keys: %lu\n", namespace->status.keys); 48 | printf("[+] pass: %s\n", namespace->password); 49 | 50 | return namespace; 51 | } 52 | 53 | size_t keylist_append(keylist_t *keylist, redisReply *reply, size_t size) { 54 | if(keylist->allocated < keylist->length + 1) { 55 | size_t newsize = sizeof(redisReply *) * (keylist->allocated + 128); 56 | 57 | if(!(keylist->keys = (redisReply **) realloc(keylist->keys, newsize))) { 58 | perror("malloc"); 59 | exit(EXIT_FAILURE); 60 | } 61 | 62 | keylist->allocated += 128; 63 | } 64 | 65 | keylist->length += 1; 66 | keylist->size += size; 67 | keylist->keys[keylist->length - 1] = reply; 68 | 69 | return keylist->length; 70 | } 71 | 72 | 73 | int fetchsync(sync_t *sync, keylist_t *keylist, status_t *status) { 74 | redisReply *input, *output; 75 | 76 | for(size_t i = 0; i < keylist->length; i++) { 77 | if(redisGetReply(sync->source, (void **) &input) == REDIS_ERR) { 78 | fprintf(stderr, "\n[-] %s\n", sync->source->errstr); 79 | exit(EXIT_FAILURE); 80 | } 81 | 82 | char *key = keylist->keys[i]->element[0]->str; 83 | size_t keylen = keylist->keys[i]->element[0]->len; 84 | 85 | if(!(output = redisCommand(sync->targets[0], "SET %b %b", key, keylen, input->str, input->len))) 86 | return 0; 87 | 88 | status->transfered += input->len; 89 | status->copied += 1; 90 | 91 | float percent = (status->transfered / (double) status->size) * 100.0; 92 | printf("\r[+] syncing: % 3.1f %% [%lu/%lu keys, %.2f MB]", percent, status->requested, status->copied, MB(status->transfered)); 93 | fflush(stdout); 94 | 95 | freeReplyObject(input); 96 | freeReplyObject(output); 97 | freeReplyObject(keylist->keys[i]); 98 | } 99 | 100 | return 0; 101 | } 102 | 103 | int replicate_from_reply(sync_t *sync, redisReply *from, status_t *status) { 104 | redisReply *reply = from; 105 | keylist_t keylist = { 106 | .length = 0, 107 | .keys = NULL, 108 | .size = 0, 109 | .allocated = 0, 110 | }; 111 | 112 | while(reply && reply->type == REDIS_REPLY_ARRAY) { 113 | float percent = (status->transfered / (double) status->size) * 100.0; 114 | 115 | printf("\r[+] syncing: % 3.1f %% [%lu/%lu keys, %.2f MB]", percent, status->requested, status->copied, MB(status->transfered)); 116 | fflush(stdout); 117 | 118 | for(size_t i = 0; i < reply->element[1]->elements; i++) { 119 | redisReply *item = reply->element[1]->element[i]; 120 | 121 | // append the get in the buffer 122 | redisAppendCommand(sync->source, "GET %b", item->element[0]->str, item->element[0]->len); 123 | keylist_append(&keylist, item, item->element[1]->integer); 124 | 125 | 126 | // one more key requested 127 | status->requested += 1; 128 | } 129 | 130 | // query next key 131 | if(!(reply = redisCommand(sync->source, "SCAN %b", reply->element[0]->str, reply->element[0]->len))) 132 | return 1; 133 | 134 | // original reply will be free'd here 135 | // that's why new reply is already prepared ^ 136 | fetchsync(sync, &keylist, status); 137 | keylist.length = 0; 138 | keylist.size = 0; 139 | } 140 | 141 | return 0; 142 | } 143 | 144 | int replicate(sync_t *sync) { 145 | redisReply *reply; 146 | namespaces_t namespaces = { 147 | .length = 0, 148 | .list = NULL 149 | }; 150 | 151 | printf("[+] preparing buffers\n"); 152 | 153 | // 154 | // fetching list of namespaces 155 | // 156 | printf("[+] preparing namespaces\n"); 157 | if(!(reply = redisCommand(sync->source, "NSLIST 0"))) 158 | return 1; 159 | 160 | namespaces.length = reply->elements; 161 | if(!(namespaces.list = (namespace_t *) calloc(sizeof(namespace_t), namespaces.length))) 162 | diep("calloc"); 163 | 164 | for(unsigned int i = 0; i < reply->elements; i++) { 165 | if(!(namespaces.list[i].name = strdup(reply->element[i]->str))) 166 | diep("strdup"); 167 | } 168 | 169 | freeReplyObject(reply); 170 | 171 | // 172 | // fetching namespaces informations 173 | // 174 | for(unsigned int i = 0; i < namespaces.length; i++) { 175 | namespace_t *namespace = &namespaces.list[i]; 176 | if(!(warmup(sync, namespace))) { 177 | fprintf(stderr, "[-] could not fetch namespace information\n"); 178 | exit(EXIT_FAILURE); 179 | } 180 | } 181 | 182 | exit(1); 183 | 184 | // printf("[+] namespace ready, %lu keys to transfert (%.2f MB)\n", status.keys, MB(status.size)); 185 | 186 | if(!(reply = redisCommand(sync->source, "SCAN"))) 187 | return 1; 188 | 189 | // int value = replicate_from_reply(sync, reply, &status); 190 | 191 | printf("\n[+] initial synchronization done\n"); 192 | return 0; 193 | } 194 | 195 | -------------------------------------------------------------------------------- /utilities/db-replicate/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = db-replicate 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall 6 | LDFLAGS += -lhiredis 7 | 8 | all: $(EXEC) 9 | 10 | release: CFLAGS += -DRELEASE 11 | release: $(EXEC) 12 | 13 | $(EXEC): $(OBJ) 14 | $(CC) -o $@ $^ $(LDFLAGS) 15 | 16 | %.o: %.c 17 | $(CC) $(CFLAGS) -c $< 18 | 19 | clean: 20 | $(RM) *.o 21 | 22 | mrproper: clean 23 | $(RM) $(EXEC) 24 | -------------------------------------------------------------------------------- /utilities/db-sync/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = db-sync 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -W -Wall 6 | LDFLAGS += -lhiredis 7 | 8 | all: $(EXEC) 9 | 10 | release: CFLAGS += -DRELEASE 11 | release: $(EXEC) 12 | 13 | $(EXEC): $(OBJ) 14 | $(CC) -o $@ $^ $(LDFLAGS) 15 | 16 | %.o: %.c 17 | $(CC) $(CFLAGS) -c $< 18 | 19 | clean: 20 | $(RM) *.o 21 | 22 | mrproper: clean 23 | $(RM) $(EXEC) 24 | -------------------------------------------------------------------------------- /zdbd/Makefile: -------------------------------------------------------------------------------- 1 | EXEC = zdb 2 | SRC = $(wildcard *.c) 3 | OBJ = $(SRC:.c=.o) 4 | 5 | CFLAGS += -g -std=gnu99 -O0 -W -Wall -Wextra -Wno-implicit-fallthrough -I../libzdb 6 | LDFLAGS += -rdynamic ../libzdb/libzdb.a 7 | 8 | MACHINE := $(shell uname -m) 9 | ifeq ($(MACHINE),x86_64) 10 | # explicit sse4.2 flags 11 | CFLAGS += -msse4.2 12 | endif 13 | 14 | # grab version from git, if possible 15 | REVISION := $(shell git describe --abbrev=8 --dirty --always --tags) 16 | ifeq ($(REVISION),) 17 | REVISION := $(shell grep ZDBD_VERSION zdbd.h | awk '{ print $$3 }' | sed s/'"'//g) 18 | endif 19 | 20 | CFLAGS += -DZDBD_REVISION=\"$(REVISION)\" 21 | 22 | ifeq ($(STATIC),1) 23 | LDFLAGS += -static 24 | endif 25 | 26 | ifeq ($(COVERAGE),1) 27 | CFLAGS += -pg -coverage -fprofile-arcs -ftest-coverage 28 | LDFLAGS += -lgcov --coverage 29 | endif 30 | 31 | ifeq ($(PROFILE),1) 32 | CFLAGS += -pg 33 | LDFLAGS += -pg 34 | endif 35 | 36 | ifeq ($(PREFIX),) 37 | PREFIX := /usr/local 38 | endif 39 | 40 | all: $(EXEC) 41 | 42 | release: CFLAGS += -DRELEASE -O2 43 | release: clean $(EXEC) 44 | 45 | $(EXEC): $(OBJ) 46 | $(CC) -o $@ $^ $(LDFLAGS) 47 | 48 | %.o: %.c 49 | $(CC) $(CFLAGS) -c $< 50 | 51 | clean: 52 | $(RM) *.o 53 | 54 | mrproper: clean 55 | $(RM) $(EXEC) 56 | $(RM) *.gcno *.gcda *.gcov 57 | 58 | install: $(EXEC) 59 | install -d $(DESTDIR)$(PREFIX)/bin 60 | install -m 755 $(EXEC) $(DESTDIR)$(PREFIX)/bin/ 61 | -------------------------------------------------------------------------------- /zdbd/auth.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include "libzdb.h" 6 | #include "zdbd.h" 7 | 8 | // check if user input (buffer with buffer length) 9 | // does match with expected password string, unified way 10 | int zdbd_password_check(char *input, int length, char *expected) { 11 | char password[192]; 12 | sprintf(password, "%.*s", length, input); 13 | 14 | if(strcmp(password, expected) == 0) { 15 | zdbd_debug("[+] password: access granted\n"); 16 | return 1; 17 | } 18 | 19 | zdbd_debug("[-] password: wrong password\n"); 20 | return 0; 21 | } 22 | 23 | 24 | -------------------------------------------------------------------------------- /zdbd/auth.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDBD_AUTH_H 2 | #define ZDBD_AUTH_H 3 | 4 | int zdbd_password_check(char *input, int length, char *expected); 5 | #endif 6 | -------------------------------------------------------------------------------- /zdbd/commands.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_H 2 | #define ZDB_COMMANDS_H 3 | 4 | #define COMMAND_MAXLEN 256 5 | 6 | int redis_dispatcher(redis_client_t *client); 7 | 8 | int command_args_validate(redis_client_t *client, int expected); 9 | int command_args_validate_min(redis_client_t *client, int expected); 10 | int command_args_validate_null(redis_client_t *client, int expected); 11 | 12 | // avoid too long argument 13 | int command_args_overflow(redis_client_t *client, int argidx, int maxlen); 14 | 15 | // validate a argument as namespace 16 | int command_args_namespace(redis_client_t *client, int argidx); 17 | 18 | int command_admin_authorized(redis_client_t *client); 19 | int command_wait(redis_client_t *client); 20 | int command_asterisk(redis_client_t *client); 21 | 22 | int command_error_locked(redis_client_t *client); 23 | int command_error_frozen(redis_client_t *client); 24 | 25 | // defined in commands_set.c 26 | // used by commands_set.c and commands_dataset.c 27 | time_t timestamp_from_set(resp_request_t *request, int field); 28 | #endif 29 | -------------------------------------------------------------------------------- /zdbd/commands_auth.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "libzdb.h" 13 | #include "zdbd.h" 14 | #include "redis.h" 15 | #include "commands.h" 16 | #include "auth.h" 17 | 18 | // AUTH SECURE CHALLENGE 19 | static int command_auth_challenge(redis_client_t *client) { 20 | char *string; 21 | 22 | if(!(string = zdb_challenge())) { 23 | redis_hardsend(client, "-Internal generator error"); 24 | return 1; 25 | } 26 | 27 | // free (possible) previously allocated nonce 28 | free(client->nonce); 29 | 30 | // set new challenge 31 | client->nonce = string; 32 | 33 | // send challenge to client 34 | char response[32]; 35 | sprintf(response, "+%s\r\n", client->nonce); 36 | 37 | redis_reply_stack(client, response, strlen(response)); 38 | 39 | return 0; 40 | } 41 | 42 | // AUTH 43 | static int command_auth_regular(redis_client_t *client) { 44 | resp_request_t *request = client->request; 45 | 46 | zdbd_debug("[+] auth: regular authentication requested\n"); 47 | 48 | // authentication not enabled, nothing to do 49 | if(!zdbd_rootsettings.adminpwd) { 50 | redis_hardsend(client, "-Authentication disabled"); 51 | return 0; 52 | } 53 | 54 | // generic args validation 55 | if(!command_args_validate(client, 2)) 56 | return 1; 57 | 58 | if(request->argv[1]->length > 128) { 59 | redis_hardsend(client, "-Password too long"); 60 | return 1; 61 | } 62 | 63 | resp_object_t *user = request->argv[1]; 64 | 65 | if(zdbd_password_check(user->buffer, user->length, zdbd_rootsettings.adminpwd)) { 66 | zdbd_debug("[+] auth: regular authentication granted\n"); 67 | 68 | client->admin = 1; 69 | redis_hardsend(client, "+OK"); 70 | return 0; 71 | 72 | } 73 | 74 | redis_hardsend(client, "-Access denied"); 75 | return 1; 76 | } 77 | 78 | // AUTH SECURE 79 | // AUTH SECURE CHALLENGE 80 | static int command_auth_secure(redis_client_t *client) { 81 | resp_request_t *request = client->request; 82 | 83 | zdbd_debug("[+] auth: secure authentication requested\n"); 84 | 85 | // generic args validation 86 | if(!command_args_validate(client, 3)) 87 | return 1; 88 | 89 | // only accept 90 | if(strncasecmp(request->argv[1]->buffer, "SECURE", request->argv[1]->length) != 0) { 91 | redis_hardsend(client, "-Only SECURE extra method supported"); 92 | return 1; 93 | } 94 | 95 | // check if this is not a CHALLENGE request 96 | if(strncasecmp(request->argv[2]->buffer, "CHALLENGE", request->argv[2]->length) == 0) { 97 | zdbd_debug("[+] auth: challenge requested\n"); 98 | return command_auth_challenge(client); 99 | } 100 | 101 | // only process authentication if there are any authentication 102 | // we don't test before, we need to be able to request a CHALLENGE 103 | // for other authentication (like SELECT) 104 | if(!zdbd_rootsettings.adminpwd) { 105 | redis_hardsend(client, "-Authentication disabled"); 106 | return 0; 107 | } 108 | 109 | // only accept client which previously requested nonce challenge 110 | if(client->nonce == NULL) { 111 | redis_hardsend(client, "-No CHALLENGE requested, secure authentication not available"); 112 | return 1; 113 | } 114 | 115 | zdbd_debug("[+] auth: challenge: %s\n", client->nonce); 116 | 117 | // expecting sha1 hexa-string input 118 | if(request->argv[2]->length != ZDB_SHA1_DIGEST_STR_LENGTH) { 119 | redis_hardsend(client, "-Invalid hash length"); 120 | return 1; 121 | } 122 | 123 | char *expected; 124 | 125 | if(!(expected = zdb_hash_password(client->nonce, zdbd_rootsettings.adminpwd))) { 126 | redis_hardsend(client, "-Internal generator error"); 127 | return 1; 128 | } 129 | 130 | zdbd_debug("[+] auth: expected hash: %s\n", expected); 131 | 132 | resp_object_t *user = request->argv[2]; 133 | int granted = 0; 134 | 135 | if(zdbd_password_check(user->buffer, user->length, expected)) { 136 | zdbd_debug("[+] auth: secure authentication granted\n"); 137 | 138 | // flag user as authenticated 139 | client->admin = 1; 140 | granted = 1; 141 | 142 | redis_hardsend(client, "+OK"); 143 | 144 | } else { 145 | // wrong password 146 | redis_hardsend(client, "-Access denied"); 147 | } 148 | 149 | // free and reset nonce 150 | free(expected); 151 | free(client->nonce); 152 | client->nonce = NULL; 153 | 154 | return granted; 155 | } 156 | 157 | int command_auth(redis_client_t *client) { 158 | // AUTH 159 | if(client->request->argc == 2) 160 | return command_auth_regular(client); 161 | 162 | // AUTH SECURE 163 | // AUTH SECURE CHALLENGE 164 | if(client->request->argc == 3) 165 | return command_auth_secure(client); 166 | 167 | // invalid arguments 168 | redis_hardsend(client, "-Unexpected arguments"); 169 | return 1; 170 | } 171 | 172 | -------------------------------------------------------------------------------- /zdbd/commands_auth.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_AUTH_H 2 | #define ZDB_COMMANDS_AUTH_H 3 | 4 | int command_auth(redis_client_t *client); 5 | #endif 6 | -------------------------------------------------------------------------------- /zdbd/commands_dataset.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_DATASET_H 2 | #define ZDB_COMMANDS_DATASET_H 3 | 4 | int command_exists(redis_client_t *client); 5 | int command_check(redis_client_t *client); 6 | int command_del(redis_client_t *client); 7 | int command_length(redis_client_t *client); 8 | int command_keytime(redis_client_t *client); 9 | #endif 10 | -------------------------------------------------------------------------------- /zdbd/commands_get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "libzdb.h" 10 | #include "index.h" 11 | #include "zdbd.h" 12 | #include "redis.h" 13 | #include "commands.h" 14 | 15 | static int command_get_single(redis_client_t *client, char *buffer, int length) { 16 | index_entry_t *entry = NULL; 17 | 18 | // fetching index entry for this key 19 | if(!(entry = index_get(client->ns->index, buffer, length))) { 20 | zdbd_debug("[-] command: get: key not found\n"); 21 | redis_hardsend(client, "$-1"); 22 | return 1; 23 | } 24 | 25 | // key found but deleted 26 | if(entry->flags & INDEX_ENTRY_DELETED) { 27 | zdbd_verbose("[-] command: get: key deleted\n"); 28 | redis_hardsend(client, "$-1"); 29 | return 1; 30 | } 31 | 32 | // key found and valid, let's check the contents 33 | zdbd_debug("[+] command: get: entry found, flags: %x, data length: %" PRIu32 "\n", entry->flags, entry->length); 34 | zdbd_debug("[+] command: get: data file: %d, data offset: %" PRIu32 "\n", entry->dataid, entry->offset); 35 | 36 | data_root_t *data = client->ns->data; 37 | data_payload_t payload = data_get(data, entry->offset, entry->length, entry->dataid, entry->idlength); 38 | 39 | if(!payload.buffer) { 40 | zdb_log("[-] command: get: cannot read payload\n"); 41 | redis_hardsend(client, "-Internal Error"); 42 | free(payload.buffer); 43 | return 0; 44 | } 45 | 46 | redis_bulk_t response = redis_bulk(payload.buffer, payload.length); 47 | if(!response.buffer) { 48 | redis_hardsend(client, "$-1"); 49 | return 0; 50 | } 51 | 52 | redis_reply_heap(client, response.buffer, response.length, free); 53 | free(payload.buffer); 54 | 55 | return 0; 56 | 57 | } 58 | 59 | int command_get(redis_client_t *client) { 60 | resp_request_t *request = client->request; 61 | 62 | if(!command_args_validate(client, 2)) 63 | return 1; 64 | 65 | if(request->argv[1]->length > MAX_KEY_LENGTH) { 66 | zdbd_debug("[-] command: get: invalid key size (too big)\n"); 67 | redis_hardsend(client, "-Invalid key"); 68 | return 1; 69 | } 70 | 71 | if(namespace_is_frozen(client->ns)) 72 | return command_error_frozen(client); 73 | 74 | return command_get_single(client, request->argv[1]->buffer, request->argv[1]->length); 75 | } 76 | 77 | int command_mget(redis_client_t *client) { 78 | resp_request_t *request = client->request; 79 | 80 | if(client->request->argc < 2) { 81 | redis_hardsend(client, "-Invalid arguments"); 82 | return 1; 83 | } 84 | 85 | // validating each keys 86 | for(int i = 1; i < client->request->argc; i++) { 87 | if(request->argv[i]->length > MAX_KEY_LENGTH) { 88 | zdbd_debug("[-] command: get: invalid key %d size (too big)\n", i); 89 | redis_hardsend(client, "-Invalid key"); 90 | return 1; 91 | } 92 | } 93 | 94 | if(namespace_is_frozen(client->ns)) 95 | return command_error_frozen(client); 96 | 97 | // streaming response to client 98 | char line[512]; 99 | int length = client->request->argc - 1; 100 | 101 | sprintf(line, "*%d\r\n", length); 102 | redis_reply_stack(client, line, strlen(line)); 103 | 104 | zdbd_debug("[+] command: mget: sending %d responses\n", length); 105 | 106 | for(int i = 1; i < request->argc; i++) { 107 | command_get_single(client, request->argv[i]->buffer, request->argv[i]->length); 108 | } 109 | 110 | return 0; 111 | } 112 | 113 | -------------------------------------------------------------------------------- /zdbd/commands_get.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_GET_H 2 | #define ZDB_COMMANDS_GET_H 3 | 4 | int command_get(redis_client_t *client); 5 | int command_mget(redis_client_t *client); 6 | #endif 7 | -------------------------------------------------------------------------------- /zdbd/commands_history.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_HISTORY_H 2 | #define ZDB_COMMANDS_HISTORY_H 3 | 4 | int command_history(redis_client_t *client); 5 | #endif 6 | -------------------------------------------------------------------------------- /zdbd/commands_namespace.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_NAMESPACE_H 2 | #define ZDB_COMMANDS_NAMESPACE_H 3 | 4 | int command_nsnew(redis_client_t *client); 5 | int command_nsdel(redis_client_t *client); 6 | int command_select(redis_client_t *client); 7 | int command_nslist(redis_client_t *client); 8 | int command_nsinfo(redis_client_t *client); 9 | int command_nsset(redis_client_t *client); 10 | int command_nsjump(redis_client_t *client); 11 | int command_dbsize(redis_client_t *client); 12 | int command_reload(redis_client_t *client); 13 | int command_flush(redis_client_t *client); 14 | #endif 15 | -------------------------------------------------------------------------------- /zdbd/commands_replicate.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "libzdb.h" 10 | #include "zdbd.h" 11 | #include "redis.h" 12 | #include "commands.h" 13 | #include "commands_replicate.h" 14 | 15 | int command_mirror(redis_client_t *client) { 16 | if(!command_admin_authorized(client)) 17 | return 1; 18 | 19 | client->mirror = 1; 20 | redis_hardsend(client, "+Starting mirroring"); 21 | 22 | return 0; 23 | } 24 | 25 | int command_master(redis_client_t *client) { 26 | if(!command_admin_authorized(client)) 27 | return 1; 28 | 29 | client->master = 1; 30 | redis_hardsend(client, "+Hello, master"); 31 | 32 | return 0; 33 | } 34 | 35 | -------------------------------------------------------------------------------- /zdbd/commands_replicate.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_MIRROR_H 2 | #define ZDB_COMMANDS_MIRROR_H 3 | 4 | int command_mirror(redis_client_t *client); 5 | int command_master(redis_client_t *client); 6 | #endif 7 | -------------------------------------------------------------------------------- /zdbd/commands_scan.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_SCAN_H 2 | #define ZDB_COMMANDS_SCAN_H 3 | 4 | int command_scan(redis_client_t *client); 5 | int command_rscan(redis_client_t *client); 6 | int command_keycur(redis_client_t *client); 7 | int command_kscan(redis_client_t *client); 8 | 9 | typedef struct scan_info_t { 10 | fileid_t dataid; 11 | fileid_t idxid; 12 | size_t idxoffset; 13 | 14 | } scan_info_t; 15 | 16 | typedef struct scan_list_t { 17 | size_t length; 18 | size_t allocated; 19 | index_item_t **items; 20 | scan_info_t *scansinfo; 21 | 22 | } scan_list_t; 23 | 24 | typedef struct list_t { 25 | void **items; 26 | size_t length; 27 | size_t allocated; 28 | 29 | } list_t; 30 | 31 | // one call to SCAN/RSCAN can take up to 32 | // 2000 microseconds (2 milliseconds) 33 | #define SCAN_TIMESLICE_US 2000 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /zdbd/commands_set.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_SET_H 2 | #define ZDB_COMMANDS_SET_H 3 | 4 | int command_set(redis_client_t *client); 5 | #endif 6 | -------------------------------------------------------------------------------- /zdbd/commands_system.h: -------------------------------------------------------------------------------- 1 | #ifndef ZDB_COMMANDS_SYSTEM_H 2 | #define ZDB_COMMANDS_SYSTEM_H 3 | 4 | int command_ping(redis_client_t *client); 5 | int command_time(redis_client_t *client); 6 | int command_auth(redis_client_t *client); 7 | int command_stop(redis_client_t *client); 8 | int command_info(redis_client_t *client); 9 | 10 | int command_hooks(redis_client_t *client); 11 | int command_index(redis_client_t *client); 12 | int command_data(redis_client_t *client); 13 | #endif 14 | -------------------------------------------------------------------------------- /zdbd/socket_epoll.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | // this implementation is only used on linux 10 | #ifdef __linux__ 11 | 12 | #include 13 | #include "libzdb.h" 14 | #include "zdbd.h" 15 | #include "redis.h" 16 | 17 | #define MAXEVENTS 64 18 | #define EVTIMEOUT 200 19 | 20 | static int socket_client_accept(redis_handler_t *redis, int fd) { 21 | int clientfd; 22 | 23 | if((clientfd = accept(fd, NULL, NULL)) == -1) { 24 | zdbd_verbosep("socket_event", "accept"); 25 | return 0; 26 | } 27 | 28 | socket_nonblock(clientfd); 29 | socket_keepalive(clientfd); 30 | socket_client_new(clientfd); 31 | 32 | zdbd_verbose("[+] incoming connection (socket %d)\n", clientfd); 33 | 34 | // add client to the epoll list 35 | struct epoll_event event; 36 | 37 | memset(&event, 0, sizeof(struct epoll_event)); 38 | 39 | event.data.fd = clientfd; 40 | event.events = EPOLLIN | EPOLLOUT | EPOLLET; 41 | 42 | // we use edge-level because of how the 43 | // upload works (need to be notified when client 44 | // is ready to receive data, only one time) 45 | 46 | if(epoll_ctl(redis->evfd, EPOLL_CTL_ADD, clientfd, &event) < 0) { 47 | zdbd_verbosep("socket_event", "epoll_ctl"); 48 | return 0; 49 | } 50 | 51 | return 1; 52 | } 53 | 54 | static int socket_event(struct epoll_event *events, int notified, redis_handler_t *redis) { 55 | struct epoll_event *ev; 56 | 57 | for(int i = 0; i < notified; i++) { 58 | int newclient = 0; 59 | ev = events + i; 60 | 61 | // epoll issue 62 | // discard this client 63 | if((ev->events & EPOLLERR) || (ev->events & EPOLLHUP)) { 64 | zdbd_verbosep("socket_event", "epoll"); 65 | socket_client_free(ev->data.fd); 66 | continue; 67 | } 68 | 69 | // main socket event: we have a new client 70 | // create the new client and accept it 71 | for(int i = 0; i < redis->fdlen; i++) { 72 | if(ev->data.fd == redis->mainfd[i]) { 73 | socket_client_accept(redis, ev->data.fd); 74 | newclient = 1; 75 | } 76 | } 77 | 78 | // we don't need to proceed more if it's 79 | // a new client connection 80 | if(newclient) 81 | continue; 82 | 83 | // data available for reading 84 | // let's read what's available and check 85 | // the response code 86 | if(ev->events & EPOLLIN) { 87 | // call the redis chunk event handler 88 | resp_status_t ctrl = redis_chunk_read(ev->data.fd); 89 | 90 | // client error, we discard it 91 | if(ctrl == RESP_STATUS_DISCARD || ctrl == RESP_STATUS_DISCONNECTED) { 92 | socket_client_free(ev->data.fd); 93 | continue; 94 | } 95 | 96 | // (dirty) way the STOP event is handled 97 | if(ctrl == RESP_STATUS_SHUTDOWN) { 98 | zdb_log("[+] stopping daemon\n"); 99 | 100 | for(int i = 0; i < redis->fdlen; i++) 101 | close(redis->mainfd[i]); 102 | 103 | return 1; 104 | } 105 | } 106 | 107 | // client is ready for writing, let's check if any 108 | // data still needs to be sent or not 109 | if(ev->events & EPOLLOUT) { 110 | redis_delayed_write(ev->data.fd); 111 | } 112 | } 113 | 114 | return 0; 115 | } 116 | 117 | int socket_handler(redis_handler_t *handler) { 118 | struct epoll_event event; 119 | struct epoll_event *events = NULL; 120 | zdbd_stats_t *dstats = &zdbd_rootsettings.stats; 121 | 122 | // initialize empty struct 123 | memset(&event, 0, sizeof(struct epoll_event)); 124 | 125 | if((handler->evfd = epoll_create1(0)) < 0) 126 | zdbd_diep("epoll_create1"); 127 | 128 | 129 | for(int i = 0; i < handler->fdlen; i++) { 130 | event.data.fd = handler->mainfd[i]; 131 | event.events = EPOLLIN; 132 | 133 | if(epoll_ctl(handler->evfd, EPOLL_CTL_ADD, handler->mainfd[i], &event) < 0) 134 | zdbd_diep("epoll_ctl"); 135 | } 136 | 137 | events = calloc(MAXEVENTS, sizeof event); 138 | 139 | // wait for clients 140 | // this is how we support multi-client using a single thread 141 | // note that, we will only handle one request at a time 142 | // allows multiple clients to be connected 143 | 144 | while(1) { 145 | int n = epoll_wait(handler->evfd, events, MAXEVENTS, EVTIMEOUT); 146 | dstats->netevents += 1; 147 | 148 | if(n == 0) { 149 | // timeout reached, checking for background 150 | // or pending recurring task to do 151 | redis_idle_process(); 152 | continue; 153 | } 154 | 155 | if(socket_event(events, n, handler) == 1) { 156 | free(events); 157 | return 1; 158 | } 159 | 160 | // force idle process trigger after fixed amount 161 | // of commands, otherwise spamming the server enough 162 | // would never trigger it 163 | if(dstats->netevents % 100 == 0) { 164 | zdbd_debug("[+] sockets: forcing idle process [%lu]\n", dstats->netevents); 165 | redis_idle_process(); 166 | } 167 | } 168 | 169 | return 0; 170 | } 171 | 172 | #endif // __linux__ 173 | -------------------------------------------------------------------------------- /zdbd/socket_kqueue.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | // this implementation is used on macos and freebsd 10 | #ifdef __APPLE__ 11 | 12 | #include 13 | #include "libzdb.h" 14 | #include "zdbd.h" 15 | #include "redis.h" 16 | 17 | #define MAXEVENTS 64 18 | #define EVTIMEOUT 150 19 | struct kevent evset; 20 | 21 | static int socket_client_accept(redis_handler_t *redis, int fd) { 22 | int clientfd; 23 | 24 | if((clientfd = accept(fd, NULL, NULL)) == -1) { 25 | zdbd_warnp("accept"); 26 | return 1; 27 | } 28 | 29 | socket_nonblock(clientfd); 30 | socket_keepalive(clientfd); 31 | socket_client_new(clientfd); 32 | 33 | zdbd_verbose("[+] incoming connection (socket %d)\n", clientfd); 34 | 35 | EV_SET(&evset, clientfd, EVFILT_READ, EV_ADD, 0, 0, NULL); 36 | if(kevent(redis->evfd, &evset, 1, NULL, 0, NULL) == -1) { 37 | zdbd_warnp("kevent: filter read"); 38 | return 1; 39 | } 40 | 41 | EV_SET(&evset, clientfd, EVFILT_WRITE, EV_ADD | EV_ONESHOT, 0, 0, NULL); 42 | if(kevent(redis->evfd, &evset, 1, NULL, 0, NULL) == -1) { 43 | zdbd_warnp("kevent: filter write"); 44 | return 1; 45 | } 46 | 47 | return 1; 48 | } 49 | 50 | static int socket_event(struct kevent *events, int notified, redis_handler_t *redis) { 51 | struct kevent *ev; 52 | 53 | for(int i = 0; i < notified; i++) { 54 | int newclient = 0; 55 | ev = events + i; 56 | 57 | if(ev->flags & EV_EOF) { 58 | EV_SET(&evset, ev->ident, EVFILT_READ, EV_DELETE, 0, 0, NULL); 59 | 60 | if(kevent(redis->evfd, &evset, 1, NULL, 0, NULL) == -1) 61 | zdbd_diep("kevent"); 62 | 63 | socket_client_free(ev->ident); 64 | continue; 65 | 66 | } 67 | 68 | // main socket event: we have a new client 69 | // creating the new client and accepting it 70 | for(int i = 0; i < redis->fdlen; i++) { 71 | if((int) ev->ident == redis->mainfd[i]) { 72 | socket_client_accept(redis, (int) ev->ident); 73 | newclient = 1; 74 | } 75 | } 76 | 77 | // we don't need to proceed more if it's 78 | // a new client connection 79 | if(newclient) 80 | continue; 81 | 82 | if(ev->filter == EVFILT_READ) { 83 | // call the redis chunk event handler 84 | resp_status_t ctrl = redis_chunk_read(ev->ident); 85 | 86 | // client error, we discard it 87 | if(ctrl == RESP_STATUS_DISCARD || ctrl == RESP_STATUS_DISCONNECTED) { 88 | socket_client_free(ev->ident); 89 | continue; 90 | } 91 | 92 | // (dirty) way the STOP event is handled 93 | if(ctrl == RESP_STATUS_SHUTDOWN) { 94 | zdb_log("[+] stopping daemon\n"); 95 | 96 | for(int i = 0; i < redis->fdlen; i++) 97 | close(redis->mainfd[i]); 98 | 99 | return 1; 100 | } 101 | } 102 | 103 | if(ev->filter == EVFILT_WRITE) { 104 | redis_delayed_write(ev->ident); 105 | } 106 | } 107 | 108 | return 0; 109 | } 110 | 111 | int socket_handler(redis_handler_t *handler) { 112 | zdbd_stats_t *dstats = &zdbd_rootsettings.stats; 113 | struct kevent evlist[MAXEVENTS]; 114 | struct timespec timeout = { 115 | .tv_sec = 0, 116 | .tv_nsec = EVTIMEOUT * 1000000 117 | }; 118 | 119 | 120 | if((handler->evfd = kqueue()) < 0) 121 | zdbd_diep("kqueue"); 122 | 123 | for(int i = 0; i < handler->fdlen; i++) { 124 | // initialize an empty struct 125 | EV_SET(&evset, handler->mainfd[i], EVFILT_READ, EV_ADD, 0, 0, NULL); 126 | 127 | if(kevent(handler->evfd, &evset, 1, NULL, 0, NULL) == -1) 128 | zdbd_diep("kevent"); 129 | } 130 | 131 | // wait for clients 132 | // this is how we support multi-client using a single thread 133 | // note that, we will only handle one request at a time 134 | // allows multiple clients to be connected 135 | 136 | while(1) { 137 | int n = kevent(handler->evfd, NULL, 0, evlist, MAXEVENTS, &timeout); 138 | dstats->netevents += 1; 139 | 140 | if(n == 0) { 141 | // timeout reached, checking for background 142 | // or pending recurring task to do 143 | redis_idle_process(); 144 | continue; 145 | } 146 | 147 | if(socket_event(evlist, n, handler) == 1) 148 | return 1; 149 | 150 | // force idle process trigger after fixed amount 151 | // of commands, otherwise spamming the server enough 152 | // would never trigger it 153 | if(dstats->netevents % 100 == 0) { 154 | zdbd_debug("[+] sockets: forcing idle process [%llu]\n", dstats->netevents); 155 | redis_idle_process(); 156 | } 157 | } 158 | 159 | return 0; 160 | } 161 | 162 | #endif // __APPLE__ 163 | -------------------------------------------------------------------------------- /zdbd/zdbd.h: -------------------------------------------------------------------------------- 1 | #ifndef __ZDBD_H 2 | #define __ZDBD_H 3 | 4 | #ifndef ZDBD_REVISION 5 | #define ZDBD_REVISION "(unknown)" 6 | #endif 7 | 8 | #define ZDBD_DEFAULT_LISTENADDR "::" 9 | #define ZDBD_DEFAULT_PORT "9900" 10 | 11 | #define ZDBD_PATH_MAX 4096 12 | 13 | // define here version of 0-db itself 14 | // version is made as following: 15 | // 16 | // Major.Minor.Review 17 | // 18 | // ^-- incremented on bug fix and small changes 19 | // 20 | // ^-- incremented on important new features 21 | // 22 | // ^--- will only change if data format change 23 | // and not assure retro-compatibility 24 | // (eg: files written on version 1.x.x won't works 25 | // out of box on a version 2.x.x) 26 | #define ZDBD_VERSION "2.0.8" 27 | 28 | typedef struct zdbd_stats_t { 29 | // boottime is kept for zdbd uptime statistics (for INFO command) 30 | // but we use the libzdb inittime for logs 31 | struct timeval boottime; // timestamp when zdb started 32 | uint32_t clients; // lifetime amount of clients connected 33 | 34 | // commands 35 | uint64_t cmdsvalid; // amount of commands (found) executed 36 | uint64_t cmdsfailed; // amount of commands nof found received 37 | uint64_t adminfailed; // amount of authentication failed 38 | 39 | // network 40 | uint64_t networkrx; // amount of bytes received over the network 41 | uint64_t networktx; // amount of bytes transmitted over the network 42 | uint64_t netevents; // amount of socket events received 43 | 44 | } zdbd_stats_t; 45 | 46 | typedef struct zdbd_settings_t { 47 | char *listen; // network listen address 48 | char *port; // network listen port 49 | int verbose; // enable verbose print (function 'verbose') 50 | char *adminpwd; // admin password, if NULL, all users are admin 51 | char *socket; // unix socket path 52 | int background; // flag to run in background 53 | char *logfile; // where to redirect logs in background mode 54 | int protect; // flag default namespace to use admin password (for writing) 55 | int dualnet; // support for dual socket listening 56 | int rotatesec; // amount of seconds before forcing rotation of index/data 57 | 58 | zdbd_stats_t stats; 59 | 60 | } zdbd_settings_t; 61 | 62 | void zdbd_hexdump(void *buffer, size_t length); 63 | void zdbd_fulldump(void *data, size_t len); 64 | 65 | #define zdbd_log(fmt, ...) { zdbd_timelog(stdout); printf(fmt, ##__VA_ARGS__); } 66 | #define zdbd_logerr(fmt, ...) { zdbd_timelog(stderr); fprintf(stderr, fmt, ##__VA_ARGS__); } 67 | 68 | #define zdbd_danger(fmt, ...) { zdbd_timelog(stdout); printf(COLOR_RED fmt COLOR_RESET "\n", ##__VA_ARGS__); } 69 | #define zdbd_warning(fmt, ...) { zdbd_timelog(stdout); printf(COLOR_YELLOW fmt COLOR_RESET "\n", ##__VA_ARGS__); } 70 | #define zdbd_success(fmt, ...) { zdbd_timelog(stdout); printf(COLOR_GREEN fmt COLOR_RESET "\n", ##__VA_ARGS__); } 71 | #define zdbd_notice(fmt, ...) { zdbd_timelog(stdout); printf(COLOR_CYAN fmt COLOR_RESET "\n", ##__VA_ARGS__); } 72 | 73 | #ifndef RELEASE 74 | #define zdbd_verbose(...) { zdbd_timelog(stdout); printf(__VA_ARGS__); } 75 | #define zdbd_debug(...) { zdbd_timelog(stdout); printf(__VA_ARGS__); } 76 | #define zdbd_debughex(...) { zdbd_hexdump(__VA_ARGS__); } 77 | #else 78 | #define zdbd_verbose(...) { if(zdbd_rootsettings.verbose) { zdbd_timelog(stdout); printf(__VA_ARGS__); } } 79 | #define zdbd_debug(...) ((void)0) 80 | #define zdbd_debughex(...) ((void)0) 81 | #endif 82 | 83 | extern zdbd_settings_t zdbd_rootsettings; 84 | 85 | void zdbd_timelog(FILE *fp); 86 | void zdbd_diep(char *str); 87 | void zdbd_dieg(char *str, int status); 88 | void *zdbd_warnp(char *str); 89 | void zdbd_verbosep(char *prefix, char *str); 90 | 91 | #ifdef __APPLE__ 92 | #define bswap_16(value) ((((value) & 0xff) << 8) | ((value) >> 8)) 93 | #define bswap_32(value) (((uint32_t) bswap_16((uint16_t)((value) & 0xffff)) << 16) | (uint32_t) bswap_16((uint16_t)((value) >> 16))) 94 | #define bswap_64(value) (((uint64_t) bswap_32((uint32_t)((value) & 0xffffffff)) << 32) | (uint64_t) bswap_32((uint32_t)((value) >> 32))) 95 | #else 96 | #include 97 | #endif 98 | #endif 99 | --------------------------------------------------------------------------------