├── src ├── rapidjson │ ├── CPPLINT.cfg │ ├── README.md │ ├── stringbuffer.h │ ├── license.txt │ └── prettywriter.h ├── commands │ ├── json.forget.json │ ├── json.debug.json │ ├── json.arrlen.json │ ├── json.type.json │ ├── json.clear.json │ ├── json.del.json │ ├── json.objlen.json │ ├── json.objkeys.json │ ├── json.strlen.json │ ├── json.mget.json │ ├── json.resp.json │ ├── json.toggle.json │ ├── json.numincrby.json │ ├── json.nummultby.json │ ├── json.arrappend.json │ ├── json.strappend.json │ ├── json.arrpop.json │ ├── json.set.json │ ├── json.arrtrim.json │ ├── json.arrinsert.json │ ├── json.get.json │ ├── json.mset.json │ └── json.arrindex.json ├── json │ ├── CPPLINT.cfg │ ├── rapidjson_includes.h │ ├── shared_api.h │ ├── json.h │ ├── shared_api.cc │ ├── alloc.h │ ├── alloc.cc │ ├── json_api.h │ ├── util.h │ ├── stats.h │ ├── json_api.cc │ ├── memory.h │ ├── util.cc │ ├── stats.cc │ └── memory.cc └── CMakeLists.txt ├── requirements.txt ├── tst ├── unit │ ├── CPPLINT.cfg │ ├── module_sim.h │ ├── stats_test.cc │ ├── module_sim.cc │ ├── CMakeLists.txt │ ├── traps_test.cc │ ├── hashtable_test.cc │ ├── util_test.cc │ ├── json_test.cc │ └── keytable_test.cc ├── integration │ ├── data │ │ ├── wikipedia_compact.json │ │ ├── wikipedia.json │ │ ├── store.json │ │ ├── webxml.json │ │ └── truenull.json │ ├── README.md │ ├── error_handlers.py │ ├── utils_json.py │ ├── run.sh │ ├── test_rdb.py │ └── json_test_case.py └── CMakeLists.txt ├── .gitignore ├── .config └── typos.toml ├── 00-RELEASENOTES ├── .github └── workflows │ ├── spell-check.yml │ ├── trigger-json-release.yml │ └── ci.yml ├── utils └── load_1file_hostport.py ├── LICENSE ├── README.md └── CMakeLists.txt /src/rapidjson/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | exclude_files=.* 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | valkey 2 | pytest==6 3 | pytest-html -------------------------------------------------------------------------------- /src/commands/json.forget.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.FORGET": { 3 | "summary": "An alias of JSON.DEL.", 4 | "group": "json" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tst/unit/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | filter=-build/include_subdir 2 | # STL allocator needs implicit single arg constructor which CPPLINT doesn't like by default 3 | filter=-runtime/explicit 4 | filter=-runtime/string 5 | filter=-runtime/int 6 | linelength=120 7 | -------------------------------------------------------------------------------- /src/json/CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | filter=-build/include_order,-legal/copyright,-whitespace/braces,-build/c++11,-runtime/references,-build/include_what_you_use,-readability/casting,-build/header_guard,-runtime/int,-build/namespaces,-runtime/explicit 2 | linelength=120 3 | -------------------------------------------------------------------------------- /tst/integration/data/wikipedia_compact.json: -------------------------------------------------------------------------------- 1 | {"firstName":"John","lastName":"Smith","age":27,"weight":135.17,"isAlive":true,"address":{"street":"21 2nd Street","city":"New York","state":"NY","zipcode":"10021-3100"},"phoneNumbers":[{"type":"home","number":"212 555-1234"},{"type":"office","number":"646 555-4567"}],"children":[],"spouse":null,"groups":{}} -------------------------------------------------------------------------------- /tst/integration/README.md: -------------------------------------------------------------------------------- 1 | # Integration Tests 2 | 3 | This directory contains integration tests that verify the interaction between vlkaye-server and valkey-json module features working together. Unlike unit tests that test individual components in isolation, these tests validate the system's behavior as a whole. 4 | 5 | ## Requirements 6 | 7 | ```text 8 | python 3.9 9 | pytest 4 10 | ``` -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE files 2 | .idea/ 3 | *.vscode 4 | .vscode/* 5 | 6 | # Build temp files 7 | *build 8 | cmake-build-*/ 9 | 10 | # Auto-generated files 11 | **/__pycache__/* 12 | test-data 13 | *.pyc 14 | *.bin 15 | *.o 16 | *.xo 17 | *.so 18 | *.d 19 | *.a 20 | *.log 21 | *.out 22 | 23 | # Others 24 | .DS_Store 25 | .attach_pid* 26 | venv/ 27 | core.* 28 | valkeytests 29 | **/include 30 | **/report.html 31 | **/assets -------------------------------------------------------------------------------- /tst/unit/module_sim.h: -------------------------------------------------------------------------------- 1 | // 2 | // Simulate the Valkey Module Environment 3 | // 4 | #ifndef VALKEYJSONMODULE_TST_UNIT_MODULE_SIM_H_ 5 | #define VALKEYJSONMODULE_TST_UNIT_MODULE_SIM_H_ 6 | 7 | #include 8 | #include 9 | 10 | extern size_t malloced; // Total currently allocated memory 11 | void setupValkeyModulePointers(); 12 | std::string test_getLogText(); 13 | 14 | #endif // VALKEYJSONMODULE_TST_UNIT_MODULE_SIM_H_ 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /.config/typos.toml: -------------------------------------------------------------------------------- 1 | # See https://github.com/crate-ci/typos/blob/master/docs/reference.md to configure typos 2 | 3 | [files] 4 | extend-exclude = [ 5 | ".git/", 6 | "deps/", 7 | "tst/", 8 | "build/", 9 | "cmake-build-debug/", 10 | "rapidjson/", 11 | "CMakeLists.txt" 12 | ] 13 | 14 | [default.extend-words] 15 | Valkey = "Valkey" 16 | valkey = "valkey" 17 | threadsave = "threadsave" 18 | 19 | [default.extend-identifiers] 20 | ctrlChar_2ndPart = "ctrlChar_2ndPart" 21 | INSERTs = "INSERTs" -------------------------------------------------------------------------------- /tst/integration/data/wikipedia.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstName": "John", 3 | "lastName": "Smith", 4 | "age": 27, 5 | "weight": 135.17, 6 | "isAlive": true, 7 | "address": { 8 | "street": "21 2nd Street", 9 | "city": "New York", 10 | "state": "NY", 11 | "zipcode": "10021-3100" 12 | }, 13 | "phoneNumbers": [ 14 | { 15 | "type": "home", 16 | "number": "212 555-1234" 17 | }, 18 | { 19 | "type": "office", 20 | "number": "646 555-4567" 21 | } 22 | ], 23 | "children": [], 24 | "spouse": null, 25 | "groups": {} 26 | } -------------------------------------------------------------------------------- /src/commands/json.debug.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.DEBUG": { 3 | "summary": "Reports information. Supported subcommands are: MEMORY, DEPTH, FIELDS, HELP", 4 | "complexity": "O(1)", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "SLOW", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "subcommand & arguments", 16 | "type": "string" 17 | } 18 | ] 19 | } 20 | } -------------------------------------------------------------------------------- /00-RELEASENOTES: -------------------------------------------------------------------------------- 1 | Hello! This file is just a placeholder, since this is the "unstable" branch 2 | of Valkey-json, the place where all the development happens. 3 | 4 | There is no release notes for this branch, it gets forked into another branch 5 | every time there is a partial feature freeze in order to eventually create 6 | a new stable release. 7 | 8 | Usually "unstable" is stable enough for you to use it in development environments 9 | however you should never use it in production environments. 10 | 11 | More information is available at https://valkey.io 12 | 13 | Happy hacking! 14 | -------------------------------------------------------------------------------- /src/rapidjson/README.md: -------------------------------------------------------------------------------- 1 | # RapidJSON Source Code 2 | * The original RapidJSON Source Code is cloned at build time using CMAKELISTS 3 | * Last commit on the master branch: ebd87cb468fb4cb060b37e579718c4a4125416c1, 2024-12-02 4 | 5 | # Modifications 6 | We made a few changes to the RapidJSON source code. Before the changes are pushed to the open source, 7 | we have to include a private copy of the file. Modified RapidJSON code is under src/rapidjson. 8 | 9 | ## document.h` 10 | We need to modify RapidJSON's document.h to support JSON depth limit. 11 | 12 | ### reader.h 13 | Modified reader.h to only generate integers in int64 range. 14 | 15 | -------------------------------------------------------------------------------- /tst/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # We use GoogleTest for both unit and system tests 3 | ################################################## 4 | message("tst/CMakeLists.txt") 5 | 6 | if(ENABLE_UNIT_TESTS) 7 | # Fetch GoogleTest. 8 | include(FetchContent) 9 | 10 | FetchContent_Declare( 11 | googletest 12 | GIT_REPOSITORY https://github.com/google/googletest.git 13 | GIT_TAG 58d77fa8070e8cec2dc1ed015d66b454c8d78850 # release-1.12.1 14 | OVERRIDE_FIND_PACKAGE) 15 | FetchContent_MakeAvailable(googletest) 16 | 17 | 18 | include(GoogleTest) 19 | 20 | add_subdirectory(unit) 21 | endif() -------------------------------------------------------------------------------- /src/json/rapidjson_includes.h: -------------------------------------------------------------------------------- 1 | #ifndef _RAPIDJSON_INCLUDES_H 2 | #define _RAPIDJSON_INCLUDES_H 3 | 4 | /* 5 | * This file includes all RapidJSON Files (modified or original). Any RAPIDJSON-global #defines, etc. belong here 6 | */ 7 | 8 | #if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || \ 9 | defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) 10 | #define RAPIDJSON_SSE42 1 11 | #endif 12 | 13 | #if defined(__ARM_NEON) || defined(__ARM_NEON__) 14 | #define RAPIDJSON_NEON 1 15 | #endif 16 | 17 | #define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 18 | 19 | #include "rapidjson/prettywriter.h" 20 | #include "rapidjson/document.h" 21 | #include 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /src/commands/json.arrlen.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRLEN": { 3 | "summary": "Get length of the array at the path.", 4 | "complexity": "O(N) where N is the number of json arrays matched at the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.type.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.TYPE": { 3 | "summary": "Report the type of the values at the given path.", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.clear.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.CLEAR": { 3 | "summary": "Clear the arrays or an object at the specified path.", 4 | "complexity": "O(N) where N is the number of json arrays/objects matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.del.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.DEL": { 3 | "summary": "Delete the JSON values at the specified path in a document key.", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.objlen.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.OBJLEN": { 3 | "summary": "Get the number of keys in the object at the specified path.", 4 | "complexity": "O(N) where N is the number of json objects matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.objkeys.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.OBJKEYS": { 3 | "summary": "Retrieve the key names from the objects at the specified path.", 4 | "complexity": "O(N) where N is the number of json objects matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.strlen.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.STRLEN": { 3 | "summary": "Get the length of the JSON string values at the specified path.", 4 | "complexity": "O(N) where N is the number of string values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.mget.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.MGET": { 3 | "summary": "Get serialized JSONs at the path from multiple document keys. Return null for non-existent key or JSON path.", 4 | "complexity": "O(N) where N is the number of keys", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": -3, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "multiple": true, 18 | "key_spec_index": 0 19 | }, 20 | { 21 | "name": "path", 22 | "type": "string" 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.resp.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.RESP": { 3 | "summary": "Return the JSON value at the given path in Redis Serialization Protocol (RESP).", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.toggle.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.TOGGLE": { 3 | "summary": "Toggle boolean values between true and false at the specified path.", 4 | "complexity": "O(N) where N is the number of json boolean values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /src/commands/json.numincrby.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.NUMINCRBY": { 3 | "summary": "Increment the number values at the path by a given number.", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 4, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "number", 25 | "type": "integer" 26 | } 27 | ] 28 | } 29 | } -------------------------------------------------------------------------------- /src/commands/json.nummultby.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.NUMMULTBY": { 3 | "summary": "Multiply the numeric values at the path by a given number.", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 4, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "number", 25 | "type": "integer" 26 | } 27 | ] 28 | } 29 | } -------------------------------------------------------------------------------- /src/commands/json.arrappend.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRAPPEND": { 3 | "summary": "Append one or more values to the array values at the path.", 4 | "complexity": "O(N) where N is the number of values", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": -4, 8 | "acl_categories": [ 9 | "JSON", 10 | "WRITE", 11 | "FAST" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "json", 25 | "type": "string", 26 | "multiple": true 27 | } 28 | ] 29 | } 30 | } -------------------------------------------------------------------------------- /src/commands/json.strappend.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.STRAPPEND": { 3 | "summary": "Append a string to the JSON strings at the specified path.", 4 | "complexity": "O(N) where N is the number of string values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 3, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | }, 24 | { 25 | "name": "json_string", 26 | "type": "string" 27 | } 28 | ] 29 | } 30 | } -------------------------------------------------------------------------------- /src/json/shared_api.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-json contributors 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD 3-Clause 5 | * 6 | */ 7 | 8 | #ifndef VALKEYJSONMODULE_SHARED_API_H_ 9 | #define VALKEYJSONMODULE_SHARED_API_H_ 10 | 11 | #include "./include/valkeymodule.h" 12 | 13 | #include 14 | 15 | // 16 | // Fetch JSON text associated with "path". 17 | // 18 | // Errors: 19 | // 20 | // If the key isn't JSON, then you'll get VALKEYMODULE_ERR for the return. 21 | // If the key is JSON, but the path doesn't identify anything, then you'll get a nullptr for the result. 22 | // Otherwise you get the JSON text that matches the path. 23 | // 24 | int SharedJSON_Get(ValkeyModuleKey *key, const char *path, ValkeyModuleString **result); 25 | 26 | // 27 | // Internal. 28 | // 29 | void SharedAPI_Register(ValkeyModuleCtx *ctx); 30 | 31 | 32 | #endif // VALKEYJSONMODULE_SHARED_API_H_ 33 | -------------------------------------------------------------------------------- /src/json/json.h: -------------------------------------------------------------------------------- 1 | #ifndef VALKEYJSONMODULE_JSON_H_ 2 | #define VALKEYJSONMODULE_JSON_H_ 3 | 4 | #include "./include/valkeymodule.h" 5 | #include "json/util.h" 6 | 7 | #include 8 | 9 | size_t json_get_max_document_size(); 10 | size_t json_get_defrag_threshold(); 11 | size_t json_get_max_path_limit(); 12 | size_t json_get_max_parser_recursion_depth(); 13 | size_t json_get_max_recursive_descent_tokens(); 14 | size_t json_get_max_query_string_size(); 15 | 16 | bool json_is_instrument_enabled_insert(); 17 | bool json_is_instrument_enabled_update(); 18 | bool json_is_instrument_enabled_delete(); 19 | bool json_is_instrument_enabled_dump_doc_before(); 20 | bool json_is_instrument_enabled_dump_doc_after(); 21 | bool json_is_instrument_enabled_dump_value_before_delete(); 22 | 23 | JsonUtilCode verify_open_doc_key(ValkeyModuleKey *key); 24 | 25 | #define DOUBLE_CHARS_CUTOFF 24 26 | 27 | #endif // VALKEYJSONMODULE_JSON_H_ 28 | -------------------------------------------------------------------------------- /src/commands/json.arrpop.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRPOP": { 3 | "summary": "Remove and returns the element at the given index. Popping an empty array returns null.", 4 | "complexity": "O(N) where N is the number of jsons arrays matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string", 22 | "optional": true 23 | }, 24 | { 25 | "name": "index", 26 | "type": "integer", 27 | "optional": true 28 | } 29 | ] 30 | } 31 | } -------------------------------------------------------------------------------- /src/commands/json.set.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.SET": { 3 | "summary": "Set JSON values at the specified path.", 4 | "complexity": "O(N) where N is the number of json values matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 4, 8 | "acl_categories": [ 9 | "WRITE", 10 | "SLOW", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "json", 25 | "type": "string" 26 | }, 27 | { 28 | "name": "options", 29 | "type": "string", 30 | "optional": true 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /src/commands/json.arrtrim.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRTRIM": { 3 | "summary": "Trim the array at the path so that it becomes subarray [start, end], both inclusive.", 4 | "complexity": "O(N) where N is the number of json arrays matched by the path.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 5, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "start", 25 | "type": "integer" 26 | }, 27 | { 28 | "name": "end", 29 | "type": "integer" 30 | } 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /.github/workflows/spell-check.yml: -------------------------------------------------------------------------------- 1 | # A CI action that using codespell to check spell. 2 | # .github/.codespellrc is a config file. 3 | # .github/wordlist.txt is a list of words that will ignore word checks. 4 | # More details please check the following link: 5 | # https://github.com/codespell-project/codespell 6 | name: Spellcheck 7 | 8 | on: 9 | push: 10 | pull_request: 11 | 12 | concurrency: 13 | group: spellcheck-${{ github.head_ref || github.ref }} 14 | cancel-in-progress: true 15 | 16 | permissions: 17 | contents: read 18 | 19 | jobs: 20 | build: 21 | name: Spellcheck 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 27 | 28 | - name: Install typos 29 | uses: taiki-e/install-action@fe9759bf4432218c779595708e80a1aadc85cedc # v2.46.10 30 | with: 31 | tool: typos 32 | 33 | - name: Spell check 34 | run: typos --config=./.config/typos.toml 35 | -------------------------------------------------------------------------------- /src/commands/json.arrinsert.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRINSERT": { 3 | "summary": "Insert one or more values into an array at the given path before the specified index.", 4 | "complexity": "O(N) where N is the length of the array.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": -5, 8 | "acl_categories": [ 9 | "WRITE", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "index", 25 | "type": "integer" 26 | }, 27 | { 28 | "name": "json", 29 | "type": "string", 30 | "multiple": true 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /src/commands/json.get.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.GET": { 3 | "summary": "Get the serialized JSON at one or multiple paths.", 4 | "complexity": "O(N) where N is the number of paths", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 2, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "INDENT/NEWLINE/SPACE", 21 | "type": "string", 22 | "optional": true 23 | }, 24 | { 25 | "name": "NOESCAPE", 26 | "type": "string", 27 | "optional": true 28 | }, 29 | { 30 | "name": "path", 31 | "type": "string", 32 | "multiple": true, 33 | "optional": true 34 | } 35 | ] 36 | } 37 | } -------------------------------------------------------------------------------- /src/commands/json.mset.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.MSET": { 3 | "summary": "Set multiple JSON values at the path to multiple keys.", 4 | "complexity": "O(N) where N is the number of keys", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": -4, 8 | "acl_categories": [ 9 | "WRITE", 10 | "SLOW", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "data", 16 | "type": "block", 17 | "multiple": true, 18 | "arguments": [ 19 | { 20 | "name": "key", 21 | "type": "key", 22 | "key_spec_index": 0 23 | }, 24 | { 25 | "name": "path", 26 | "type": "string" 27 | }, 28 | { 29 | "name": "json", 30 | "type": "string" 31 | } 32 | ] 33 | } 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /src/json/shared_api.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-json contributors 3 | * All rights reserved. 4 | * SPDX-License-Identifier: BSD 3-Clause 5 | * 6 | */ 7 | 8 | #include "shared_api.h" 9 | #include "json/json.h" 10 | #include "json/dom.h" 11 | #include "json/selector.h" 12 | 13 | void SharedAPI_Register(ValkeyModuleCtx *ctx) { 14 | if(ValkeyModule_ExportSharedAPI(ctx, "JSON_GetValue", (void *)SharedJSON_Get) != VALKEYMODULE_OK) { 15 | ValkeyModule_Assert(false); 16 | } 17 | } 18 | 19 | int SharedJSON_Get(ValkeyModuleKey *key, const char *path, ValkeyModuleString **result) { 20 | if (verify_open_doc_key(key) != JSONUTIL_SUCCESS) { 21 | return VALKEYMODULE_ERR; 22 | } 23 | // Fetch the document from the key 24 | JDocument *doc = static_cast(ValkeyModule_ModuleTypeGetValue(key)); 25 | rapidjson::StringBuffer output; 26 | if (dom_get_value_as_str(doc, path, nullptr, output) == JSONUTIL_SUCCESS) { 27 | *result = ValkeyModule_CreateString(nullptr, output.GetString(), output.GetLength()); 28 | return VALKEYMODULE_OK; 29 | } else { 30 | return VALKEYMODULE_ERR; 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | message("src/CMakeLists.txt: Build valkey-json") 2 | 3 | set(OBJECT_TARGET json-objects CACHE INTERNAL "Object target for json module") 4 | add_library(${OBJECT_TARGET} OBJECT "") 5 | 6 | # Build with C11 & C++17 7 | set_target_properties( 8 | ${OBJECT_TARGET} 9 | PROPERTIES 10 | C_STANDARD 11 11 | C_STANDARD_REQUIRED ON 12 | CXX_STANDARD 17 13 | CXX_STANDARD_REQUIRED ON 14 | POSITION_INDEPENDENT_CODE ON 15 | ) 16 | 17 | target_include_directories(${OBJECT_TARGET} 18 | 19 | # Need to make the source files public within CMake 20 | # so that they are used when building the tests. 21 | PUBLIC 22 | ${CMAKE_CURRENT_SOURCE_DIR} 23 | ${CMAKE_CURRENT_SOURCE_DIR}/include 24 | ${rapidjson_SOURCE_DIR}/include 25 | ) 26 | 27 | # Add source files for the JSON module 28 | target_sources(${OBJECT_TARGET} 29 | PRIVATE 30 | json/json.cc 31 | json/dom.cc 32 | json/alloc.cc 33 | json/util.cc 34 | json/stats.cc 35 | json/selector.cc 36 | json/keytable.cc 37 | json/memory.cc 38 | json/json_api.cc 39 | json/shared_api.cc 40 | ) 41 | 42 | add_library(${JSON_MODULE_LIB} SHARED $) 43 | add_dependencies(${OBJECT_TARGET} valkey) 44 | -------------------------------------------------------------------------------- /src/commands/json.arrindex.json: -------------------------------------------------------------------------------- 1 | { 2 | "JSON.ARRINDEX": { 3 | "summary": "Search for the first occurrence of a scalar JSON value in arrays located at the specified path. Indices out of range are adjusted.", 4 | "complexity": "O(N), where N is the length of the array.", 5 | "group": "json", 6 | "module_since": "1.0.0", 7 | "arity": 4, 8 | "acl_categories": [ 9 | "READ", 10 | "FAST", 11 | "JSON" 12 | ], 13 | "arguments": [ 14 | { 15 | "name": "key", 16 | "type": "key", 17 | "key_spec_index": 0 18 | }, 19 | { 20 | "name": "path", 21 | "type": "string" 22 | }, 23 | { 24 | "name": "json-scalar", 25 | "type": "string" 26 | }, 27 | { 28 | "name": "start", 29 | "type": "integer", 30 | "optional": true 31 | }, 32 | { 33 | "name": "end", 34 | "type": "integer", 35 | "optional": true 36 | } 37 | ] 38 | } 39 | } -------------------------------------------------------------------------------- /tst/integration/error_handlers.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | class ErrorStringTester: 5 | def is_syntax_error(string): 6 | return string.startswith("SYNTAXERR") or \ 7 | string.startswith("unknown subcommand") 8 | 9 | def is_nonexistent_error(string): 10 | return string.startswith("NONEXISTENT") 11 | 12 | def is_wrongtype_error(string): 13 | return string.startswith("WRONGTYPE") 14 | 15 | def is_number_overflow_error(string): 16 | return string.startswith("OVERFLOW") 17 | 18 | def is_outofboundaries_error(string): 19 | return string.startswith("OUTOFBOUNDARIES") 20 | 21 | def is_limit_exceeded_error(string): 22 | return string.startswith("LIMIT") 23 | 24 | def is_write_error(string): 25 | return string.startswith("ERROR") or string.startswith("OUTOFBOUNDARIES") or \ 26 | string.startswith("WRONGTYPE") or string.startswith("NONEXISTENT") 27 | 28 | # NOTE: Uses .find instead of .startswith in case prefix added in the future 29 | def is_wrong_number_of_arguments_error(string): 30 | return string.find("wrong number of arguments") >= 0 or \ 31 | string.lower().find('invalid number of arguments') >= 0 32 | -------------------------------------------------------------------------------- /.github/workflows/trigger-json-release.yml: -------------------------------------------------------------------------------- 1 | name: Trigger Valkey Bundle Update - JSON 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | inputs: 8 | version: 9 | description: Version of Valkey JSON module that was released 10 | required: true 11 | 12 | jobs: 13 | trigger: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Determine version 17 | id: determine-vars 18 | run: | 19 | if [[ "${{ github.event_name }}" == "release" ]]; then 20 | echo "Triggered by a release event." 21 | VERSION=${{ github.event.release.tag_name }} 22 | else 23 | echo "Triggered by workflow_dispatch." 24 | VERSION=${{ inputs.version }} 25 | fi 26 | 27 | echo "version=$VERSION" >> $GITHUB_OUTPUT 28 | 29 | - name: Trigger valkey-bundle update 30 | uses: peter-evans/repository-dispatch@v3 31 | with: 32 | token: ${{ secrets.EXTENSION_PAT }} 33 | repository: ${{ github.repository_owner }}/valkey-bundle 34 | event-type: json-release 35 | client-payload: > 36 | { 37 | "version": "${{ steps.determine-vars.outputs.version }}", 38 | "component": "json" 39 | } -------------------------------------------------------------------------------- /tst/integration/utils_json.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import random 4 | import string 5 | from valkey.exceptions import ResponseError 6 | 7 | JSON_MODULE_NAME = 'json' 8 | JSON_INFO_NAMES = { 9 | 'num_documents': JSON_MODULE_NAME + '_num_documents', 10 | 'total_memory_bytes': JSON_MODULE_NAME + '_total_memory_bytes', 11 | 'doc_histogram': JSON_MODULE_NAME + '_doc_histogram', 12 | 'read_histogram': JSON_MODULE_NAME + '_read_histogram', 13 | 'insert_histogram': JSON_MODULE_NAME + '_insert_histogram', 14 | 'update_histogram': JSON_MODULE_NAME + '_update_histogram', 15 | 'delete_histogram': JSON_MODULE_NAME + '_delete_histogram', 16 | 'max_path_depth_ever_seen': JSON_MODULE_NAME + '_max_path_depth_ever_seen', 17 | 'max_document_size_ever_seen': JSON_MODULE_NAME + '_max_document_size_ever_seen', 18 | 'total_malloc_bytes_used': JSON_MODULE_NAME + "_total_malloc_bytes_used", 19 | 'memory_traps_enabled': JSON_MODULE_NAME + "_memory_traps_enabled", 20 | } 21 | SIZE_64MB = 64 * 1024 * 1024 22 | DEFAULT_MAX_PATH_LIMIT = 128 23 | DEFAULT_WIKIPEDIA_PATH = 'data/wikipedia.json' 24 | DEFAULT_WIKIPEDIA_COMPACT_PATH = 'data/wikipedia_compact.json' 25 | DEFAULT_STORE_PATH = 'data/store.json' 26 | JSON_INFO_METRICS_SECTION = JSON_MODULE_NAME + '_core_metrics' 27 | 28 | JSON_MODULE_NAME = 'json' 29 | -------------------------------------------------------------------------------- /utils/load_1file_hostport.py: -------------------------------------------------------------------------------- 1 | #!python3 2 | # 3 | # Load a JSON file and create a key. 4 | # Usage: 5 | # [HOST=] [PORT=] [SSL=] python3 load_1file_hostport.py 6 | # 7 | # e.g. 8 | # python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia 9 | # PORT=6380 python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia 10 | # 11 | import redis, sys, os, logging 12 | from redis.exceptions import ResponseError, ConnectionError, TimeoutError 13 | 14 | if len(sys.argv) < 3: 15 | print("Usage: [HOST=] [PORT=] [SSL=] python3 load_1file_hostport.py ") 16 | exit(1) 17 | 18 | host = os.getenv('HOST', 'localhost') 19 | port = os.getenv('PORT', '6379') 20 | ssl = os.getenv('SSL', 'False') 21 | is_ssl = (ssl == 'True') 22 | json_file_path = sys.argv[1] 23 | key = sys.argv[2] 24 | 25 | r = redis.Redis(host=host, port=port, ssl=is_ssl, socket_timeout=3) 26 | try: 27 | r.ping() 28 | logging.info(f"Connected to valkey {host}:{port}, ssl: {is_ssl}") 29 | except (ConnectionError, TimeoutError): 30 | logging.error(f"Failed to connect to valkey {host}:{port}, ssl: {is_ssl}") 31 | exit(1) 32 | 33 | def load_file(json_file_path, key): 34 | with open(json_file_path, 'r') as f: 35 | data = f.read() 36 | r.execute_command('json.set', key, '.', data) 37 | logging.info("Created key %s" %key) 38 | 39 | load_file(json_file_path, key) 40 | -------------------------------------------------------------------------------- /tst/unit/stats_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "json/stats.h" 3 | 4 | class StatsTest : public ::testing::Test { 5 | }; 6 | 7 | TEST_F(StatsTest, testFindBucket) { 8 | EXPECT_EQ(jsonstats_find_bucket(0), 0); 9 | EXPECT_EQ(jsonstats_find_bucket(200), 0); 10 | EXPECT_EQ(jsonstats_find_bucket(256), 1); 11 | EXPECT_EQ(jsonstats_find_bucket(500), 1); 12 | EXPECT_EQ(jsonstats_find_bucket(1024), 2); 13 | EXPECT_EQ(jsonstats_find_bucket(2000), 2); 14 | EXPECT_EQ(jsonstats_find_bucket(4*1024), 3); 15 | EXPECT_EQ(jsonstats_find_bucket(5000), 3); 16 | EXPECT_EQ(jsonstats_find_bucket(16*1024), 4); 17 | EXPECT_EQ(jsonstats_find_bucket(50000), 4); 18 | EXPECT_EQ(jsonstats_find_bucket(64*1024), 5); 19 | EXPECT_EQ(jsonstats_find_bucket(100000), 5); 20 | EXPECT_EQ(jsonstats_find_bucket(256*1024), 6); 21 | EXPECT_EQ(jsonstats_find_bucket(1000000), 6); 22 | EXPECT_EQ(jsonstats_find_bucket(1024*1024), 7); 23 | EXPECT_EQ(jsonstats_find_bucket(4000000), 7); 24 | EXPECT_EQ(jsonstats_find_bucket(4*1024*1024), 8); 25 | EXPECT_EQ(jsonstats_find_bucket(5000000), 8); 26 | EXPECT_EQ(jsonstats_find_bucket(16*1024*1024), 9); 27 | EXPECT_EQ(jsonstats_find_bucket(20000000), 9); 28 | EXPECT_EQ(jsonstats_find_bucket(60*1024*1024), 9); 29 | EXPECT_EQ(jsonstats_find_bucket(64*1024*1024), 10); 30 | EXPECT_EQ(jsonstats_find_bucket(90000000), 10); 31 | EXPECT_EQ(jsonstats_find_bucket(1024*1024*1024), 10); 32 | } 33 | -------------------------------------------------------------------------------- /src/json/alloc.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This C module is the JSON memory allocator (also called DOM allocator), which wraps around Valkey's built-in 3 | * allocation functions - ValkeyModule_Alloc, ValkeyModule_Free and ValkeyModule_Realloc. All memory allocations, 4 | * permanent or transient, should be done through this interface, so that allocated memories are correctly 5 | * reported to the Valkey engine (MEMORY STATS). 6 | * 7 | * Besides correctly reporting memory usage to Valkey, it also provides a facility to track memory usage of JSON 8 | * objects, so that we can achieve the following: 9 | * 1. To track total memory allocated to JSON objects. This is done through an atomic global counter. Note that 10 | * Valkey engine only reports total memories for all keys, not by key type. This JSON memory allocator overcomes 11 | * such deficiency. 12 | * 2. To track each JSON document object's memory size. This is done through a thread local counter. With the ability 13 | * to track individual document's footprint, we can maintain a few interesting histograms that will provide 14 | * insights into data distribution and API access patterns. 15 | */ 16 | #ifndef VALKEYJSONMODULE_ALLOC_H_ 17 | #define VALKEYJSONMODULE_ALLOC_H_ 18 | 19 | #include 20 | 21 | #include "json/memory.h" 22 | 23 | void *dom_alloc(size_t size); 24 | void dom_free(void *ptr); 25 | void *dom_realloc(void *orig_ptr, size_t new_size); 26 | char *dom_strdup(const char *s); 27 | char *dom_strndup(const char *s, const size_t n); 28 | 29 | #endif // VALKEYJSONMODULE_ALLOC_H_ 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024, Valkey 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /src/json/alloc.cc: -------------------------------------------------------------------------------- 1 | #include "json/memory.h" 2 | #include "json/alloc.h" 3 | #include "json/stats.h" 4 | #include 5 | #include 6 | 7 | extern "C" { 8 | #define VALKEYMODULE_EXPERIMENTAL_API 9 | #include <./include/valkeymodule.h> 10 | } 11 | 12 | void *dom_alloc(size_t size) { 13 | void *ptr = memory_alloc(size); 14 | // actually allocated size may not be same as the requested size 15 | size_t real_size = memory_allocsize(ptr); 16 | jsonstats_increment_used_mem(real_size); 17 | return ptr; 18 | } 19 | 20 | void dom_free(void *ptr) { 21 | size_t size = memory_allocsize(ptr); 22 | memory_free(ptr); 23 | jsonstats_decrement_used_mem(size); 24 | } 25 | 26 | void *dom_realloc(void *orig_ptr, size_t new_size) { 27 | // We need to handle the following two edge cases first. Otherwise, the following 28 | // calculation of the incremented/decremented amount will fail. 29 | if (new_size == 0 && orig_ptr != nullptr) { 30 | dom_free(orig_ptr); 31 | return nullptr; 32 | } 33 | if (orig_ptr == nullptr) return dom_alloc(new_size); 34 | 35 | size_t orig_size = memory_allocsize(orig_ptr); 36 | void *new_ptr = memory_realloc(orig_ptr, new_size); 37 | // actually allocated size may not be same as the requested size 38 | size_t real_new_size = memory_allocsize(new_ptr); 39 | if (real_new_size > orig_size) 40 | jsonstats_increment_used_mem(real_new_size - orig_size); 41 | else if (real_new_size < orig_size) 42 | jsonstats_decrement_used_mem(orig_size - real_new_size); 43 | 44 | return new_ptr; 45 | } 46 | 47 | char *dom_strdup(const char *s) { 48 | size_t size = strlen(s) + 1; 49 | char *dup = static_cast(dom_alloc(size)); 50 | strncpy(dup, s, size); 51 | return dup; 52 | } 53 | 54 | char *dom_strndup(const char *s, const size_t n) { 55 | char *dup = static_cast(dom_alloc(n + 1)); 56 | strncpy(dup, s, n); 57 | dup[n] = '\0'; 58 | return dup; 59 | } 60 | -------------------------------------------------------------------------------- /tst/integration/data/store.json: -------------------------------------------------------------------------------- 1 | { 2 | "store": { 3 | "books": [ 4 | { 5 | "category": "reference", 6 | "author": "Nigel Rees", 7 | "title": "Sayings of the Century", 8 | "price": 8.95, 9 | "in-stock": true 10 | }, 11 | { 12 | "category": "fiction", 13 | "author": "Evelyn Waugh", 14 | "title": "Sword of Honour", 15 | "price": 12.99, 16 | "in-stock": true, 17 | "movies": [ 18 | { 19 | "title": "Sword of Honour - movie", 20 | "realisator": { 21 | "first_name": "Bill", 22 | "last_name": "Anderson" 23 | } 24 | } 25 | ] 26 | }, 27 | { 28 | "category": "fiction", 29 | "author": "Herman Melville", 30 | "title": "Moby Dick", 31 | "isbn": "0-553-21311-3", 32 | "price": 9, 33 | "in-stock": false 34 | }, 35 | { 36 | "category": "fiction", 37 | "author": "J. R. R. Tolkien", 38 | "title": "The Lord of the Rings", 39 | "isbn": "0-115-03266-2", 40 | "price": 22.99, 41 | "in-stock": true 42 | }, 43 | { 44 | "category": "reference", 45 | "author": "William Jr. Strunk", 46 | "title": "The Elements of Style", 47 | "price": 6.99, 48 | "in-stock": false 49 | }, 50 | { 51 | "category": "fiction", 52 | "author": "Leo Tolstoy", 53 | "title": "Anna Karenina", 54 | "price": 22.99, 55 | "in-stock": true 56 | }, 57 | { 58 | "category": "reference", 59 | "author": "Sarah Janssen", 60 | "title": "The World Almanac and Book of Facts 2021", 61 | "isbn": "0-925-23305-2", 62 | "price": 10.69, 63 | "in-stock": false 64 | }, 65 | { 66 | "category": "reference", 67 | "author": "Kate L. Turabian", 68 | "title": "Manual for Writers of Research Papers", 69 | "isbn": "0-675-16695-1", 70 | "price": 8.59, 71 | "in-stock": true 72 | } 73 | ], 74 | "bicycle": { 75 | "color": "red", 76 | "price": 19.64, 77 | "in-stock": true 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /tst/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Sometimes processes are left running when test is cancelled. 4 | # Therefore, before build start, we kill all running test processes left from previous test run. 5 | echo "Kill old running test" 6 | pkill -9 -x Pytest || true 7 | pkill -9 -f "valkey-server.*:" || true 8 | pkill -9 -f Valgrind || true 9 | pkill -9 -f "valkey-benchmark" || true 10 | 11 | # If environment variable SERVER_VERSION is not set, default to "unstable" 12 | if [ -z "$SERVER_VERSION" ]; then 13 | echo "SERVER_VERSION environment variable is not set. Defaulting to \"unstable\"." 14 | export SERVER_VERSION="unstable" 15 | fi 16 | 17 | # cd to the current directory of the script 18 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 19 | cd "${DIR}" 20 | 21 | export SOURCE_DIR=$2 22 | export MODULE_PATH=${SOURCE_DIR}/build/src/libjson.so 23 | echo "Running integration tests against Valkey version $SERVER_VERSION" 24 | 25 | if [[ ! -z "${TEST_PATTERN}" ]] ; then 26 | export TEST_PATTERN="-k ${TEST_PATTERN}" 27 | fi 28 | 29 | BINARY_PATH=".build/binaries/${SERVER_VERSION}/valkey-server" 30 | 31 | if [[ ! -f "${BINARY_PATH}" ]] ; then 32 | echo "${BINARY_PATH} missing" 33 | exit 1 34 | fi 35 | 36 | if [[ $1 == "test" ]] ; then 37 | if [ ! -z "${ASAN_BUILD}" ]; then 38 | echo "Running tests and checking for memory leaks" 39 | python -m pytest --capture=sys --html=report.html --cache-clear -v ${TEST_FLAG} ./ ${TEST_PATTERN} 2>&1 | tee test_output.tmp 40 | # Check for memory leaks in the output 41 | if grep -q "LeakSanitizer: detected memory leaks" test_output.tmp; then 42 | RED='\033[0;31m' 43 | echo -e "${RED}Memory leaks detected in the following tests:" 44 | LEAKING_TESTS=$(grep -B 2 "LeakSanitizer: detected memory leaks" test_output.tmp | \ 45 | grep -v "LeakSanitizer" | \ 46 | grep ".*\.py::") 47 | 48 | LEAK_COUNT=$(echo "$LEAKING_TESTS" | wc -l) 49 | 50 | # Output each leaking test 51 | echo "$LEAKING_TESTS" | while read -r line; do 52 | echo "::error::Test with leak: $line" 53 | done 54 | 55 | echo -e "\n$LEAK_COUNT python integration tests have leaks detected in them" 56 | rm test_output.tmp 57 | exit 1 58 | fi 59 | rm test_output.tmp 60 | else 61 | python -m pytest --html=report.html --cache-clear -v ${TEST_FLAG} ./ ${TEST_PATTERN} 62 | fi 63 | else 64 | echo "Unknown target: $1" 65 | exit 1 66 | fi 67 | 68 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build-release: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | server_version: ['unstable', '8.0', '8.1', '9.0'] 12 | steps: 13 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 14 | - name: Set the server version 15 | run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV 16 | - name: Build release 17 | run: ./build.sh 18 | 19 | unit-tests: 20 | runs-on: ubuntu-latest 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | server_version: ['unstable', '8.0', '8.1', '9.0'] 25 | steps: 26 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 27 | - name: Set the server version 28 | run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV 29 | - name: Run unit tests 30 | run: ./build.sh --unit 31 | 32 | integration-tests: 33 | runs-on: ubuntu-latest 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | server_version: ['unstable', '8.0', '8.1', '9.0'] 38 | steps: 39 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 40 | - name: Set the server version 41 | run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV 42 | - name: Set up Python 43 | uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 44 | with: 45 | python-version: '3.9' 46 | - name: Install dependencies 47 | run: | 48 | python -m pip install --upgrade pip 49 | pip install -r requirements.txt 50 | - name: Run integration tests 51 | run: ./build.sh --integration 52 | 53 | asan-tests: 54 | runs-on: ubuntu-latest 55 | strategy: 56 | fail-fast: false 57 | matrix: 58 | server_version: ['unstable', '8.0', '8.1', '9.0'] 59 | steps: 60 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 61 | - name: Set the server version 62 | run: echo "SERVER_VERSION=${{ matrix.server_version }}" >> $GITHUB_ENV 63 | - name: Set up Python 64 | uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 65 | with: 66 | python-version: '3.9' 67 | - name: Install dependencies 68 | run: | 69 | python -m pip install --upgrade pip 70 | pip install -r requirements.txt 71 | - name: Run ASAN integration tests 72 | run: | 73 | export ASAN_BUILD=true 74 | ./build.sh --integration 75 | -------------------------------------------------------------------------------- /tst/integration/test_rdb.py: -------------------------------------------------------------------------------- 1 | from utils_json import DEFAULT_MAX_PATH_LIMIT, DEFAULT_STORE_PATH 2 | from json_test_case import JsonTestCase 3 | from valkeytests.conftest import resource_port_tracker 4 | import logging, os, pathlib 5 | import pytest 6 | from error_handlers import ErrorStringTester 7 | 8 | logging.basicConfig(level=logging.DEBUG) 9 | 10 | class TestRdb(JsonTestCase): 11 | 12 | def setup_data(self): 13 | client = self.server.get_new_client() 14 | client.config_set( 15 | 'json.max-path-limit', DEFAULT_MAX_PATH_LIMIT) 16 | # Need the following line when executing the test against a running Valkey. 17 | # Otherwise, data from previous test cases will interfere current test case. 18 | client.execute_command("FLUSHDB") 19 | 20 | # Load strore sample JSONs. We use strore.json as input to create a document key. Then, use 21 | # strore_compact.json, which does not have indent/space/newline, to verify correctness of serialization. 22 | with open(DEFAULT_STORE_PATH, 'r') as file: 23 | self.data_store = file.read() 24 | assert b'OK' == client.execute_command( 25 | 'JSON.SET', 'store', '.', self.data_store) 26 | 27 | @pytest.fixture(autouse=True) 28 | def setup_test(self, setup): 29 | # Check if we should use external server 30 | use_external = os.environ.get("VALKEY_EXTERNAL_SERVER", "false").lower() == "true" 31 | 32 | if use_external: 33 | # Use external server 34 | external_host = os.environ.get("VALKEY_HOST", "localhost") 35 | external_port = int(os.environ.get("VALKEY_PORT", "6379")) 36 | self.server, self.client = self.create_server( 37 | testdir=self.testdir, 38 | bind_ip=external_host, 39 | port=external_port, 40 | external_server=True 41 | ) 42 | else: 43 | # Original local server setup 44 | server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server" 45 | args = {'loadmodule': os.getenv('MODULE_PATH'), "enable-debug-command": "local", 'enable-protected-configs': 'yes'} 46 | self.server, self.client = self.create_server(testdir=self.testdir, server_path=server_path, args=args) 47 | 48 | self.error_class = ErrorStringTester 49 | self.setup_data() 50 | 51 | def test_rdb_saverestore(self): 52 | """ 53 | Test RDB saving 54 | """ 55 | client = self.server.get_new_client() 56 | assert True == client.execute_command('save') 57 | client.execute_command('FLUSHDB') 58 | assert b'OK' == client.execute_command('DEBUG', 'RELOAD', 'NOSAVE') 59 | -------------------------------------------------------------------------------- /src/json/json_api.h: -------------------------------------------------------------------------------- 1 | /** 2 | * JSON C API 3 | */ 4 | #ifndef VALKEYJSONMODULE_JSON_API_H_ 5 | #define VALKEYJSONMODULE_JSON_API_H_ 6 | 7 | #include 8 | 9 | typedef struct ValkeyModuleCtx ValkeyModuleCtx; 10 | typedef struct ValkeyModuleKey ValkeyModuleKey; 11 | typedef struct ValkeyModuleString ValkeyModuleString; 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | * Is it a JSON key? 19 | */ 20 | int is_json_key(ValkeyModuleCtx *ctx, ValkeyModuleKey *key); 21 | 22 | /** 23 | * Another version of is_json_key, given key name as ValkeyModuleString 24 | */ 25 | int is_json_key2(ValkeyModuleCtx *ctx, ValkeyModuleString *keystr); 26 | 27 | /** 28 | * Get the type of the JSON value at the path. The path is expected to point to a single value. 29 | * If multiple values match the path, only the type of the first one is returned. 30 | * 31 | * @type output param, JSON type. The caller is responsible for freeing the memory. 32 | * @len output param, length of JSON type. 33 | * @return 0 - success, 1 - error 34 | */ 35 | int get_json_value_type(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char *path, 36 | char **type, size_t *len); 37 | 38 | /** 39 | * Get serialized JSON value at the path. The path is expected to point to a single value. 40 | * If multiple values match the path, only the first one is returned. 41 | * 42 | * @value output param, serialized JSON string. The caller is responsible for freeing the memory. 43 | * @len output param, length of JSON string. 44 | * @return 0 - success, 1 - error 45 | */ 46 | int get_json_value(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char *path, 47 | char **value, size_t *len); 48 | 49 | /** 50 | * Get serialized JSON values and JSON types at multiple paths. Each path is expected to point to a single value. 51 | * If multiple values match the path, only the first one is returned. 52 | * 53 | * @values Output param, array of JSON strings. 54 | * The caller is responsible for freeing the memory: the array '*values' as well as all the strings '(*values)[i]'. 55 | * @lengths Output param, array of lengths of each JSON string. 56 | * The caller is responsible for freeing the memory: the array '*lengths'. 57 | * @types Output param, array of types as strings. The caller is responsible for freeing the memory. 58 | * The caller is responsible for freeing the memory: the array '*types' as well as all the strings '(*types)[i]'. 59 | * @type_lengths Output param, array of lengths of each type string. 60 | * The caller is responsible for freeing the memory: the array '*type_lengths'. 61 | * @return 0 - success, 1 - error 62 | */ 63 | int get_json_values_and_types(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char **paths, 64 | const int num_paths, char ***values, size_t **lengths, char ***types, size_t **type_lengths); 65 | 66 | #ifdef __cplusplus 67 | } 68 | #endif 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /tst/unit/module_sim.cc: -------------------------------------------------------------------------------- 1 | #undef NDEBUG 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "json/alloc.h" 16 | #include "json/dom.h" 17 | #include "json/stats.h" 18 | #include "json/selector.h" 19 | #include "module_sim.h" 20 | 21 | // 22 | // Simulate underlying zmalloc stuff, including malloc-size 23 | // 24 | static std::map malloc_sizes; 25 | size_t malloced = 0; 26 | std::string logtext; 27 | 28 | static void *test_malloc(size_t s) { 29 | void *ptr = malloc(s); 30 | assert(malloc_sizes.find(ptr) == malloc_sizes.end()); 31 | malloc_sizes[ptr] = s; 32 | malloced += s; 33 | return ptr; 34 | } 35 | 36 | static size_t test_malloc_size(void *ptr) { 37 | if (!ptr) return 0; 38 | assert(malloc_sizes.find(ptr) != malloc_sizes.end()); 39 | return malloc_sizes[ptr]; 40 | } 41 | 42 | static void test_free(void *ptr) { 43 | if (!ptr) return; 44 | assert(malloc_sizes.find(ptr) != malloc_sizes.end()); 45 | ASSERT_GE(malloced, malloc_sizes[ptr]); 46 | malloced -= malloc_sizes[ptr]; 47 | malloc_sizes.erase(malloc_sizes.find(ptr)); 48 | free(ptr); 49 | } 50 | 51 | static void *test_realloc(void *old_ptr, size_t new_size) { 52 | if (old_ptr == nullptr) return test_malloc(new_size); 53 | assert(malloc_sizes.find(old_ptr) != malloc_sizes.end()); 54 | assert(malloced >= malloc_sizes[old_ptr]); 55 | malloced -= malloc_sizes[old_ptr]; 56 | malloc_sizes.erase(malloc_sizes.find(old_ptr)); 57 | void *new_ptr = realloc(old_ptr, new_size); 58 | assert(malloc_sizes.find(new_ptr) == malloc_sizes.end()); 59 | malloc_sizes[new_ptr] = new_size; 60 | malloced += new_size; 61 | return new_ptr; 62 | } 63 | 64 | std::string test_getLogText() { 65 | std::string result = logtext; 66 | logtext.resize(0); 67 | return result; 68 | } 69 | 70 | static void test_log(ValkeyModuleCtx *ctx, const char *level, const char *fmt, ...) { 71 | (void)ctx; 72 | char buffer[256]; 73 | va_list arg; 74 | va_start(arg, fmt); 75 | int len = vsnprintf(buffer, sizeof(buffer), fmt, arg); 76 | va_end(arg); 77 | std::cerr << "Log(" << level << "): " << std::string(buffer, len) << "\n"; // make visible to ASSERT_EXIT 78 | } 79 | 80 | static void test__assert(const char *estr, const char *file, int line) { 81 | ASSERT_TRUE(0) << "Assert(" << file << ":" << line << "): " << estr; 82 | } 83 | 84 | static long long test_Milliseconds() { 85 | struct timespec t; 86 | clock_gettime(CLOCK_REALTIME, &t); 87 | return (t.tv_sec * 1000) + (t.tv_nsec / 1000000); 88 | } 89 | 90 | void setupValkeyModulePointers() { 91 | ValkeyModule_Alloc = test_malloc; 92 | ValkeyModule_Free = test_free; 93 | ValkeyModule_Realloc = test_realloc; 94 | ValkeyModule_MallocSize = test_malloc_size; 95 | ValkeyModule_Log = test_log; 96 | ValkeyModule__Assert = test__assert; 97 | ValkeyModule_Strdup = strdup; 98 | ValkeyModule_Milliseconds = test_Milliseconds; 99 | memory_traps_control(true); 100 | } 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Valkey JSON 2 | 3 | Valkey-json is a Valkey module written in C++ that provides native JSON (JavaScript Object Notation) support for Valkey. The implementation complies with RFC7159 and ECMA-404 JSON data interchange standards. Users can natively store, query, and modify JSON data structures using the JSONPath query language. The query expressions support advanced capabilities including wildcard selections, filter expressions, array slices, union operations, and recursive searches. 4 | 5 | Valkey-json leverages [RapidJSON](https://rapidjson.org/), a high-performance JSON parser and generator for C++, chosen for its small footprint and exceptional performance and memory efficiency. As a header-only library with no external dependencies, RapidJSON provides robust Unicode support while maintaining a compact memory profile of just 16 bytes per JSON value on most 32/64-bit machines. 6 | 7 | ### Building and testing the module 8 | Valkey JSON uses CMake for its build system. To simplify, a build script is provided. 9 | 10 | To build only valkey-json module, use: 11 | ```text 12 | ./build.sh 13 | ``` 14 | To build and run unit tests, use: 15 | ```text 16 | ./build.sh --unit 17 | ``` 18 | To view the available arguments, use: 19 | ```text 20 | ./build.sh --help 21 | ``` 22 | 23 | The default valkey version is "unstable" for integration tests. To override it, do: 24 | ```text 25 | # Run integration tests with valkey-server (8.1) 26 | SERVER_VERSION=8.1 ./build.sh --integration 27 | ``` 28 | 29 | Custom compiler flags can be passed to the build script via environment variable CFLAGS. For example: 30 | ```text 31 | CFLAGS="-O0 -Wno-unused-function" ./build.sh 32 | ``` 33 | 34 | To run single integration test: 35 | ```text 36 | TEST_PATTERN= ./build.sh --integration 37 | ``` 38 | e.g., 39 | ```text 40 | TEST_PATTERN=test_sanity ./build.sh --integration 41 | TEST_PATTERN=test_rdb.py ./build.sh --integration 42 | ``` 43 | 44 | `ASAN_BUILD` works with any of the build script options and enables memory leak 45 | checks in the integration tests. 46 | 47 | To build the module with ASAN and run tests 48 | ```text 49 | export ASAN_BUILD=true 50 | ./build.sh --integration 51 | ``` 52 | 53 | ## Cleaning 54 | ```text 55 | # Clean build artifacts 56 | ./build.sh --clean 57 | ``` 58 | 59 | ## Load the Module 60 | To load the module on Valkey, use any of the following: 61 | 62 | #### Using valkey.conf: 63 | ``` 64 | 1. Add the following to valkey.conf: 65 | loadmodule /path/to/libjson.so 66 | 2. Start valkey-server: 67 | valkey-server /path/to/valkey.conf 68 | ``` 69 | 70 | #### Starting valkey with --loadmodule option: 71 | ```text 72 | valkey-server --loadmodule /path/to/libjson.so 73 | ``` 74 | 75 | #### Using Valkey command MODULE LOAD: 76 | ``` 77 | 1. Connect to a running Valkey instance using valkey-cli 78 | 2. Execute Valkey command: 79 | MODULE LOAD /path/to/libjson.so 80 | ``` 81 | ## Supported Module Commands 82 | ```text 83 | JSON.ARRAPPEND 84 | JSON.ARRINDEX 85 | JSON.ARRINSERT 86 | JSON.ARRLEN 87 | JSON.ARRPOP 88 | JSON.ARRTRIM 89 | JSON.CLEAR 90 | JSON.DEBUG 91 | JSON.DEL 92 | JSON.FORGET 93 | JSON.GET 94 | JSON.MGET 95 | JSON.MSET 96 | JSON.NUMINCRBY 97 | JSON.NUMMULTBY 98 | JSON.OBJKEYS 99 | JSON.OBJLEN 100 | JSON.RESP 101 | JSON.SET 102 | JSON.STRAPPEND 103 | JSON.STRLEN 104 | JSON.TOGGLE 105 | JSON.TYPE 106 | ``` 107 | -------------------------------------------------------------------------------- /tst/integration/json_test_case.py: -------------------------------------------------------------------------------- 1 | import valkey 2 | import pytest 3 | import os 4 | import logging 5 | import shutil 6 | import time 7 | from valkeytests.valkey_test_case import ValkeyTestCase, ValkeyServerHandle 8 | from valkey import ResponseError 9 | from error_handlers import ErrorStringTester 10 | 11 | 12 | class SimpleTestCase(ValkeyTestCase): 13 | ''' 14 | Simple test case, single server without loading JSON module. 15 | ''' 16 | @pytest.fixture(autouse=True) 17 | def setup_test(self, setup): 18 | use_external = os.environ.get("VALKEY_EXTERNAL_SERVER", "false").lower() == "true" 19 | 20 | if use_external: 21 | # Use external server 22 | external_host = os.environ.get("VALKEY_HOST", "localhost") 23 | external_port = int(os.environ.get("VALKEY_PORT", "6379")) 24 | self.server, self.client = self.create_server( 25 | testdir=self.testdir, 26 | bind_ip=external_host, 27 | port=external_port, 28 | external_server=True 29 | ) 30 | else: 31 | # Original local server 32 | server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server" 33 | self.server, self.client = self.create_server(testdir=self.testdir, server_path=server_path) 34 | 35 | def teardown(self): 36 | if self.is_connected(): 37 | self.client.execute_command("FLUSHALL SYNC") 38 | logging.info("executed FLUSHALL at teardown") 39 | super(SimpleTestCase, self).teardown() 40 | 41 | def is_connected(self): 42 | try: 43 | self.client.ping() 44 | return True 45 | except: 46 | return False 47 | 48 | 49 | class JsonTestCase(SimpleTestCase): 50 | ''' 51 | Base class for JSON test, single server with JSON module loaded. 52 | ''' 53 | 54 | def verify_error_response(self, client, cmd, expected_err_reply): 55 | try: 56 | client.execute_command(cmd) 57 | assert False 58 | except ResponseError as e: 59 | assert_error_msg = f"Actual error message: '{str(e)}' is different from expected error message '{expected_err_reply}'" 60 | assert str(e) == expected_err_reply, assert_error_msg 61 | 62 | @pytest.fixture(autouse=True) 63 | def setup_test(self, setup): 64 | use_external = os.environ.get("VALKEY_EXTERNAL_SERVER", "false").lower() == "true" 65 | 66 | if use_external: 67 | # Use external server 68 | external_host = os.environ.get("VALKEY_HOST", "localhost") 69 | external_port = int(os.environ.get("VALKEY_PORT", "6379")) 70 | self.server, self.client = self.create_server( 71 | testdir=self.testdir, 72 | bind_ip=external_host, 73 | port=external_port, 74 | external_server=True 75 | ) 76 | else: 77 | # Original local server 78 | server_path = f"{os.path.dirname(os.path.realpath(__file__))}/.build/binaries/{os.environ['SERVER_VERSION']}/valkey-server" 79 | args = {'loadmodule': os.getenv('MODULE_PATH'), "enable-debug-command": "local", 'enable-protected-configs': 'yes'} 80 | self.server, self.client = self.create_server(testdir=self.testdir, server_path=server_path, args=args) 81 | 82 | self.error_class = ErrorStringTester 83 | 84 | def teardown(self): 85 | super(JsonTestCase, self).teardown() 86 | -------------------------------------------------------------------------------- /tst/integration/data/webxml.json: -------------------------------------------------------------------------------- 1 | {"web-app": { 2 | "servlet": [ 3 | { 4 | "servlet-name": "cofaxCDS", 5 | "servlet-class": "org.cofax.cds.CDSServlet", 6 | "init-param": { 7 | "configGlossary:installationAt": "Philadelphia, PA", 8 | "configGlossary:adminEmail": "ksm@pobox.com", 9 | "configGlossary:poweredBy": "Cofax", 10 | "configGlossary:poweredByIcon": "/images/cofax.gif", 11 | "configGlossary:staticPath": "/content/static", 12 | "templateProcessorClass": "org.cofax.WysiwygTemplate", 13 | "templateLoaderClass": "org.cofax.FilesTemplateLoader", 14 | "templatePath": "templates", 15 | "templateOverridePath": "", 16 | "defaultListTemplate": "listTemplate.htm", 17 | "defaultFileTemplate": "articleTemplate.htm", 18 | "useJSP": false, 19 | "jspListTemplate": "listTemplate.jsp", 20 | "jspFileTemplate": "articleTemplate.jsp", 21 | "cachePackageTagsTrack": 200, 22 | "cachePackageTagsStore": 200, 23 | "cachePackageTagsRefresh": 60, 24 | "cacheTemplatesTrack": 100, 25 | "cacheTemplatesStore": 50, 26 | "cacheTemplatesRefresh": 15, 27 | "cachePagesTrack": 200, 28 | "cachePagesStore": 100, 29 | "cachePagesRefresh": 10, 30 | "cachePagesDirtyRead": 10, 31 | "searchEngineListTemplate": "forSearchEnginesList.htm", 32 | "searchEngineFileTemplate": "forSearchEngines.htm", 33 | "searchEngineRobotsDb": "WEB-INF/robots.db", 34 | "useDataStore": true, 35 | "dataStoreClass": "org.cofax.SqlDataStore", 36 | "redirectionClass": "org.cofax.SqlRedirection", 37 | "dataStoreName": "cofax", 38 | "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", 39 | "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", 40 | "dataStoreUser": "sa", 41 | "dataStorePassword": "dataStoreTestQuery", 42 | "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", 43 | "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", 44 | "dataStoreInitConns": 10, 45 | "dataStoreMaxConns": 100, 46 | "dataStoreConnUsageLimit": 100, 47 | "dataStoreLogLevel": "debug", 48 | "maxUrlLength": 500}}, 49 | { 50 | "servlet-name": "cofaxEmail", 51 | "servlet-class": "org.cofax.cds.EmailServlet", 52 | "init-param": { 53 | "mailHost": "mail1", 54 | "mailHostOverride": "mail2"}}, 55 | { 56 | "servlet-name": "cofaxAdmin", 57 | "servlet-class": "org.cofax.cds.AdminServlet"}, 58 | 59 | { 60 | "servlet-name": "fileServlet", 61 | "servlet-class": "org.cofax.cds.FileServlet"}, 62 | { 63 | "servlet-name": "cofaxTools", 64 | "servlet-class": "org.cofax.cms.CofaxToolsServlet", 65 | "init-param": { 66 | "templatePath": "toolstemplates/", 67 | "log": 1, 68 | "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", 69 | "logMaxSize": "", 70 | "dataLog": 1, 71 | "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", 72 | "dataLogMaxSize": "", 73 | "removePageCache": "/content/admin/remove?cache=pages&id=", 74 | "removeTemplateCache": "/content/admin/remove?cache=templates&id=", 75 | "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", 76 | "lookInContext": 1, 77 | "adminGroupID": 4, 78 | "betaServer": true}}], 79 | "servlet-mapping": { 80 | "cofaxCDS": "/", 81 | "cofaxEmail": "/cofaxutil/aemail/*", 82 | "cofaxAdmin": "/admin/*", 83 | "fileServlet": "/static/*", 84 | "cofaxTools": "/tools/*"}, 85 | 86 | "taglib": { 87 | "taglib-uri": "cofax.tld", 88 | "taglib-location": "/WEB-INF/tlds/cofax.tld"}}} 89 | -------------------------------------------------------------------------------- /tst/unit/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ######################################### 2 | # Define unit tests 3 | ######################################### 4 | message("tst/unit/CMakeLists.txt: Define unit tests") 5 | 6 | # This is the set of sources for the basic test 7 | file(GLOB_RECURSE UNIT_TEST_SRC "*.cc") 8 | 9 | ######################################### 10 | # Tell Cmake how to run the unit tests 11 | ######################################### 12 | 13 | # A brief philosophical thought, about unit tests: if possible, it's preferable to have all unit 14 | # tests in a single (or a low number of) binary executable. This is disk-space efficient for the 15 | # test suite, avoids unnecessary linking steps, and provides a nice, simple way to interface with 16 | # the test suite (should you need to do so manually, or, for instance, with a debugger). Large 17 | # numbers of test binaries are certainly possible, and in some rare cases are even necessary, but 18 | # don't provide many advantages over a single binary in the average case. 19 | # This file defines a single test executable, "unitTests", which uses the Googletest framework. 20 | 21 | # This defines each unit test and associates it with its sources 22 | add_executable(unitTests ${UNIT_TEST_SRC}) 23 | 24 | if(ENABLE_ASAN) 25 | message(STATUS "Enabling Address Sanitizer for unit tests") 26 | set(GTEST_PROPERTIES "${CMAKE_CXX_FLAGS}") 27 | else() 28 | set(GTEST_PROPERTIES "") 29 | endif() 30 | 31 | # Build with C11 & C++17 32 | set_target_properties( 33 | unitTests 34 | PROPERTIES 35 | C_STANDARD 11 36 | C_STANDARD_REQUIRED ON 37 | CXX_STANDARD 17 38 | CXX_STANDARD_REQUIRED ON 39 | POSITION_INDEPENDENT_CODE ON 40 | COMPILE_FLAGS "${GTEST_PROPERTIES} -Wall -Wextra" 41 | ) 42 | 43 | target_include_directories(unitTests 44 | PRIVATE 45 | ${PROJECT_SOURCE_DIR}/src 46 | ${rapidjson_SOURCE_DIR}/include 47 | ) 48 | 49 | # Add dependency to the code under test. 50 | target_link_libraries(unitTests ${JSON_MODULE_LIB}) 51 | 52 | # Link GoogleTest libraries after fetch 53 | target_link_libraries(unitTests 54 | GTest::gtest_main # Link the main GoogleTest library 55 | GTest::gmock_main # Link GoogleMock 56 | ) 57 | 58 | # This tells CTest about this unit test executable 59 | # The TEST_PREFIX prepends "unit_" to the name of these tests in the output, 60 | # which makes them easier to identify at a glance 61 | # The TEST_LIST settings creates a CMake list of all of the tests in the 62 | # binary. This is useful for, for instance, the set_tests_properties statement 63 | # below 64 | # For more information, see: https://cmake.org/cmake/help/v3.12/module/GoogleTest.html 65 | # To get this to work properly in a cross-compile environment, you need to set up 66 | # CROSSCOMPILING_EMULATOR (see https://cmake.org/cmake/help/v3.12/prop_tgt/CROSSCOMPILING_EMULATOR.html) 67 | # DISCOVERY_TIMEOUT - number of seconds given for Gtest to discover the tests to run, it should 68 | # be big enough so the tests can start on MacOS and can be any number, 59 is just prime number 69 | # close to 1 minute ;) 70 | gtest_discover_tests(unitTests 71 | TEST_PREFIX unit_ 72 | TEST_LIST unit_gtests 73 | DISCOVERY_TIMEOUT 59 74 | ) 75 | 76 | # This tells the CTest harness about how it should treat these tests. For 77 | # instance, you can uncomment the RUN_SERIAL line to force the tests to run 78 | # sequentially (e.g. if the tests are not thread-safe... in most cases, tests 79 | # SHOULD be thread-safe). We also set a high-level timeout: if the test takes 80 | # longer than the specified time, it is killed by the harness and reported as a 81 | # failure. And finally, we provide a "label" that is used by CTest when 82 | # reporting result statistics (e.g. "UnitTests: 72 successes, 3 failures"). 83 | # For more properties that can be set, see: 84 | # https://cmake.org/cmake/help/v3.9/manual/cmake-properties.7.html#test-properties 85 | set_tests_properties(${unit_gtests} PROPERTIES 86 | # RUN_SERIAL 1 87 | TIMEOUT 10 # seconds 88 | LABELS UnitTests 89 | ) 90 | 91 | 92 | add_custom_target(unit 93 | COMMAND ${CMAKE_BINARY_DIR}/tst/unit/unitTests 94 | DEPENDS unitTests 95 | WORKING_DIRECTORY ${CMAKE_BINARY_DIR} 96 | COMMENT "Running unit tests..." 97 | ) -------------------------------------------------------------------------------- /src/rapidjson/stringbuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_STRINGBUFFER_H_ 16 | #define RAPIDJSON_STRINGBUFFER_H_ 17 | 18 | #include 19 | #include 20 | 21 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 22 | #include // std::move 23 | #endif 24 | 25 | #if defined(__clang__) 26 | RAPIDJSON_DIAG_PUSH 27 | RAPIDJSON_DIAG_OFF(c++98-compat) 28 | #endif 29 | 30 | RAPIDJSON_NAMESPACE_BEGIN 31 | 32 | //! Represents an in-memory output stream. 33 | /*! 34 | \tparam Encoding Encoding of the stream. 35 | \tparam Allocator type for allocating memory buffer. 36 | \note implements Stream concept 37 | */ 38 | template 39 | class GenericStringBuffer { 40 | public: 41 | typedef typename Encoding::Ch Ch; 42 | 43 | GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 44 | 45 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 46 | GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} 47 | GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { 48 | if (&rhs != this) 49 | stack_ = std::move(rhs.stack_); 50 | return *this; 51 | } 52 | #endif 53 | 54 | void Put(Ch c) { *stack_.template Push() = c; } 55 | void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } 56 | void Flush() {} 57 | 58 | void Clear() { stack_.Clear(); } 59 | void ShrinkToFit() { 60 | // Push and pop a null terminator. This is safe. 61 | *stack_.template Push() = '\0'; 62 | stack_.ShrinkToFit(); 63 | stack_.template Pop(1); 64 | } 65 | 66 | void Reserve(size_t count) { stack_.template Reserve(count); } 67 | Ch* Push(size_t count) { return stack_.template Push(count); } 68 | Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } 69 | void Pop(size_t count) { stack_.template Pop(count); } 70 | 71 | const Ch* GetString() const { 72 | // Push and pop a null terminator. This is safe. 73 | *stack_.template Push() = '\0'; 74 | stack_.template Pop(1); 75 | 76 | return stack_.template Bottom(); 77 | } 78 | 79 | //! Get the size of string in bytes in the string buffer. 80 | size_t GetSize() const { return stack_.GetSize(); } 81 | 82 | //! Get the length of string in Ch in the string buffer. 83 | size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } 84 | 85 | static const size_t kDefaultCapacity = 256; 86 | mutable internal::Stack stack_; 87 | 88 | private: 89 | // Prohibit copy constructor & assignment operator. 90 | GenericStringBuffer(const GenericStringBuffer&); 91 | GenericStringBuffer& operator=(const GenericStringBuffer&); 92 | }; 93 | 94 | //! String buffer with UTF8 encoding 95 | typedef GenericStringBuffer > StringBuffer; 96 | 97 | template 98 | inline void PutReserve(GenericStringBuffer& stream, size_t count) { 99 | stream.Reserve(count); 100 | } 101 | 102 | template 103 | inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { 104 | stream.PutUnsafe(c); 105 | } 106 | 107 | //! Implement specialized version of PutN() with memset() for better performance. 108 | template<> 109 | inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { 110 | std::memset(stream.stack_.Push(n), c, n * sizeof(c)); 111 | } 112 | 113 | RAPIDJSON_NAMESPACE_END 114 | 115 | #if defined(__clang__) 116 | RAPIDJSON_DIAG_POP 117 | #endif 118 | 119 | #endif // RAPIDJSON_STRINGBUFFER_H_ 120 | -------------------------------------------------------------------------------- /src/rapidjson/license.txt: -------------------------------------------------------------------------------- 1 | Tencent is pleased to support the open source community by making RapidJSON available. 2 | 3 | Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | 5 | If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. 6 | If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. 7 | A copy of the MIT License is included in this file. 8 | 9 | Other dependencies and licenses: 10 | 11 | Open Source Software Licensed Under the BSD License: 12 | -------------------------------------------------------------------- 13 | 14 | The msinttypes r29 15 | Copyright (c) 2006-2013 Alexander Chemeris 16 | All rights reserved. 17 | 18 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 19 | 20 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 21 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 22 | * Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | 26 | Open Source Software Licensed Under the JSON License: 27 | -------------------------------------------------------------------- 28 | 29 | json.org 30 | Copyright (c) 2002 JSON.org 31 | All Rights Reserved. 32 | 33 | JSON_checker 34 | Copyright (c) 2002 JSON.org 35 | All Rights Reserved. 36 | 37 | 38 | Terms of the JSON License: 39 | --------------------------------------------------- 40 | 41 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 42 | 43 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 44 | 45 | The Software shall be used for Good, not Evil. 46 | 47 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 48 | 49 | 50 | Terms of the MIT License: 51 | -------------------------------------------------------------------- 52 | 53 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 54 | 55 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 56 | 57 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 58 | -------------------------------------------------------------------------------- /src/json/util.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This is the utility module, containing shared utility and helper code. 3 | * 4 | * Coding Conventions & Best Practices: 5 | * 1. Every public interface method declared in this file should be prefixed with "jsonutil_". 6 | * 2. Generally speaking, interface methods should not have Valkey module types such as ValkeyModuleCtx 7 | * or ValkeyModuleString, because that would make unit tests hard to write, unless gmock classes 8 | * have been developed. 9 | */ 10 | #ifndef VALKEYJSONMODULE_JSON_UTIL_H_ 11 | #define VALKEYJSONMODULE_JSON_UTIL_H_ 12 | 13 | #include 14 | 15 | extern "C" { 16 | #define VALKEYMODULE_EXPERIMENTAL_API 17 | #include <./include/valkeymodule.h> 18 | } 19 | 20 | typedef enum { 21 | JSONUTIL_SUCCESS = 0, 22 | JSONUTIL_WRONG_NUM_ARGS, 23 | JSONUTIL_JSON_PARSE_ERROR, 24 | JSONUTIL_NX_XX_CONDITION_NOT_SATISFIED, 25 | JSONUTIL_NX_XX_SHOULD_BE_MUTUALLY_EXCLUSIVE, 26 | JSONUTIL_INVALID_JSON_PATH, 27 | JSONUTIL_INVALID_USE_OF_WILDCARD, 28 | JSONUTIL_INVALID_MEMBER_NAME, 29 | JSONUTIL_INVALID_NUMBER, 30 | JSONUTIL_INVALID_IDENTIFIER, 31 | JSONUTIL_INVALID_DOT_SEQUENCE, 32 | JSONUTIL_EMPTY_EXPR_TOKEN, 33 | JSONUTIL_ARRAY_INDEX_NOT_NUMBER, 34 | JSONUTIL_STEP_CANNOT_NOT_BE_ZERO, 35 | JSONUTIL_JSON_PATH_NOT_EXIST, 36 | JSONUTIL_PARENT_ELEMENT_NOT_EXIST, 37 | JSONUTIL_DOCUMENT_KEY_NOT_FOUND, 38 | JSONUTIL_NOT_A_DOCUMENT_KEY, 39 | JSONUTIL_FAILED_TO_DELETE_VALUE, 40 | JSONUTIL_JSON_ELEMENT_NOT_NUMBER, 41 | JSONUTIL_JSON_ELEMENT_NOT_BOOL, 42 | JSONUTIL_JSON_ELEMENT_NOT_STRING, 43 | JSONUTIL_JSON_ELEMENT_NOT_OBJECT, 44 | JSONUTIL_JSON_ELEMENT_NOT_ARRAY, 45 | JSONUTIL_VALUE_NOT_NUMBER, 46 | JSONUTIL_VALUE_NOT_STRING, 47 | JSONUTIL_VALUE_NOT_INTEGER, 48 | JSONUTIL_PATH_SHOULD_BE_AT_THE_END, 49 | JSONUTIL_COMMAND_SYNTAX_ERROR, 50 | JSONUTIL_MULTIPLICATION_OVERFLOW, 51 | JSONUTIL_ADDITION_OVERFLOW, 52 | JSONUTIL_EMPTY_JSON_OBJECT, 53 | JSONUTIL_EMPTY_JSON_ARRAY, 54 | JSONUTIL_INDEX_OUT_OF_ARRAY_BOUNDARIES, 55 | JSONUTIL_UNKNOWN_SUBCOMMAND, 56 | JSONUTIL_FAILED_TO_CREATE_THREAD_SPECIFIC_DATA_KEY, 57 | JSONUTIL_DOCUMENT_SIZE_LIMIT_EXCEEDED, 58 | JSONUTIL_DOCUMENT_PATH_LIMIT_EXCEEDED, 59 | JSONUTIL_PARSER_RECURSION_DEPTH_LIMIT_EXCEEDED, 60 | JSONUTIL_RECURSIVE_DESCENT_TOKEN_LIMIT_EXCEEDED, 61 | JSONUTIL_QUERY_STRING_SIZE_LIMIT_EXCEEDED, 62 | JSONUTIL_CANNOT_INSERT_MEMBER_INTO_NON_OBJECT_VALUE, 63 | JSONUTIL_INVALID_RDB_FORMAT, 64 | JSONUTIL_DOLLAR_CANNOT_APPLY_TO_NON_ROOT, 65 | JSONUTIL_ALLOCATION_FAILURE, 66 | JSONUTIL_KEY_OPEN_ERROR, 67 | JSONUTIL_LAST 68 | } JsonUtilCode; 69 | 70 | typedef struct { 71 | const char *newline; 72 | const char *space; 73 | const char *indent; 74 | } PrintFormat; 75 | 76 | /* Enums for buffer sizes used in conversion of double to json or double to rapidjson */ 77 | enum { BUF_SIZE_DOUBLE_JSON = 32, BUF_SIZE_DOUBLE_RAPID_JSON = 25}; 78 | 79 | /* Get message for a given code. */ 80 | const char *jsonutil_code_to_message(JsonUtilCode code); 81 | 82 | /* Convert a double value to string. This method is used to help serializing numbers to strings. 83 | * Trailing zeros will be removed. For example, 135.250000 will be converted to string 135.25. 84 | */ 85 | size_t jsonutil_double_to_string(const double val, char *double_to_string_buf, size_t len); 86 | 87 | /** 88 | * Convert double to string using the same format as RapidJSON's Writer::WriteDouble does. 89 | */ 90 | size_t jsonutil_double_to_string_rapidjson(const double val, char* double_to_string_buf_rapidjson, size_t len); 91 | 92 | /* Check if a double value is int64. 93 | * If the given double does not equal an integer (int64), return false. 94 | * If the given double is out of range of int64, return false. 95 | */ 96 | bool jsonutil_is_int64(const double a); 97 | 98 | /* Multiple two double numbers with overflow check. 99 | * @param res - OUTPUT parameter, *res stores the result of multiplication. 100 | * @return JSONUTIL_SUCCESS if successful, JSONUTIL_MULTIPLICATION_OVERFLOW if the result overflows. 101 | */ 102 | JsonUtilCode jsonutil_multiply_double(const double a, const double b, double *res); 103 | 104 | /* Multiple two int64 numbers with overflow check. 105 | * @param res - OUTPUT parameter, *res stores the result of multiplication. 106 | * @return JSONUTIL_SUCCESS if successful, JSONUTIL_MULTIPLICATION_OVERFLOW if the result overflows. 107 | */ 108 | JsonUtilCode jsonutil_multiply_int64(const int64_t a, const int64_t b, int64_t *res); 109 | 110 | /* Add two double numbers with overflow check. 111 | * @param res - OUTPUT parameter, *res stores the result of addition. 112 | * @return JSONUTIL_SUCCESS if successful, JSONUTIL_ADDITION_OVERFLOW if the result overflows. 113 | */ 114 | JsonUtilCode jsonutil_add_double(const double a, const double b, double *res); 115 | 116 | /* Add two int64 numbers with overflow check. 117 | * @param res - OUTPUT parameter, *res stores the result of addition. 118 | * @return JSONUTIL_SUCCESS if successful, JSONUTIL_ADDITION_OVERFLOW if the result overflows. 119 | */ 120 | JsonUtilCode jsonutil_add_int64(const int64_t a, const int64_t b, int64_t *res); 121 | 122 | bool jsonutil_is_root_path(const char *json_path); 123 | 124 | #endif // VALKEYJSONMODULE_JSON_UTIL_H_ 125 | -------------------------------------------------------------------------------- /tst/unit/traps_test.cc: -------------------------------------------------------------------------------- 1 | #undef NDEBUG 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "json/alloc.h" 25 | #include "json/dom.h" 26 | #include "json/stats.h" 27 | #include "json/selector.h" 28 | #include "module_sim.h" 29 | 30 | extern size_t hash_function(const char *, size_t); 31 | 32 | /* Since unit tests run outside of Valkey server, we need to map Valkey' 33 | * memory management functions to cstdlib functions. */ 34 | static void SetupAllocFuncs(size_t numShards) { 35 | setupValkeyModulePointers(); 36 | // 37 | // Now setup the KeyTable, the RapidJson library now depends on it 38 | // 39 | KeyTable::Config c; 40 | c.malloc = memory_alloc; 41 | c.free = memory_free; 42 | c.hash = hash_function; 43 | c.numShards = numShards; 44 | keyTable = new KeyTable(c); 45 | } 46 | 47 | class TrapsTest : public ::testing::Test { 48 | protected: 49 | void SetUp() override { 50 | JsonUtilCode rc = jsonstats_init(); 51 | ASSERT_EQ(rc, JSONUTIL_SUCCESS); 52 | SetupAllocFuncs(16); 53 | } 54 | 55 | void TearDown() override { 56 | delete keyTable; 57 | keyTable = nullptr; 58 | } 59 | }; 60 | 61 | // 62 | // See if we can startup and shutdown with no failures 63 | // 64 | TEST_F(TrapsTest, sanity) { 65 | void *ptr = dom_alloc(15); 66 | dom_free(ptr); 67 | } 68 | 69 | enum JTYPE { 70 | JT_BOOLEAN, 71 | JT_INTEGER, 72 | JT_SHORT_STRING, 73 | JT_LONG_STRING, 74 | JT_SHORT_DOUBLE, 75 | JT_LONG_DOUBLE, 76 | JT_ARRAY, 77 | JT_OBJECT, 78 | JT_OBJECT_HT, 79 | JT_NUM_TYPES 80 | }; 81 | 82 | static void makeValue(JValue *v, JTYPE jt) { 83 | std::string json; 84 | switch (jt) { 85 | case JT_BOOLEAN: 86 | json = "true"; 87 | break; 88 | case JT_INTEGER: 89 | json = "1"; 90 | break; 91 | case JT_SHORT_STRING: 92 | json = "\"short\""; 93 | break; 94 | case JT_LONG_STRING: 95 | json = "\"string of length large\""; 96 | break; 97 | case JT_SHORT_DOUBLE: 98 | json = "1.2"; 99 | break; 100 | case JT_LONG_DOUBLE: 101 | json = "1.23456789101112"; 102 | break; 103 | case JT_ARRAY: 104 | json = "[1,2,3,4,5]"; 105 | break; 106 | case JT_OBJECT: 107 | json = "{\"a\":1}"; 108 | break; 109 | case JT_OBJECT_HT: 110 | json = "{"; 111 | for (auto s = 0; s < 1000; ++s) { 112 | if (s != 0) json += ','; 113 | json += '\"'; 114 | json += std::to_string(s); 115 | json += "\":1"; 116 | } 117 | json += '}'; 118 | break; 119 | default: 120 | ASSERT_TRUE(0); 121 | } 122 | JParser parser; 123 | *v = parser.Parse(json.c_str(), json.length()).GetJValue(); 124 | } 125 | 126 | // 127 | // Test that keys properly honor corruption 128 | // 129 | TEST_F(TrapsTest, handle_corruption) { 130 | for (auto corruption : {CORRUPT_PREFIX, CORRUPT_LENGTH, CORRUPT_SUFFIX}) { 131 | for (auto jt : {JT_OBJECT, JT_OBJECT_HT}) { 132 | JValue *v = new JValue; 133 | makeValue(v, jt); 134 | auto first = v->MemberBegin(); 135 | auto trap_pointer = &*(first->name); 136 | memory_corrupt_memory(trap_pointer, corruption); 137 | // 138 | // Serialize this object 139 | // 140 | rapidjson::StringBuffer oss; 141 | ASSERT_EXIT(dom_serialize_value(*v, nullptr, oss), testing::ExitedWithCode(1), "Validation Failure"); 142 | // 143 | // Destruct it 144 | // 145 | ASSERT_EXIT(delete v, testing::ExitedWithCode(1), "Validation Failure"); 146 | // 147 | // Cleanup 148 | // 149 | memory_uncorrupt_memory(trap_pointer, corruption); 150 | delete v; 151 | } 152 | } 153 | } 154 | 155 | // 156 | // Test out the JValue validate and dump functions 157 | // 158 | TEST_F(TrapsTest, jvalue_validation) { 159 | std::string json = 160 | "{ \"a\":1, \"b\":[1,2,\"this is a long string\",\"shortstr\",false,true,1.0,1.23456789012345,null]}"; 161 | JParser parser; 162 | JValue *v = new JValue; 163 | *v = parser.Parse(json.c_str(), json.length()).GetJValue(); 164 | std::ostringstream os; 165 | DumpRedactedJValue(os, *v); 166 | std::cerr << os.str() << "\n"; 167 | delete v; 168 | } 169 | 170 | // 171 | // Test Log Stream 172 | // 173 | TEST_F(TrapsTest, test_log_stream) { 174 | JValue v, v0; 175 | v.SetArray(); 176 | v.PushBack(v0, allocator); 177 | DumpRedactedJValue(v, nullptr, "level"); 178 | std::string log = test_getLogText(); 179 | std::cerr << log; 180 | } 181 | -------------------------------------------------------------------------------- /src/json/stats.h: -------------------------------------------------------------------------------- 1 | /** 2 | * The STATS module's main responsibility is to track memory usage at the level of the custom memory allocator, 3 | * which provides the capability of tracking memory usage per JSON write operation. When a JSON key is mutated, 4 | * we call API jsonstats_begin_track_mem() and jsonstats_end_track_mem() at the beginning and end of the write 5 | * operation respectively, to calculate the delta of the memory usage. Then, we update the document size meta data. 6 | * 7 | * The module also maintains the following global info metrics: 8 | * json_total_memory_bytes: total memory allocated to JSON objects 9 | * json_num_documents: number of document keys in Valkey 10 | */ 11 | #ifndef VALKEYJSONMODULE_JSON_STATS_H_ 12 | #define VALKEYJSONMODULE_JSON_STATS_H_ 13 | 14 | #include "json/dom.h" 15 | 16 | typedef enum { 17 | JSONSTATS_READ = 0, 18 | JSONSTATS_INSERT, 19 | JSONSTATS_UPDATE, 20 | JSONSTATS_DELETE 21 | } JsonCommandType; 22 | 23 | /* Initialize statistics counters and thread local storage (TLS) keys. */ 24 | JsonUtilCode jsonstats_init(); 25 | 26 | /* Begin tracking memory usage. 27 | * @return value of the thread local counter. 28 | */ 29 | int64_t jsonstats_begin_track_mem(); 30 | 31 | /* End tracking memory usage. 32 | * @param begin_val - previous saved thread local value that is returned from jsonstats_begin_track_memory(). 33 | * @return delta of used memory 34 | */ 35 | int64_t jsonstats_end_track_mem(const int64_t begin_val); 36 | 37 | /* Get the total memory allocated to JSON objects. */ 38 | unsigned long long jsonstats_get_used_mem(); 39 | 40 | /* The following two methods are invoked by the DOM memory allocator upon every malloc/free/realloc. 41 | * Two memory counters are updated: A global atomic counter and a thread local counter (per thread). 42 | */ 43 | void jsonstats_increment_used_mem(size_t delta); 44 | void jsonstats_decrement_used_mem(size_t delta); 45 | 46 | // get counters 47 | unsigned long long jsonstats_get_num_doc_keys(); 48 | 49 | unsigned long long jsonstats_get_max_depth_ever_seen(); 50 | void jsonstats_update_max_depth_ever_seen(const size_t max_depth); 51 | unsigned long long jsonstats_get_max_size_ever_seen(); 52 | 53 | unsigned long long jsonstats_get_defrag_count(); 54 | void jsonstats_increment_defrag_count(); 55 | 56 | unsigned long long jsonstats_get_defrag_bytes(); 57 | void jsonstats_increment_defrag_bytes(const size_t amount); 58 | 59 | // updating stats on read/insert/update/delete operation 60 | void jsonstats_update_stats_on_read(const size_t fetched_val_size); 61 | void jsonstats_update_stats_on_insert(JDocument *doc, const bool is_delete_doc_key, const size_t orig_size, 62 | const size_t new_size, const size_t inserted_val_size); 63 | void jsonstats_update_stats_on_update(JDocument *doc, const size_t orig_size, const size_t new_size, 64 | const size_t input_json_size); 65 | void jsonstats_update_stats_on_delete(JDocument *doc, const bool is_delete_doc_key, const size_t orig_size, 66 | const size_t new_size, const size_t deleted_val_size); 67 | 68 | // helper methods for printing histograms into C string 69 | void jsonstats_sprint_hist_buckets(char *buf, const size_t buf_size); 70 | void jsonstats_sprint_doc_hist(char *buf, const size_t buf_size); 71 | void jsonstats_sprint_read_hist(char *buf, const size_t buf_size); 72 | void jsonstats_sprint_insert_hist(char *buf, const size_t buf_size); 73 | void jsonstats_sprint_update_hist(char *buf, const size_t buf_size); 74 | void jsonstats_sprint_delete_hist(char *buf, const size_t buf_size); 75 | 76 | /* Given a size (bytes), find the histogram bucket index using binary search. 77 | */ 78 | uint32_t jsonstats_find_bucket(size_t size); 79 | 80 | 81 | /* JSON logical statistics. 82 | * Used for internal tracking of elements for Skyhook Billing. 83 | * Using a similar structure to JsonStats. 84 | * We don't track the logical bytes themselves here as they are tracked by Skyhook Metering. 85 | * We are using size_t to match Valkey Module API for Data Metering. 86 | */ 87 | typedef struct _LogicalStats { 88 | std::atomic_size_t boolean_count; // 16 bytes 89 | std::atomic_size_t number_count; // 16 bytes 90 | std::atomic_size_t sum_extra_numeric_chars; // 1 byte per char 91 | std::atomic_size_t string_count; // 16 bytes 92 | std::atomic_size_t sum_string_chars; // 1 byte per char 93 | std::atomic_size_t null_count; // 16 bytes 94 | std::atomic_size_t array_count; // 16 bytes 95 | std::atomic_size_t sum_array_elements; // internal metric 96 | std::atomic_size_t object_count; // 16 bytes 97 | std::atomic_size_t sum_object_members; // internal metric 98 | std::atomic_size_t sum_object_key_chars; // 1 byte per char 99 | 100 | void reset() { 101 | boolean_count = 0; 102 | number_count = 0; 103 | sum_extra_numeric_chars = 0; 104 | string_count = 0; 105 | sum_string_chars = 0; 106 | null_count = 0; 107 | array_count = 0; 108 | sum_array_elements = 0; 109 | object_count = 0; 110 | sum_object_members = 0; 111 | sum_object_key_chars = 0; 112 | } 113 | } LogicalStats; 114 | extern LogicalStats logical_stats; 115 | 116 | #define DOUBLE_CHARS_CUTOFF 24 117 | 118 | #endif // VALKEYJSONMODULE_JSON_STATS_H_ 119 | -------------------------------------------------------------------------------- /src/json/json_api.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "json/json_api.h" 7 | #include "json/dom.h" 8 | #include "json/memory.h" 9 | 10 | extern ValkeyModuleType* DocumentType; 11 | 12 | int is_json_key(ValkeyModuleCtx *ctx, ValkeyModuleKey *key) { 13 | VALKEYMODULE_NOT_USED(ctx); 14 | if (key == nullptr || ValkeyModule_KeyType(key) == VALKEYMODULE_KEYTYPE_EMPTY) return 0; 15 | return (ValkeyModule_ModuleTypeGetType(key) == DocumentType? 1: 0); 16 | } 17 | 18 | int is_json_key2(ValkeyModuleCtx *ctx, ValkeyModuleString *keystr) { 19 | ValkeyModuleKey *key = static_cast(ValkeyModule_OpenKey(ctx, keystr, VALKEYMODULE_READ)); 20 | int is_json = is_json_key(ctx, key); 21 | ValkeyModule_CloseKey(key); 22 | return is_json; 23 | } 24 | 25 | static JDocument* get_json_document(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len) { 26 | ValkeyModuleString *keystr = ValkeyModule_CreateString(ctx, keyname, key_len); 27 | ValkeyModuleKey *key = static_cast(ValkeyModule_OpenKey(ctx, keystr, VALKEYMODULE_READ)); 28 | if (!is_json_key(ctx, key)) { 29 | ValkeyModule_CloseKey(key); 30 | ValkeyModule_FreeString(ctx, keystr); 31 | return nullptr; 32 | } 33 | JDocument *doc = static_cast(ValkeyModule_ModuleTypeGetValue(key)); 34 | ValkeyModule_CloseKey(key); 35 | ValkeyModule_FreeString(ctx, keystr); 36 | return doc; 37 | } 38 | 39 | int get_json_value_type(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char *path, 40 | char **type, size_t *len) { 41 | *type = nullptr; 42 | *len = 0; 43 | JDocument *doc = get_json_document(ctx, keyname, key_len); 44 | if (doc == nullptr) return -1; 45 | 46 | jsn::vector vec; 47 | bool is_v2_path; 48 | JsonUtilCode rc = dom_value_type(doc, path, vec, is_v2_path); 49 | if (rc != JSONUTIL_SUCCESS || vec.empty()) return -1; 50 | *type = static_cast(ValkeyModule_Alloc(vec[0].length() + 1)); 51 | *len = vec[0].length(); 52 | snprintf(*type, *len + 1, "%s", vec[0].c_str()); 53 | return 0; 54 | } 55 | 56 | int get_json_value(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char *path, 57 | char **value, size_t *len) { 58 | *value = nullptr; 59 | *len = 0; 60 | JDocument *doc = get_json_document(ctx, keyname, key_len); 61 | if (doc == nullptr) return -1; 62 | 63 | rapidjson::StringBuffer buf; 64 | JsonUtilCode rc = dom_get_value_as_str(doc, path, nullptr, buf, false); 65 | if (rc != JSONUTIL_SUCCESS) return -1; 66 | *len = buf.GetLength(); 67 | *value = static_cast(ValkeyModule_Alloc(*len + 1)); 68 | snprintf(*value, *len + 1, "%s", buf.GetString()); 69 | return 0; 70 | } 71 | 72 | int get_json_values_and_types(ValkeyModuleCtx *ctx, const char *keyname, const size_t key_len, const char **paths, 73 | const int num_paths, char ***values, size_t **lengths, char ***types, size_t **type_lengths) { 74 | ValkeyModule_Assert(values != nullptr); 75 | ValkeyModule_Assert(lengths != nullptr); 76 | *values = nullptr; 77 | *lengths = nullptr; 78 | if (types != nullptr) *types = nullptr; 79 | if (type_lengths != nullptr) *type_lengths = nullptr; 80 | JDocument *doc = get_json_document(ctx, keyname, key_len); 81 | if (doc == nullptr) return -1; 82 | 83 | *values = static_cast(ValkeyModule_Alloc(num_paths * sizeof(char *))); 84 | *lengths = static_cast(ValkeyModule_Alloc(num_paths * sizeof(size_t))); 85 | memset(*values, 0, num_paths * sizeof(char *)); 86 | memset(*lengths, 0, num_paths * sizeof(size_t)); 87 | for (int i = 0; i < num_paths; i++) { 88 | rapidjson::StringBuffer buf; 89 | JsonUtilCode rc = dom_get_value_as_str(doc, paths[i], nullptr, buf, false); 90 | if (rc == JSONUTIL_SUCCESS) { 91 | (*lengths)[i] = buf.GetLength(); 92 | (*values)[i] = static_cast(ValkeyModule_Alloc((*lengths)[i] + 1)); 93 | snprintf((*values)[i], (*lengths)[i] + 1, "%s", buf.GetString()); 94 | } else { 95 | (*values)[i] = nullptr; 96 | } 97 | } 98 | 99 | if (types != nullptr) { 100 | ValkeyModule_Assert(type_lengths != nullptr); 101 | 102 | *types = static_cast(ValkeyModule_Alloc(num_paths * sizeof(char *))); 103 | *type_lengths = static_cast(ValkeyModule_Alloc(num_paths * sizeof(size_t))); 104 | memset(*types, 0, num_paths * sizeof(char *)); 105 | memset(*type_lengths, 0, num_paths * sizeof(size_t)); 106 | for (int i = 0; i< num_paths; i++) { 107 | jsn::vector vec; 108 | bool is_v2_path; 109 | JsonUtilCode rc = dom_value_type(doc, paths[i], vec, is_v2_path); 110 | if (rc == JSONUTIL_SUCCESS && !vec.empty()) { 111 | (*type_lengths)[i] = vec[0].length(); 112 | (*types)[i] = static_cast(ValkeyModule_Alloc((*type_lengths)[i] + 1)); 113 | snprintf((*types)[i], (*type_lengths)[i] + 1, "%s", vec[0].c_str()); 114 | } else { 115 | (*types)[i] = nullptr; 116 | } 117 | } 118 | } 119 | return 0; 120 | } 121 | -------------------------------------------------------------------------------- /src/json/memory.h: -------------------------------------------------------------------------------- 1 | /** 2 | */ 3 | #ifndef VALKEYJSONMODULE_MEMORY_H_ 4 | #define VALKEYJSONMODULE_MEMORY_H_ 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | // 16 | // Trap implementation 17 | // 18 | // Memory traps are a diagnostic tool intended to catch some categories of memory usage errors. 19 | // 20 | // The Trap system is conceptually a shim layer between the client application and the lower level memory allocator. 21 | // Traps operate by adding to each memory allocation a prefix and a suffix. The prefix and suffix contain known 22 | // data patterns and some internal trap metadata. Subsequent memory operations validate the correctness of the 23 | // prefix and suffix. A special interface is provided to allow any client application to voluntarily request 24 | // memory validation -- presumably before utilizing the underlying memory. 25 | // 26 | // This strategy should catch at least three classes of memory corruption: 27 | // 28 | // (1) double free of memory. 29 | // (2) writes off the end of memory (just the prev/next word, not WAAAAY off the end of memory) 30 | // (3) dangling pointer to previously freed memory (this relies on voluntary memory validation) 31 | // 32 | // Traps can be dynamically enabled/disabled, provided that there is no outstanding memory allocation. 33 | // 34 | 35 | // 36 | // All functions in the module (outsize of memory.cc) should use these to allocate memory 37 | // instead of the ValkeyModule_xxxx functions. 38 | // 39 | extern void *(*memory_alloc)(size_t size); 40 | extern void (*memory_free)(void *ptr); 41 | extern void *(*memory_realloc)(void *orig_ptr, size_t new_size); 42 | extern size_t (*memory_allocsize)(void *ptr); 43 | 44 | // 45 | // Total memory usage. 46 | // 47 | // (1) Includes dom_alloc memory usage. dom_alloc tracks JSON data that's associated with a document 48 | // (2) Includes KeyTable usage, i.e., JSON data that's shared across documents 49 | // (3) Includes STL library allocations 50 | // 51 | extern size_t memory_usage(); 52 | 53 | // 54 | // Are traps enabled? 55 | // 56 | inline bool memory_traps_enabled() { 57 | extern bool memoryTrapsEnabled; 58 | return memoryTrapsEnabled; 59 | } 60 | 61 | // 62 | // External Interface to traps logic 63 | // 64 | // Enables/Disable traps. This can fail if there's outstanding allocated memory. 65 | // 66 | // return true => operation was successful. 67 | // return false => operation failed (there's outstanding memory) 68 | // 69 | bool memory_traps_control(bool enable); 70 | 71 | bool memory_validate_ptr(const void *ptr, bool crashOnError = true); 72 | // 73 | // This version validates memory, but crashes on an invalid pointer 74 | // 75 | template 76 | static inline t *MEMORY_VALIDATE(t *ptr, bool validate = true) { 77 | extern bool memoryTrapsEnabled; 78 | if (memoryTrapsEnabled && validate) memory_validate_ptr(ptr, true); 79 | return ptr; 80 | } 81 | 82 | // 83 | // This version validates memory, but doesn't crash 84 | // 85 | template 86 | static inline bool IS_VALID_MEMORY(t *ptr) { 87 | return memory_validate_ptr(ptr, false); 88 | } 89 | 90 | // 91 | // Classes for STL Containers that utilize memory usage and trap logic. 92 | // 93 | namespace jsn 94 | { 95 | // 96 | // Our custom allocator 97 | // 98 | template class stl_allocator { 99 | public: 100 | using value_type = T; 101 | stl_allocator() = default; 102 | stl_allocator(const stl_allocator&) noexcept = default; 103 | stl_allocator(stl_allocator&&) noexcept = default; 104 | template 105 | constexpr stl_allocator(const stl_allocator&) noexcept {} 106 | 107 | T* allocate(std::size_t n) { 108 | return static_cast(memory_alloc(n * sizeof(T))); 109 | } 110 | 111 | void deallocate(T* p, std::size_t n) { 112 | (void)n; 113 | memory_free(p); 114 | } 115 | 116 | private: 117 | static void* memory_alloc(std::size_t size) { 118 | return std::malloc(size); 119 | } 120 | 121 | static void memory_free(void* p) { 122 | std::free(p); 123 | } 124 | }; 125 | 126 | template 127 | bool operator==(const stl_allocator&, const stl_allocator&) { return true; } 128 | 129 | template 130 | bool operator!=(const stl_allocator&, const stl_allocator&) { return false; } 131 | 132 | template using vector = std::vector>; 133 | 134 | template> using set = std::set>; 135 | 136 | template, class KeyEqual = std::equal_to> 137 | using unordered_set = std::unordered_set>; 138 | 139 | typedef std::basic_string, stl_allocator> string; 140 | typedef std::basic_stringstream, stl_allocator> stringstream; 141 | 142 | } // namespace jsn 143 | 144 | // custom specialization of std::hash can be injected in namespace std 145 | template<> 146 | struct std::hash 147 | { 148 | std::size_t operator()(const jsn::string& s) const noexcept { 149 | return std::hash{}(std::string_view(s.c_str(), s.length())); 150 | } 151 | }; 152 | 153 | // 154 | // Everything below this line is private to this module, it's here for usage by unit tests 155 | // 156 | 157 | typedef enum MEMORY_TRAPS_CORRUPTION { 158 | CORRUPT_PREFIX, 159 | CORRUPT_LENGTH, 160 | CORRUPT_SUFFIX 161 | } memTrapsCorruption_t; 162 | 163 | void memory_corrupt_memory(const void *ptr, memTrapsCorruption_t corrupt); 164 | void memory_uncorrupt_memory(const void *ptr, memTrapsCorruption_t corrupt); 165 | 166 | #endif // VALKEYJSONMODULE_MEMORY_H_ 167 | -------------------------------------------------------------------------------- /tst/unit/hashtable_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "json/dom.h" 18 | #include "json/alloc.h" 19 | #include "json/stats.h" 20 | #include "json/keytable.h" 21 | #include "module_sim.h" 22 | 23 | // Cheap, predictable hash 24 | static size_t hash1(const char *ptr, size_t len) { 25 | (void)ptr; 26 | return len; 27 | } 28 | 29 | class HashTableTest : public ::testing::Test { 30 | protected: 31 | void SetUp() override { 32 | } 33 | size_t original_malloced; 34 | void TearDown() override { 35 | if (keyTable) { 36 | malloced = original_malloced; 37 | EXPECT_EQ(keyTable->validate(), ""); 38 | } 39 | delete keyTable; 40 | } 41 | void Setup1(size_t numShards = 1, size_t htsize = 0, size_t (*h)(const char *, size_t) = hash1) { 42 | setupValkeyModulePointers(); 43 | KeyTable::Config c; 44 | c.malloc = dom_alloc; 45 | c.free = dom_free; 46 | c.hash = h; 47 | c.numShards = numShards; 48 | keyTable = new KeyTable(c); 49 | rapidjson::hashTableFactors.minHTSize = htsize; 50 | original_malloced = malloced; 51 | malloced = 0; // Ignore startup memory consumption 52 | } 53 | }; 54 | 55 | TEST_F(HashTableTest, simple) { 56 | Setup1(); 57 | { 58 | JValue v; 59 | v.SetObject(); 60 | v.AddMember(JValue("True"), JValue(true), allocator); 61 | EXPECT_EQ(v.MemberCount(), 1u); 62 | EXPECT_TRUE(v["True"].IsBool()); 63 | EXPECT_GT(malloced, 0); 64 | } 65 | EXPECT_EQ(malloced, 0); 66 | } 67 | 68 | static JValue makeKey(size_t i) { 69 | return std::move(JValue().SetString(std::to_string(i), allocator)); 70 | } 71 | 72 | static JValue makeArray(size_t sz, size_t offset = 0) { 73 | JValue j; 74 | j.SetArray(); 75 | for (size_t i = 0; i < sz; ++i) { 76 | j.PushBack(JValue(static_cast(i + offset)), allocator); 77 | } 78 | return j; 79 | } 80 | 81 | static JValue makeArrayArray(size_t p, size_t q) { 82 | JValue j = makeArray(p); 83 | for (size_t i = 0; i < p; ++i) { 84 | j[i] = makeArray(q, i); 85 | } 86 | return j; 87 | } 88 | 89 | TEST_F(HashTableTest, checkeq) { 90 | Setup1(); 91 | for (size_t i : {0, 1, 10}) { 92 | ASSERT_EQ(makeArrayArray(i, i), makeArrayArray(i, i)); 93 | } 94 | } 95 | 96 | TEST_F(HashTableTest, insertAndRemoveMany) { 97 | Setup1(1, 5); 98 | for (size_t sz : {10, 50, 100}) { 99 | EXPECT_EQ(malloced, 0); 100 | rapidjson::hashTableStats.reset(); 101 | { 102 | JValue v; 103 | v.SetObject(); 104 | EXPECT_EQ(v.Validate(), ""); 105 | for (size_t i = 0; i < sz; ++i) { 106 | v.AddMember(makeKey(i), makeArrayArray(i, i), allocator); 107 | EXPECT_EQ(v.Validate(), ""); 108 | } 109 | EXPECT_EQ(v.MemberCount(), sz); 110 | EXPECT_GT(rapidjson::hashTableStats.rehashUp, 0); 111 | EXPECT_EQ(rapidjson::hashTableStats.convertToHT, 1); 112 | auto s = keyTable->getStats(); 113 | EXPECT_EQ(s.size, sz); 114 | for (size_t i = 0; i < sz; ++i) EXPECT_EQ(v[makeKey(i)], makeArrayArray(i, i)); 115 | for (size_t i = 0; i < sz; ++i) { 116 | v.RemoveMember(makeKey(i)); 117 | EXPECT_EQ(v.Validate(), ""); 118 | } 119 | EXPECT_GT(rapidjson::hashTableStats.rehashDown, 0); 120 | EXPECT_EQ(v.MemberCount(), 0); 121 | s = keyTable->getStats(); 122 | EXPECT_EQ(s.size, 0); // All entries should be gone. 123 | } 124 | EXPECT_EQ(malloced, 0); 125 | } 126 | } 127 | 128 | TEST_F(HashTableTest, SetObjectRawHT) { 129 | Setup1(); 130 | std::ostringstream os; 131 | os << "{\"a\":1"; 132 | for (size_t i = 0; i < 100; ++i) os << ",\"" << i << "\":" << i; 133 | os << "}"; 134 | JDocument *doc; 135 | JsonUtilCode rc = dom_parse(nullptr, os.str().c_str(), os.str().size(), &doc); 136 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 137 | EXPECT_EQ(rapidjson::hashTableStats.reserveHT, 1); 138 | rapidjson::StringBuffer oss; 139 | dom_serialize(doc, nullptr, oss); 140 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 141 | EXPECT_EQ(oss.GetString(), os.str()); 142 | 143 | dom_free_doc(doc); 144 | auto s = keyTable->getStats(); 145 | EXPECT_EQ(s.size, 0); // All entries should be gone. 146 | EXPECT_EQ(malloced, 0); 147 | } 148 | 149 | TEST_F(HashTableTest, CopyMembers) { 150 | Setup1(1, 5); 151 | for (size_t sz : {10, 50, 100}) { 152 | rapidjson::hashTableStats.reset(); 153 | JValue v; 154 | v.SetObject(); 155 | EXPECT_EQ(v.Validate(), ""); 156 | for (size_t i = 0; i < sz; ++i) { 157 | v.AddMember(makeKey(i), makeArrayArray(i, i), allocator); 158 | EXPECT_EQ(v.Validate(), ""); 159 | } 160 | EXPECT_EQ(v.MemberCount(), sz); 161 | EXPECT_GT(rapidjson::hashTableStats.rehashUp, 0); 162 | EXPECT_EQ(rapidjson::hashTableStats.convertToHT, 1); 163 | auto s = keyTable->getStats(); 164 | EXPECT_EQ(s.size, sz); 165 | EXPECT_EQ(s.handles, sz); 166 | { 167 | rapidjson::hashTableStats.reset(); 168 | JValue v2(v, allocator); // Invokes copymembers 169 | EXPECT_EQ(v2.Validate(), ""); 170 | EXPECT_EQ(v2.MemberCount(), sz); 171 | EXPECT_EQ(rapidjson::hashTableStats.rehashUp, 0); 172 | EXPECT_EQ(rapidjson::hashTableStats.rehashDown, 0); 173 | EXPECT_EQ(rapidjson::hashTableStats.convertToHT, 0); 174 | s = keyTable->getStats(); 175 | EXPECT_EQ(s.size, sz); 176 | EXPECT_EQ(s.handles, sz*2); 177 | for (size_t i = 0; i < sz; ++i) { 178 | EXPECT_EQ(v[makeKey(i)].GetArray(), makeArrayArray(i, i)); 179 | EXPECT_EQ(v2[makeKey(i)].GetArray(), makeArrayArray(i, i)); 180 | } 181 | } 182 | } 183 | EXPECT_EQ(malloced, 0); 184 | } 185 | 186 | // 187 | // Test that hash tables > 2^19 are properly handled. 188 | // 189 | TEST_F(HashTableTest, DistributionTest) { 190 | extern size_t hash_function(const char *, size_t); 191 | Setup1(1, 0, hash_function); 192 | enum { TABLE_SIZE_BITS = 22 }; // LOG2(Table Size) 193 | enum { TABLE_SIZE = 1ull << TABLE_SIZE_BITS }; 194 | JValue v; 195 | v.SetObject(); 196 | for (size_t i = 0; i < TABLE_SIZE; ++i) { 197 | v.AddMember(makeKey(i), JValue(true), allocator); 198 | } 199 | // 200 | // Now, compute the distribution stats, make sure the longest run is sufficiently small 201 | // 202 | std::map runs; 203 | v.getObjectDistribution(runs, 5); 204 | // std::cout << "Dist:"; 205 | // for (auto& x : runs) std::cout << x.first << ":" << x.second << ","; 206 | // std::cout << std::endl; 207 | ASSERT_NE(runs.size(), 0u); 208 | EXPECT_LT(runs.rbegin()->first, 0.0001 * TABLE_SIZE); 209 | } 210 | -------------------------------------------------------------------------------- /src/json/util.cc: -------------------------------------------------------------------------------- 1 | #include "json/util.h" 2 | #include "json/dom.h" 3 | #include "json/alloc.h" 4 | #include 5 | #include 6 | #include "json/rapidjson_includes.h" 7 | 8 | const char *jsonutil_code_to_message(JsonUtilCode code) { 9 | switch (code) { 10 | case JSONUTIL_SUCCESS: 11 | case JSONUTIL_WRONG_NUM_ARGS: 12 | case JSONUTIL_NX_XX_CONDITION_NOT_SATISFIED: 13 | // only used as code, no message needed 14 | break; 15 | case JSONUTIL_JSON_PARSE_ERROR: return "SYNTAXERR Failed to parse JSON string due to syntax error"; 16 | case JSONUTIL_NX_XX_SHOULD_BE_MUTUALLY_EXCLUSIVE: 17 | return "SYNTAXERR Option NX and XX should be mutually exclusive"; 18 | case JSONUTIL_INVALID_JSON_PATH: return "SYNTAXERR Invalid JSON path"; 19 | case JSONUTIL_INVALID_MEMBER_NAME: return "SYNTAXERR Invalid object member name"; 20 | case JSONUTIL_INVALID_NUMBER: return "SYNTAXERR Invalid number"; 21 | case JSONUTIL_INVALID_IDENTIFIER: return "SYNTAXERR Invalid identifier"; 22 | case JSONUTIL_INVALID_DOT_SEQUENCE: return "SYNTAXERR Invalid dot sequence"; 23 | case JSONUTIL_EMPTY_EXPR_TOKEN: return "SYNTAXERR Expression token cannot be empty"; 24 | case JSONUTIL_ARRAY_INDEX_NOT_NUMBER: return "SYNTAXERR Array index is not a number"; 25 | case JSONUTIL_STEP_CANNOT_NOT_BE_ZERO: return "SYNTAXERR Step in the slice cannot be zero"; 26 | case JSONUTIL_INVALID_USE_OF_WILDCARD: return "ERR Invalid use of wildcard"; 27 | case JSONUTIL_JSON_PATH_NOT_EXIST: return "NONEXISTENT JSON path does not exist"; 28 | case JSONUTIL_PARENT_ELEMENT_NOT_EXIST: return "NONEXISTENT Parent element does not exist"; 29 | case JSONUTIL_DOCUMENT_KEY_NOT_FOUND: return "NONEXISTENT Document key does not exist"; 30 | case JSONUTIL_NOT_A_DOCUMENT_KEY: return "WRONGTYPE Not a JSON document key"; 31 | case JSONUTIL_FAILED_TO_DELETE_VALUE: return "OPFAIL Failed to delete JSON value"; 32 | case JSONUTIL_JSON_ELEMENT_NOT_NUMBER: return "WRONGTYPE JSON element is not a number"; 33 | case JSONUTIL_JSON_ELEMENT_NOT_BOOL: return "WRONGTYPE JSON element is not a bool"; 34 | case JSONUTIL_JSON_ELEMENT_NOT_STRING: return "WRONGTYPE JSON element is not a string"; 35 | case JSONUTIL_JSON_ELEMENT_NOT_OBJECT: return "WRONGTYPE JSON element is not an object"; 36 | case JSONUTIL_JSON_ELEMENT_NOT_ARRAY: return "WRONGTYPE JSON element is not an array"; 37 | case JSONUTIL_VALUE_NOT_NUMBER: return "WRONGTYPE Value is not a number"; 38 | case JSONUTIL_VALUE_NOT_STRING: return "WRONGTYPE Value is not a string"; 39 | case JSONUTIL_VALUE_NOT_INTEGER: return "WRONGTYPE Value is not an integer"; 40 | case JSONUTIL_PATH_SHOULD_BE_AT_THE_END: return "SYNTAXERR Path arguments should be positioned at the end"; 41 | case JSONUTIL_COMMAND_SYNTAX_ERROR: return "SYNTAXERR Command syntax error"; 42 | case JSONUTIL_MULTIPLICATION_OVERFLOW: return "OVERFLOW Multiplication would overflow"; 43 | case JSONUTIL_ADDITION_OVERFLOW: return "OVERFLOW Addition would overflow"; 44 | case JSONUTIL_EMPTY_JSON_OBJECT: return "EMPTYVAL Empty JSON object"; 45 | case JSONUTIL_EMPTY_JSON_ARRAY: return "EMPTYVAL Empty JSON array"; 46 | case JSONUTIL_INDEX_OUT_OF_ARRAY_BOUNDARIES: return "OUTOFBOUNDARIES Array index is out of bounds"; 47 | case JSONUTIL_UNKNOWN_SUBCOMMAND: return "SYNTAXERR Unknown subcommand"; 48 | case JSONUTIL_FAILED_TO_CREATE_THREAD_SPECIFIC_DATA_KEY: 49 | return "PTHREADERR Failed to create thread-specific data key"; 50 | case JSONUTIL_DOCUMENT_SIZE_LIMIT_EXCEEDED: 51 | return "LIMIT Document size limit is exceeded"; 52 | case JSONUTIL_DOCUMENT_PATH_LIMIT_EXCEEDED: 53 | return "LIMIT Document path nesting limit is exceeded"; 54 | case JSONUTIL_PARSER_RECURSION_DEPTH_LIMIT_EXCEEDED: 55 | return "LIMIT Parser recursion depth is exceeded"; 56 | case JSONUTIL_RECURSIVE_DESCENT_TOKEN_LIMIT_EXCEEDED: 57 | return "LIMIT Total number of recursive descent tokens in the query string exceeds the limit"; 58 | case JSONUTIL_QUERY_STRING_SIZE_LIMIT_EXCEEDED: 59 | return "LIMIT Query string size limit is exceeded"; 60 | case JSONUTIL_CANNOT_INSERT_MEMBER_INTO_NON_OBJECT_VALUE: 61 | return "ERROR Cannot insert a member into a non-object value"; 62 | case JSONUTIL_INVALID_RDB_FORMAT: 63 | return "ERROR Invalid value in RDB format"; 64 | case JSONUTIL_DOLLAR_CANNOT_APPLY_TO_NON_ROOT: return "SYNTAXERR Dollar sign cannot apply to non-root element"; 65 | case JSONUTIL_ALLOCATION_FAILURE: return "ERROR Memory Allocation failed "; 66 | case JSONUTIL_KEY_OPEN_ERROR: return "ERROR Unable to open valkeymodule key"; 67 | default: ValkeyModule_Assert(false); 68 | } 69 | return ""; 70 | } 71 | 72 | size_t jsonutil_double_to_string(const double val, char *double_to_string_buf, size_t len) { 73 | // It's safe to write a double value into double_to_string_buf, because the converted string will 74 | // never exceed length of 1024. 75 | ValkeyModule_Assert(len == BUF_SIZE_DOUBLE_JSON); 76 | return snprintf(double_to_string_buf, len, "%.17g", val); 77 | } 78 | 79 | /** 80 | * Convert double to string using the same format as RapidJSON's Writer::WriteDouble does. 81 | */ 82 | size_t jsonutil_double_to_string_rapidjson(const double val, char *double_to_string_buf_rapidjson, size_t len) { 83 | // RapidJSON's Writer::WriteDouble only uses a buffer of 25 bytes. 84 | ValkeyModule_Assert(len == BUF_SIZE_DOUBLE_RAPID_JSON); 85 | char *end = rapidjson::internal::dtoa(val, double_to_string_buf_rapidjson, 86 | rapidjson::Writer::kDefaultMaxDecimalPlaces); 87 | *end = '\0'; 88 | return end - double_to_string_buf_rapidjson; 89 | } 90 | 91 | bool jsonutil_is_int64(const double a) { 92 | int64_t a_l = static_cast(a); 93 | double b = static_cast(a_l); 94 | return (a <= b && a >= b); 95 | } 96 | 97 | JsonUtilCode jsonutil_multiply_double(const double a, const double b, double *res) { 98 | double c = a * b; 99 | // check overflow 100 | if (std::isinf(c)) return JSONUTIL_MULTIPLICATION_OVERFLOW; 101 | *res = c; 102 | return JSONUTIL_SUCCESS; 103 | } 104 | 105 | JsonUtilCode jsonutil_multiply_int64(const int64_t a, const int64_t b, int64_t *res) { 106 | if (a == 0 || b == 0) { 107 | *res = 0; 108 | return JSONUTIL_SUCCESS; 109 | } 110 | // Check overflow conditions without performing multiplication 111 | if ((a > 0 && b > 0 && a > INT64_MAX / b) || // Positive * Positive overflow 112 | (a > 0 && b < 0 && b < INT64_MIN / a) || // Positive * Negative overflow 113 | (a < 0 && b > 0 && a < INT64_MIN / b) || // Negative * Positive overflow 114 | (a < 0 && b < 0 && a < INT64_MAX / b)) { // Negative * Negative overflow 115 | return JSONUTIL_MULTIPLICATION_OVERFLOW; 116 | } 117 | 118 | // If no overflow, perform the multiplication 119 | *res = a * b; 120 | return JSONUTIL_SUCCESS; 121 | return JSONUTIL_SUCCESS; 122 | } 123 | 124 | JsonUtilCode jsonutil_add_double(const double a, const double b, double *res) { 125 | double c = a + b; 126 | // check overflow 127 | if (std::isinf(c)) return JSONUTIL_ADDITION_OVERFLOW; 128 | *res = c; 129 | return JSONUTIL_SUCCESS; 130 | } 131 | 132 | JsonUtilCode jsonutil_add_int64(const int64_t a, const int64_t b, int64_t *res) { 133 | if (a >= 0) { 134 | if (b > INT64_MAX - a) return JSONUTIL_ADDITION_OVERFLOW; 135 | } else { 136 | if (b < INT64_MIN - a) return JSONUTIL_ADDITION_OVERFLOW; 137 | } 138 | *res = a + b; 139 | return JSONUTIL_SUCCESS; 140 | } 141 | 142 | bool jsonutil_is_root_path(const char *json_path) { 143 | return !strcmp(json_path, ".") || !strcmp(json_path, "$"); 144 | } 145 | -------------------------------------------------------------------------------- /tst/unit/util_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "json/util.h" 7 | #include "json/dom.h" 8 | #include "json/alloc.h" 9 | #include "json/stats.h" 10 | #include "module_sim.h" 11 | 12 | extern size_t dummy_malloc_size(void *); 13 | 14 | class UtilTest : public ::testing::Test { 15 | protected: 16 | void SetUp() override { 17 | JsonUtilCode rc = jsonstats_init(); 18 | ASSERT_EQ(rc, JSONUTIL_SUCCESS); 19 | setupValkeyModulePointers(); 20 | } 21 | }; 22 | 23 | TEST_F(UtilTest, testCodeToMessage) { 24 | for (JsonUtilCode code=JSONUTIL_SUCCESS; code < JSONUTIL_LAST; code = JsonUtilCode(code + 1)) { 25 | const char *msg = jsonutil_code_to_message(code); 26 | EXPECT_TRUE(msg != nullptr); 27 | if (code == JSONUTIL_SUCCESS || code == JSONUTIL_WRONG_NUM_ARGS || 28 | code == JSONUTIL_NX_XX_CONDITION_NOT_SATISFIED) { 29 | EXPECT_STREQ(msg, ""); 30 | } else { 31 | EXPECT_GT(strlen(msg), 0); 32 | } 33 | } 34 | } 35 | 36 | TEST_F(UtilTest, testDoubleToString) { 37 | double v = 189.31; 38 | char buf[BUF_SIZE_DOUBLE_JSON]; 39 | size_t len = jsonutil_double_to_string(v, buf, sizeof(buf)); 40 | EXPECT_STREQ(buf, "189.31"); 41 | EXPECT_EQ(len, strlen(buf)); 42 | } 43 | 44 | TEST_F(UtilTest, testDoubleToStringRapidJson) { 45 | double v = 189.31; 46 | char buf[BUF_SIZE_DOUBLE_RAPID_JSON]; 47 | size_t len = jsonutil_double_to_string_rapidjson(v, buf, sizeof(buf)); 48 | EXPECT_STREQ(buf, "189.31"); 49 | EXPECT_EQ(len, strlen(buf)); 50 | } 51 | 52 | TEST_F(UtilTest, testIsInt64) { 53 | EXPECT_TRUE(jsonutil_is_int64(0)); 54 | EXPECT_TRUE(jsonutil_is_int64(1)); 55 | EXPECT_TRUE(jsonutil_is_int64(INT8_MAX)); 56 | EXPECT_TRUE(jsonutil_is_int64(INT8_MIN)); 57 | EXPECT_TRUE(jsonutil_is_int64(INT16_MAX)); 58 | EXPECT_TRUE(jsonutil_is_int64(INT16_MIN)); 59 | EXPECT_TRUE(jsonutil_is_int64(INT32_MAX)); 60 | EXPECT_TRUE(jsonutil_is_int64(INT32_MIN)); 61 | EXPECT_TRUE(jsonutil_is_int64(INT64_MAX >> 1)); 62 | EXPECT_TRUE(jsonutil_is_int64(8223372036854775807LL)); 63 | EXPECT_TRUE(jsonutil_is_int64(INT64_MIN)); 64 | EXPECT_FALSE(jsonutil_is_int64(1e28)); // out of range of int64 65 | EXPECT_FALSE(jsonutil_is_int64(1.7e308)); // out of range of int64 66 | EXPECT_FALSE(jsonutil_is_int64(-1e28)); // out of range of int64 67 | EXPECT_FALSE(jsonutil_is_int64(-1.7e308)); // out of range of int64 68 | EXPECT_TRUE(jsonutil_is_int64(108.0)); 69 | EXPECT_FALSE(jsonutil_is_int64(108.9)); 70 | EXPECT_FALSE(jsonutil_is_int64(108.0000001)); 71 | EXPECT_TRUE(jsonutil_is_int64(-108.0)); 72 | EXPECT_FALSE(jsonutil_is_int64(-108.9)); 73 | EXPECT_FALSE(jsonutil_is_int64(-108.0000001)); 74 | } 75 | 76 | TEST_F(UtilTest, testMultiplyInt64_overflow) { 77 | // should not overflow 78 | int64_t res; 79 | JsonUtilCode rc = jsonutil_multiply_int64(INT64_MAX, 1, &res); 80 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 81 | EXPECT_EQ(res, INT64_MAX); 82 | 83 | // should overflow 84 | rc = jsonutil_multiply_int64(INT64_MAX, 2, &res); 85 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 86 | rc = jsonutil_multiply_int64(INT64_MAX, INT64_MAX >> 1, &res); 87 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 88 | rc = jsonutil_multiply_int64(INT64_MAX, INT64_MAX, &res); 89 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 90 | } 91 | 92 | TEST_F(UtilTest, testMultiplyInt64_overflow_negative) { 93 | // should not overflow 94 | int64_t res; 95 | JsonUtilCode rc = jsonutil_multiply_int64(INT64_MIN, 1, &res); 96 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 97 | EXPECT_EQ(res, INT64_MIN); 98 | 99 | // should overflow 100 | rc = jsonutil_multiply_int64(INT64_MIN, 2, &res); 101 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 102 | rc = jsonutil_multiply_int64(INT64_MIN, INT64_MIN >> 1, &res); 103 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 104 | rc = jsonutil_multiply_int64(INT64_MIN, INT64_MAX, &res); 105 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 106 | rc = jsonutil_multiply_int64(INT64_MIN, INT64_MIN, &res); 107 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 108 | } 109 | 110 | TEST_F(UtilTest, testMultiplyDouble) { 111 | double res; 112 | JsonUtilCode rc = jsonutil_multiply_double(5e30, 2, &res); 113 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 114 | EXPECT_EQ(res, 1e31); 115 | 116 | rc = jsonutil_multiply_double(5.0e30, 2.0, &res); 117 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 118 | EXPECT_EQ(res, 1.0e31); 119 | } 120 | 121 | TEST_F(UtilTest, testMultiplyDouble_overflow) { 122 | // should not overflow 123 | double res; 124 | JsonUtilCode rc = jsonutil_multiply_double(1.7e308, 1.0, &res); 125 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 126 | EXPECT_EQ(res, 1.7e308); 127 | 128 | // should overflow 129 | rc = jsonutil_multiply_double(1.7e308, 2.0, &res); 130 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 131 | rc = jsonutil_multiply_double(1.7e308, 1.7e308, &res); 132 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 133 | } 134 | 135 | TEST_F(UtilTest, testMultiplyDouble_overflow_negative) { 136 | // should not overflow 137 | double res; 138 | JsonUtilCode rc = jsonutil_multiply_double(-1.7e308, 1.0, &res); 139 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 140 | EXPECT_EQ(res, -1.7e308); 141 | 142 | // should overflow 143 | rc = jsonutil_multiply_double(-1.7e308, 2.0, &res); 144 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 145 | rc = jsonutil_multiply_double(-1.7e308, 1.7e308, &res); 146 | EXPECT_EQ(rc, JSONUTIL_MULTIPLICATION_OVERFLOW); 147 | } 148 | 149 | TEST_F(UtilTest, testAddInt64_overflow) { 150 | // should not overflow 151 | int64_t res; 152 | JsonUtilCode rc = jsonutil_add_int64(INT64_MAX, 0, &res); 153 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 154 | EXPECT_EQ(res, INT64_MAX); 155 | 156 | // should overflow 157 | rc = jsonutil_add_int64(INT64_MAX, 1, &res); 158 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 159 | rc = jsonutil_add_int64(INT64_MAX, INT64_MAX >> 1, &res); 160 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 161 | rc = jsonutil_add_int64(INT64_MAX, INT64_MAX, &res); 162 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 163 | } 164 | 165 | TEST_F(UtilTest, testAddInt64_overflow_negative) { 166 | // should not overflow 167 | int64_t res; 168 | JsonUtilCode rc = jsonutil_add_int64(INT64_MIN, 0, &res); 169 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 170 | EXPECT_EQ(res, INT64_MIN); 171 | 172 | // should overflow 173 | rc = jsonutil_add_int64(INT64_MIN, -1, &res); 174 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 175 | rc = jsonutil_add_int64(INT64_MIN, INT64_MIN >> 1, &res); 176 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 177 | rc = jsonutil_add_int64(INT64_MIN, INT64_MIN, &res); 178 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 179 | } 180 | 181 | TEST_F(UtilTest, testAddDouble_overflow) { 182 | // should not overflow 183 | double res; 184 | JsonUtilCode rc = jsonutil_add_double(1.7e308, 0.0, &res); 185 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 186 | EXPECT_EQ(res, 1.7e308); 187 | rc = jsonutil_add_double(1.7e308, 1.0, &res); 188 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 189 | EXPECT_EQ(res, 1.7e308); 190 | 191 | // should overflow 192 | rc = jsonutil_add_double(1.7e308, 0.85e308, &res); 193 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 194 | rc = jsonutil_add_double(1.7e308, 1.7e308, &res); 195 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 196 | } 197 | 198 | TEST_F(UtilTest, testAddDouble_overflow_negative) { 199 | // should not overflow 200 | double res; 201 | JsonUtilCode rc = jsonutil_add_double(-1.7e308, 0.0, &res); 202 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 203 | EXPECT_EQ(res, -1.7e308); 204 | rc = jsonutil_add_double(-1.7e308, -1.0, &res); 205 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 206 | EXPECT_EQ(res, -1.7e308); 207 | 208 | // should overflow 209 | rc = jsonutil_add_double(-1.7e308, -0.85e308, &res); 210 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 211 | rc = jsonutil_add_double(-1.7e308, -1.7e308, &res); 212 | EXPECT_EQ(rc, JSONUTIL_ADDITION_OVERFLOW); 213 | } 214 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | 3 | include(FetchContent) 4 | include(ExternalProject) 5 | 6 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 7 | 8 | # Detect the system architecture 9 | EXECUTE_PROCESS( 10 | COMMAND uname -m 11 | COMMAND tr -d '\n' 12 | OUTPUT_VARIABLE ARCHITECTURE 13 | ) 14 | 15 | if("${ARCHITECTURE}" STREQUAL "x86_64" 16 | OR "${ARCHITECTURE}" STREQUAL "aarch64" 17 | OR "${ARCHITECTURE}" STREQUAL "arm64") 18 | message("Building valkey-json for ${ARCHITECTURE}") 19 | else() 20 | message( 21 | FATAL_ERROR 22 | "Unsupported architecture: ${ARCHITECTURE}. valkey-json is only supported on x86_64, aarch64 and arm64" 23 | ) 24 | endif() 25 | 26 | # Project definition 27 | project(ValkeyJSONModule VERSION 99.99.99 LANGUAGES C CXX) # Version is 99.99.99 for unstable branch 28 | 29 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-mismatched-tags -Wno-format") 30 | 31 | # Clang related adjustments 32 | set(COMPILER_IS_CLANG OFF) 33 | if (${CMAKE_C_COMPILER_ID} STREQUAL "Clang" OR ${CMAKE_C_COMPILER_ID} STREQUAL "AppleClang") 34 | set(COMPILER_IS_CLANG ON) 35 | endif() 36 | 37 | if (COMPILER_IS_CLANG) 38 | message(STATUS "Defining VALKEYMODULE_ATTR_COMMON => __attribute__((weak))") 39 | add_definitions(-DVALKEYMODULE_ATTR_COMMON=__attribute__\(\(weak\)\)) 40 | endif() 41 | 42 | if(APPLE AND NOT COMPILER_IS_CLANG) 43 | # Force clang compiler on macOS 44 | find_program(CLANGPP "clang++") 45 | find_program(CLANG "clang") 46 | if(CLANG AND CLANGPP) 47 | message(STATUS "Found ${CLANGPP}, ${CLANG}") 48 | set(CMAKE_CXX_COMPILER ${CLANGPP}) 49 | set(CMAKE_C_COMPILER ${CLANG}) 50 | endif() 51 | endif() 52 | 53 | # ASAN build option 54 | option(ENABLE_ASAN "Enable Address Sanitizer" OFF) 55 | 56 | # ASan flags configuration 57 | if(ENABLE_ASAN) 58 | message("Building with Address Sanitizer enabled") 59 | set(ASAN_FLAGS "-fsanitize=address") 60 | endif() 61 | 62 | # Set the name of the JSON shared library 63 | set(JSON_MODULE_LIB json) 64 | 65 | option(BUILD_RELEASE "Build only valkey-json module" OFF) 66 | option(ENABLE_UNIT_TESTS "Build the module and runs unit tests" ON) 67 | option(ENABLE_INTEGRATION_TESTS "Build the module and runs integration tests" ON) 68 | 69 | # Define the Valkey directories used when building from source 70 | set(VALKEY_DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/_deps/valkey-src") 71 | set(VALKEY_BIN_DIR "${CMAKE_BINARY_DIR}/_deps/valkey-src/src/valkey/src") 72 | 73 | # Valkey version 74 | if(NOT VALKEY_VERSION) 75 | set(VALKEY_VERSION unstable) 76 | endif() 77 | message("Valkey version: ${VALKEY_VERSION}") 78 | 79 | # Compiler flags that can be overridden in command line 80 | if(NOT CFLAGS) 81 | # Include debug symbols and set optimize level 82 | set(CFLAGS "-g -O3 -fno-omit-frame-pointer -Wall -Werror -Wextra") 83 | endif() 84 | 85 | # Add ASan flags if enabled 86 | if(ENABLE_ASAN) 87 | set(CFLAGS "${CFLAGS} ${ASAN_FLAGS}") 88 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_FLAGS}") 89 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${ASAN_FLAGS}") 90 | endif() 91 | 92 | # Determine Valkey build command depending on build type 93 | if((BUILD_RELEASE OR ENABLE_UNIT_TESTS) AND NOT ENABLE_INTEGRATION_TESTS) 94 | set(VALKEY_BUILD_CMD ${CMAKE_COMMAND} -E echo "Skipping build step for release build") 95 | else() 96 | if(ENABLE_ASAN) 97 | set(VALKEY_BUILD_CMD make distclean && make -j SANITIZER=address) 98 | else() 99 | set(VALKEY_BUILD_CMD make distclean && make -j) 100 | endif() 101 | endif() 102 | 103 | ExternalProject_Add( 104 | valkey 105 | GIT_REPOSITORY https://github.com/valkey-io/valkey.git 106 | GIT_TAG ${VALKEY_VERSION} 107 | PREFIX ${VALKEY_DOWNLOAD_DIR} 108 | CONFIGURE_COMMAND "" 109 | BUILD_COMMAND ${VALKEY_BUILD_CMD} 110 | INSTALL_COMMAND "" 111 | BUILD_IN_SOURCE 1 112 | ) 113 | 114 | # Define the paths for the copied files 115 | set(VALKEY_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/include") 116 | set(VALKEY_BINARY_DEST "${CMAKE_CURRENT_SOURCE_DIR}/tst/integration/.build/binaries/${VALKEY_VERSION}") 117 | 118 | ExternalProject_Add_Step( 119 | valkey 120 | copy_header_files 121 | COMMENT "Copying header files to include/ directory" 122 | DEPENDEES download 123 | DEPENDERS configure 124 | COMMAND ${CMAKE_COMMAND} -E make_directory ${VALKEY_INCLUDE_DIR} 125 | COMMAND ${CMAKE_COMMAND} -E copy ${VALKEY_DOWNLOAD_DIR}/src/valkey/src/valkeymodule.h ${VALKEY_INCLUDE_DIR}/valkeymodule.h 126 | ALWAYS 1 127 | ) 128 | 129 | # Integration tests require the valkey-test-framework which is only needed when 130 | # building Valkey from source. 131 | if(ENABLE_INTEGRATION_TESTS) 132 | # Copy the valkey-server binary after building 133 | add_custom_command(TARGET valkey 134 | POST_BUILD 135 | COMMAND ${CMAKE_COMMAND} -E make_directory ${VALKEY_BINARY_DEST} 136 | COMMAND ${CMAKE_COMMAND} -E copy ${VALKEY_BIN_DIR}/valkey-server ${VALKEY_BINARY_DEST}/valkey-server 137 | COMMENT "Copied valkey-server to destination directory" 138 | ) 139 | # Define valkey-test-framework commit id 140 | set(VALKEY_TEST_FRAMEWORK_COMMIT "9fb28b74efd122324db00498a2fde6e5d281c90f" CACHE STRING "Valkey-test-framework commit to use") 141 | 142 | # Set the download directory for Valkey-test-framework 143 | set(VALKEY_TEST_FRAMEWORK_DOWNLOAD_DIR "${CMAKE_CURRENT_BINARY_DIR}/_deps/valkey-test-framework-src") 144 | 145 | ExternalProject_Add( 146 | valkey-test-framework 147 | GIT_REPOSITORY https://github.com/valkey-io/valkey-test-framework.git 148 | GIT_TAG ${VALKEY_TEST_FRAMEWORK_COMMIT} 149 | GIT_SHALLOW FALSE 150 | PREFIX "${VALKEY_TEST_FRAMEWORK_DOWNLOAD_DIR}" 151 | CONFIGURE_COMMAND "" 152 | BUILD_COMMAND "" 153 | INSTALL_COMMAND "" 154 | ) 155 | 156 | ExternalProject_Add_Step( 157 | valkey-test-framework 158 | copy_pytest_files 159 | COMMENT "Copying pytest files to tst/integration directory" 160 | DEPENDEES build 161 | COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_SOURCE_DIR}/tst/integration 162 | COMMAND ${CMAKE_COMMAND} -E copy_directory ${VALKEY_TEST_FRAMEWORK_DOWNLOAD_DIR}/src/valkey-test-framework/src ${CMAKE_CURRENT_SOURCE_DIR}/tst/integration/valkeytests 163 | ) 164 | endif() 165 | 166 | # Enable instrumentation options if requested 167 | if("$ENV{INSTRUMENT_V2PATH}" STREQUAL "yes") 168 | add_compile_definitions(INSTRUMENT_V2PATH) 169 | message("Enabled INSTRUMENT_V2PATH") 170 | endif() 171 | 172 | # Disable Doxygen documentation generation 173 | set(BUILD_DOCUMENTATION OFF) 174 | # When CODE_COVERAGE is ON, the package is built twice, once for debug and once for release. 175 | # To fix the problem, disable the code coverage. 176 | set(CODE_COVERAGE OFF) 177 | 178 | # Fix for linking error when code coverage is enabled on ARM 179 | if(CODE_COVERAGE AND CMAKE_BUILD_TYPE STREQUAL "Debug") 180 | add_link_options("--coverage") 181 | endif() 182 | 183 | # Set C & C++ standard versions 184 | set(CMAKE_C_STANDARD 11) 185 | set(CMAKE_C_STANDARD_REQUIRED True) 186 | set(CMAKE_CXX_STANDARD 17) 187 | set(CMAKE_CXX_STANDARD_REQUIRED True) 188 | 189 | # Additional flags for all architectures 190 | set(ADDITIONAL_FLAGS "-fPIC") 191 | 192 | # RapidJSON SIMD optimization 193 | if("${ARCHITECTURE}" STREQUAL "x86_64") 194 | set(ADDITIONAL_FLAGS "${ADDITIONAL_FLAGS} -march=nehalem") 195 | elseif("${ARCHITECTURE}" STREQUAL "aarch64") 196 | set(ADDITIONAL_FLAGS "${ADDITIONAL_FLAGS} -march=armv8-a") 197 | endif() 198 | 199 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CFLAGS} ${ADDITIONAL_FLAGS}") 200 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CFLAGS} ${ADDITIONAL_FLAGS}") 201 | message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") 202 | message("CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") 203 | 204 | # Fetch RapidJSON 205 | FetchContent_Declare( 206 | rapidjson 207 | GIT_REPOSITORY https://github.com/Tencent/rapidjson.git 208 | GIT_TAG ebd87cb468fb4cb060b37e579718c4a4125416c1 209 | ) 210 | 211 | # Disable RapidJSON tests and examples 212 | set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Build rapidjson tests" FORCE) 213 | set(RAPIDJSON_BUILD_EXAMPLES OFF CACHE BOOL "Build rapidjson examples" FORCE) 214 | set(RAPIDJSON_BUILD_DOC OFF CACHE BOOL "Build rapidjson documentation" FORCE) 215 | 216 | # Make Rapidjson available 217 | FetchContent_MakeAvailable(rapidjson) 218 | 219 | add_subdirectory(src) 220 | 221 | if(ENABLE_UNIT_TESTS OR ENABLE_INTEGRATION_TESTS) 222 | add_subdirectory(tst) 223 | endif() 224 | 225 | if(ENABLE_INTEGRATION_TESTS) 226 | message("adding test target") 227 | add_custom_target(test 228 | COMMENT "Run valkey-json integration tests" 229 | USES_TERMINAL 230 | COMMAND rm -rf ${CMAKE_BINARY_DIR}/tst/integration 231 | COMMAND mkdir -p ${CMAKE_BINARY_DIR}/tst/integration 232 | COMMAND cp -rp ${CMAKE_SOURCE_DIR}/tst/integration/. ${CMAKE_BINARY_DIR}/tst/integration/ 233 | COMMAND echo "[TARGET] begin integration tests" 234 | COMMAND ${CMAKE_SOURCE_DIR}/tst/integration/run.sh "test" ${CMAKE_SOURCE_DIR} 235 | COMMAND echo "[TARGET] end integration tests") 236 | endif() 237 | -------------------------------------------------------------------------------- /tst/unit/json_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "json/dom.h" 19 | #include "json/alloc.h" 20 | #include "json/stats.h" 21 | #include "json/selector.h" 22 | 23 | extern void SetupAllocFuncs(size_t numShards); 24 | 25 | class JsonTest : public ::testing::Test { 26 | void SetUp() override { 27 | JsonUtilCode rc = jsonstats_init(); 28 | ASSERT_EQ(rc, JSONUTIL_SUCCESS); 29 | SetupAllocFuncs(16); 30 | } 31 | void TearDown() override { 32 | delete keyTable; 33 | keyTable = nullptr; 34 | } 35 | void setShards(size_t numShards) { 36 | if (keyTable) delete keyTable; 37 | SetupAllocFuncs(numShards); 38 | } 39 | }; 40 | 41 | TEST_F(JsonTest, testArrIndex_fullobjects) { 42 | const char *input = "[5, 6, {\"a\":\"b\"}, [99,100], [\"c\"]]"; 43 | 44 | JDocument *doc; 45 | JsonUtilCode rc = dom_parse(nullptr, input, strlen(input), &doc); 46 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 47 | 48 | jsn::vector indexes; 49 | bool is_v2_path; 50 | rc = dom_array_index_of(doc, ".", "{\"a\":\"b\"}", 9, 0, 0, indexes, is_v2_path); 51 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 52 | EXPECT_FALSE(is_v2_path); 53 | EXPECT_EQ(indexes.size(), 1); 54 | EXPECT_EQ(indexes[0], 2); 55 | 56 | rc = dom_array_index_of(doc, ".", "[\"c\"]", 5, 0, 0, indexes, is_v2_path); 57 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 58 | EXPECT_FALSE(is_v2_path); 59 | EXPECT_EQ(indexes.size(), 1); 60 | EXPECT_EQ(indexes[0], 4); 61 | 62 | rc = dom_array_index_of(doc, ".", "[99,100]", 8, 0, 0, indexes, is_v2_path); 63 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 64 | EXPECT_FALSE(is_v2_path); 65 | EXPECT_EQ(indexes.size(), 1); 66 | EXPECT_EQ(indexes[0], 3); 67 | 68 | dom_free_doc(doc); 69 | } 70 | 71 | TEST_F(JsonTest, testArrIndex_arr) { 72 | const char *input = "{\"a\":[1,2,[15,50],3], \"nested\": {\"a\": [3,4,[5,5]]}}"; 73 | 74 | JDocument *doc; 75 | JsonUtilCode rc = dom_parse(nullptr, input, strlen(input), &doc); 76 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 77 | 78 | jsn::vector indexes; 79 | bool is_v2_path; 80 | rc = dom_array_index_of(doc, "$..a", "[15,50]", 7, 0, 0, indexes, is_v2_path); 81 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 82 | EXPECT_TRUE(is_v2_path); 83 | EXPECT_EQ(indexes.size(), 2); 84 | EXPECT_EQ(indexes[0], 2); 85 | EXPECT_EQ(indexes[1], -1); 86 | 87 | rc = dom_array_index_of(doc, "$..a", "3", 1, 0, 0, indexes, is_v2_path); 88 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 89 | EXPECT_TRUE(is_v2_path); 90 | EXPECT_EQ(indexes.size(), 2); 91 | EXPECT_EQ(indexes[0], 3); 92 | EXPECT_EQ(indexes[1], 0); 93 | 94 | rc = dom_array_index_of(doc, "$..a", "[5,5]", 5, 0, 0, indexes, is_v2_path); 95 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 96 | EXPECT_TRUE(is_v2_path); 97 | EXPECT_EQ(indexes.size(), 2); 98 | EXPECT_EQ(indexes[0], -1); 99 | EXPECT_EQ(indexes[1], 2); 100 | 101 | rc = dom_array_index_of(doc, "$..a", "35", 2, 0, 0, indexes, is_v2_path); 102 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 103 | EXPECT_TRUE(is_v2_path); 104 | EXPECT_EQ(indexes.size(), 2); 105 | EXPECT_EQ(indexes[0], -1); 106 | EXPECT_EQ(indexes[0], -1); 107 | 108 | dom_free_doc(doc); 109 | } 110 | 111 | TEST_F(JsonTest, testArrIndex_object) { 112 | const char *input = "{\"a\":{\"b\":[2,4,{\"a\":4},false,true,{\"b\":false}]}}"; 113 | 114 | JDocument *doc; 115 | JsonUtilCode rc = dom_parse(nullptr, input, strlen(input), &doc); 116 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 117 | 118 | jsn::vector indexes; 119 | bool is_v2_path; 120 | rc = dom_array_index_of(doc, "$.a.b", "{\"a\":4}", 7, 0, 0, indexes, is_v2_path); 121 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 122 | EXPECT_TRUE(is_v2_path); 123 | EXPECT_EQ(indexes.size(), 1); 124 | EXPECT_EQ(indexes[0], 2); 125 | 126 | rc = dom_array_index_of(doc, "$.a.b", "{\"b\":false}", 11, 0, 0, indexes, is_v2_path); 127 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 128 | EXPECT_TRUE(is_v2_path); 129 | EXPECT_EQ(indexes.size(), 1); 130 | EXPECT_EQ(indexes[0], 5); 131 | 132 | rc = dom_array_index_of(doc, "$.a.b", "false", 5, 0, 0, indexes, is_v2_path); 133 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 134 | EXPECT_TRUE(is_v2_path); 135 | EXPECT_EQ(indexes.size(), 1); 136 | EXPECT_EQ(indexes[0], 3); 137 | 138 | rc = dom_array_index_of(doc, "$..a", "{\"a\":4}", 7, 0, 0, indexes, is_v2_path); 139 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 140 | EXPECT_TRUE(is_v2_path); 141 | EXPECT_EQ(indexes.size(), 2); 142 | EXPECT_EQ(indexes[0], INT64_MAX); 143 | EXPECT_EQ(indexes[1], INT64_MAX); 144 | 145 | rc = dom_array_index_of(doc, "$..a..", "{\"a\":4}", 7, 0, 0, indexes, is_v2_path); 146 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 147 | EXPECT_TRUE(is_v2_path); 148 | EXPECT_EQ(indexes.size(), 4); 149 | EXPECT_EQ(indexes[0], INT64_MAX); 150 | EXPECT_EQ(indexes[1], 2); 151 | EXPECT_EQ(indexes[2], INT64_MAX); 152 | EXPECT_EQ(indexes[3], INT64_MAX); 153 | 154 | dom_free_doc(doc); 155 | } 156 | 157 | TEST_F(JsonTest, testArrIndex_nested_search) { 158 | const char *input = "{\"level0\":{\"level1_0\":{\"level2\":" 159 | "[1,2,3, [25, [4,5,{\"c\":\"d\"}]]]}," 160 | "\"level1_1\":{\"level2\": [[{\"a\":[2,5]}, true, null]]}}}"; 161 | 162 | JDocument *doc; 163 | JsonUtilCode rc = dom_parse(nullptr, input, strlen(input), &doc); 164 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 165 | 166 | jsn::vector indexes; 167 | bool is_v2_path; 168 | rc = dom_array_index_of(doc, "$..level0.level1_0..", "[4,5,{\"c\":\"d\"}]", 15, 0, 0, indexes, is_v2_path); 169 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 170 | EXPECT_TRUE(is_v2_path); 171 | EXPECT_EQ(indexes.size(), 5); 172 | EXPECT_EQ(indexes[0], INT64_MAX); 173 | EXPECT_EQ(indexes[1], -1); 174 | EXPECT_EQ(indexes[2], 1); 175 | EXPECT_EQ(indexes[3], -1); 176 | EXPECT_EQ(indexes[4], INT64_MAX); 177 | 178 | rc = dom_array_index_of(doc, "$..level0.level1_0..", "[25, [4,5,{\"c\":\"d\"}]]", 21, 0, 0, indexes, is_v2_path); 179 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 180 | EXPECT_TRUE(is_v2_path); 181 | EXPECT_EQ(indexes.size(), 5); 182 | EXPECT_EQ(indexes[0], INT64_MAX); 183 | EXPECT_EQ(indexes[1], 3); 184 | EXPECT_EQ(indexes[2], -1); 185 | EXPECT_EQ(indexes[3], -1); 186 | EXPECT_EQ(indexes[4], INT64_MAX); 187 | 188 | rc = dom_array_index_of(doc, "$..level0.level1_0..", "{\"c\":\"d\"}", 9, 0, 0, indexes, is_v2_path); 189 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 190 | EXPECT_TRUE(is_v2_path); 191 | EXPECT_EQ(indexes.size(), 5); 192 | EXPECT_EQ(indexes[0], INT64_MAX); 193 | EXPECT_EQ(indexes[1], -1); 194 | EXPECT_EQ(indexes[2], -1); 195 | EXPECT_EQ(indexes[3], 2); 196 | EXPECT_EQ(indexes[4], INT64_MAX); 197 | 198 | rc = dom_array_index_of(doc, "$..level0.level1_0..", "[4,5,{\"a\":\"b\"}]", 15, 0, 0, indexes, is_v2_path); 199 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 200 | EXPECT_TRUE(is_v2_path); 201 | EXPECT_EQ(indexes.size(), 5); 202 | EXPECT_EQ(indexes[0], INT64_MAX); 203 | EXPECT_EQ(indexes[1], -1); 204 | EXPECT_EQ(indexes[2], -1); 205 | EXPECT_EQ(indexes[3], -1); 206 | EXPECT_EQ(indexes[4], INT64_MAX); 207 | 208 | rc = dom_array_index_of(doc, "$..level0.level1_1..", "[null,true,{\"a\":[2,5]}]", 23, 0, 0, indexes, is_v2_path); 209 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 210 | EXPECT_TRUE(is_v2_path); 211 | EXPECT_EQ(indexes.size(), 5); 212 | EXPECT_EQ(indexes[0], INT64_MAX); 213 | EXPECT_EQ(indexes[1], -1); 214 | EXPECT_EQ(indexes[2], -1); 215 | EXPECT_EQ(indexes[3], INT64_MAX); 216 | EXPECT_EQ(indexes[4], -1); 217 | 218 | rc = dom_array_index_of(doc, "$..level0.level1_1..", "[{\"a\":[2,5]},true,null]", 23, 0, 0, indexes, is_v2_path); 219 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 220 | EXPECT_TRUE(is_v2_path); 221 | EXPECT_EQ(indexes.size(), 5); 222 | EXPECT_EQ(indexes[0], INT64_MAX); 223 | EXPECT_EQ(indexes[1], 0); 224 | EXPECT_EQ(indexes[2], -1); 225 | EXPECT_EQ(indexes[3], INT64_MAX); 226 | EXPECT_EQ(indexes[4], -1); 227 | 228 | rc = dom_array_index_of(doc, "$..level0.level1_1..", "[{\"a\":[2,5]},true]", 18, 0, 0, indexes, is_v2_path); 229 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 230 | EXPECT_TRUE(is_v2_path); 231 | EXPECT_EQ(indexes.size(), 5); 232 | EXPECT_EQ(indexes[0], INT64_MAX); 233 | EXPECT_EQ(indexes[1], -1); 234 | EXPECT_EQ(indexes[2], -1); 235 | EXPECT_EQ(indexes[3], INT64_MAX); 236 | EXPECT_EQ(indexes[4], -1); 237 | 238 | rc = dom_array_index_of(doc, "$..level0.level1_0..", "[4,{\"c\":\"d\"}]", 13, 0, 0, indexes, is_v2_path); 239 | EXPECT_EQ(rc, JSONUTIL_SUCCESS); 240 | EXPECT_TRUE(is_v2_path); 241 | EXPECT_EQ(indexes.size(), 5); 242 | EXPECT_EQ(indexes[0], INT64_MAX); 243 | EXPECT_EQ(indexes[1], -1); 244 | EXPECT_EQ(indexes[2], -1); 245 | EXPECT_EQ(indexes[3], -1); 246 | EXPECT_EQ(indexes[4], INT64_MAX); 247 | 248 | dom_free_doc(doc); 249 | } 250 | -------------------------------------------------------------------------------- /src/json/stats.cc: -------------------------------------------------------------------------------- 1 | #include "json/stats.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | extern "C" { 9 | #define VALKEYMODULE_EXPERIMENTAL_API 10 | #include <./include/valkeymodule.h> 11 | } 12 | 13 | #define STATIC /* decorator for static functions, remove so that backtrace symbols include these */ 14 | 15 | LogicalStats logical_stats; // initialize global variable 16 | 17 | // Thread local storage (TLS) key for calculating used memory per thread. 18 | static pthread_key_t thread_local_mem_counter_key; 19 | 20 | /* JSON statistics struct. 21 | * Use atomic integers due to possible multi-threading execution of rdb_load and 22 | * also the overhead of atomic operations are negligible. 23 | */ 24 | typedef struct _JsonStats { 25 | std::atomic_ullong used_mem; // global used memory counter 26 | std::atomic_ullong num_doc_keys; 27 | std::atomic_ullong max_depth_ever_seen; 28 | std::atomic_ullong max_size_ever_seen; 29 | std::atomic_ullong defrag_count; 30 | std::atomic_ullong defrag_bytes; 31 | 32 | void reset() { 33 | used_mem = 0; 34 | num_doc_keys = 0; 35 | max_depth_ever_seen = 0; 36 | max_size_ever_seen = 0; 37 | defrag_count = 0; 38 | defrag_bytes = 0; 39 | } 40 | } JsonStats; 41 | static JsonStats jsonstats; 42 | 43 | // histograms 44 | #define NUM_BUCKETS (11) 45 | static size_t buckets[] = { 46 | 0, 256, 1024, 4*1024, 16*1024, 64*1024, 256*1024, 1024*1024, 47 | 4*1024*1024, 16*1024*1024, 64*1024*1024, SIZE_MAX 48 | }; 49 | 50 | // static histogram showing document size distribution 51 | static size_t doc_hist[NUM_BUCKETS]; 52 | // dynamic histogram for read operations (JSON.GET and JSON.MGET only) 53 | static size_t read_hist[NUM_BUCKETS]; 54 | // dynamic histogram for insert operations (JSON.SET and JSON.ARRINSERT) 55 | static size_t insert_hist[NUM_BUCKETS]; 56 | // dynamic histogram for update operations (JSON.SET, JSON.STRAPPEND and JSON.ARRAPPEND) 57 | static size_t update_hist[NUM_BUCKETS]; 58 | // dynamic histogram for delete operations (JSON.DEL, JSON.FORGET, JSON.ARRPOP and JSON.ARRTRIM) 59 | static size_t delete_hist[NUM_BUCKETS]; 60 | 61 | JsonUtilCode jsonstats_init() { 62 | ValkeyModule_Assert(jsonstats.used_mem == 0); // Otherwise you'll lose memory accounting 63 | // Create thread local key. No need to have destructor hook, as the key is created on stack. 64 | if (pthread_key_create(&thread_local_mem_counter_key, nullptr) != 0) 65 | return JSONUTIL_FAILED_TO_CREATE_THREAD_SPECIFIC_DATA_KEY; 66 | 67 | jsonstats.reset(); 68 | logical_stats.reset(); 69 | memset(doc_hist, 0, sizeof(doc_hist)); 70 | memset(read_hist, 0, sizeof(read_hist)); 71 | memset(insert_hist, 0, sizeof(insert_hist)); 72 | memset(update_hist, 0, sizeof(update_hist)); 73 | memset(delete_hist, 0, sizeof(delete_hist)); 74 | return JSONUTIL_SUCCESS; 75 | } 76 | 77 | int64_t jsonstats_begin_track_mem() { 78 | return reinterpret_cast(pthread_getspecific(thread_local_mem_counter_key)); 79 | } 80 | 81 | int64_t jsonstats_end_track_mem(const int64_t begin_val) { 82 | int64_t end_val = reinterpret_cast(pthread_getspecific(thread_local_mem_counter_key)); 83 | return end_val - begin_val; 84 | } 85 | 86 | void jsonstats_increment_used_mem(size_t delta) { 87 | // update the atomic global counter 88 | jsonstats.used_mem += delta; 89 | 90 | // update the thread local counter 91 | int64_t curr_val = reinterpret_cast(pthread_getspecific(thread_local_mem_counter_key)); 92 | pthread_setspecific(thread_local_mem_counter_key, reinterpret_cast(curr_val + delta)); 93 | } 94 | 95 | void jsonstats_decrement_used_mem(size_t delta) { 96 | // update the atomic global counter 97 | ValkeyModule_Assert(delta <= jsonstats.used_mem); 98 | jsonstats.used_mem -= delta; 99 | 100 | // update the thread local counter 101 | int64_t curr_val = reinterpret_cast(pthread_getspecific(thread_local_mem_counter_key)); 102 | pthread_setspecific(thread_local_mem_counter_key, reinterpret_cast(curr_val - delta)); 103 | } 104 | 105 | unsigned long long jsonstats_get_used_mem() { 106 | return jsonstats.used_mem; 107 | } 108 | 109 | unsigned long long jsonstats_get_num_doc_keys() { 110 | return jsonstats.num_doc_keys; 111 | } 112 | 113 | unsigned long long jsonstats_get_max_depth_ever_seen() { 114 | return jsonstats.max_depth_ever_seen; 115 | } 116 | 117 | void jsonstats_update_max_depth_ever_seen(const size_t max_depth) { 118 | if (max_depth > jsonstats.max_depth_ever_seen) { 119 | jsonstats.max_depth_ever_seen = max_depth; 120 | } 121 | } 122 | 123 | unsigned long long jsonstats_get_max_size_ever_seen() { 124 | return jsonstats.max_size_ever_seen; 125 | } 126 | 127 | void jsonstats_update_max_size_ever_seen(const size_t max_size) { 128 | if (max_size > jsonstats.max_size_ever_seen) { 129 | jsonstats.max_size_ever_seen = max_size; 130 | } 131 | } 132 | 133 | unsigned long long jsonstats_get_defrag_count() { 134 | return jsonstats.defrag_count; 135 | } 136 | 137 | void jsonstats_increment_defrag_count() { 138 | jsonstats.defrag_count++; 139 | } 140 | 141 | unsigned long long jsonstats_get_defrag_bytes() { 142 | return jsonstats.defrag_bytes; 143 | } 144 | 145 | void jsonstats_increment_defrag_bytes(const size_t amount) { 146 | jsonstats.defrag_bytes += amount; 147 | } 148 | 149 | /* Given a size (bytes), find histogram bucket index using binary search. 150 | */ 151 | uint32_t jsonstats_find_bucket(size_t size) { 152 | int lo = 0; 153 | int hi = NUM_BUCKETS; // length of buckets[] is NUM_BUCKETS + 1 154 | while (hi - lo > 1) { 155 | uint32_t mid = (lo + hi) / 2; 156 | if (size < buckets[mid]) 157 | hi = mid; 158 | else if (size > buckets[mid]) 159 | lo = mid; 160 | else 161 | return mid; 162 | } 163 | return lo; 164 | } 165 | 166 | /* Update the static document histogram */ 167 | STATIC void update_doc_hist(JDocument *doc, const size_t orig_size, const size_t new_size, 168 | const JsonCommandType cmd_type) { 169 | switch (cmd_type) { 170 | case JSONSTATS_INSERT: { 171 | if (orig_size == 0) { 172 | uint32_t new_bucket = jsonstats_find_bucket(new_size); 173 | doc_hist[new_bucket]++; 174 | dom_set_bucket_id(doc, new_bucket); 175 | } else { 176 | update_doc_hist(doc, orig_size, new_size, JSONSTATS_UPDATE); 177 | } 178 | break; 179 | } 180 | case JSONSTATS_UPDATE: { 181 | if (orig_size != new_size) { 182 | uint32_t orig_bucket = dom_get_bucket_id(doc); 183 | uint32_t new_bucket = jsonstats_find_bucket(new_size); 184 | if (orig_bucket != new_bucket) { 185 | doc_hist[orig_bucket]--; 186 | doc_hist[new_bucket]++; 187 | dom_set_bucket_id(doc, new_bucket); 188 | } 189 | } 190 | break; 191 | } 192 | case JSONSTATS_DELETE: { 193 | uint32_t orig_bucket = dom_get_bucket_id(doc); 194 | if (new_size == 0) { 195 | doc_hist[orig_bucket]--; 196 | } else { 197 | uint32_t new_bucket = jsonstats_find_bucket(new_size); 198 | if (new_bucket != orig_bucket) { 199 | doc_hist[orig_bucket]--; 200 | doc_hist[new_bucket]++; 201 | dom_set_bucket_id(doc, new_bucket); 202 | } 203 | } 204 | break; 205 | } 206 | default: 207 | break; 208 | } 209 | } 210 | 211 | void jsonstats_sprint_hist_buckets(char *buf, const size_t buf_size) { 212 | std::ostringstream oss; 213 | oss << "["; 214 | for (size_t i=0; i < NUM_BUCKETS; i++) { 215 | if (i > 0) oss << ","; 216 | oss << buckets[i]; 217 | } 218 | oss << ",INF]"; 219 | std::string str = oss.str(); 220 | ValkeyModule_Assert(str.length() <= buf_size); 221 | memcpy(buf, str.c_str(), str.length()); 222 | buf[str.length()] = '\0'; 223 | } 224 | 225 | STATIC void sprint_hist(size_t *arr, const size_t len, char *buf, const size_t buf_size) { 226 | std::ostringstream oss; 227 | oss << "["; 228 | for (size_t i=0; i < len; i++) { 229 | if (i > 0) oss << ","; 230 | oss << arr[i]; 231 | } 232 | oss << "]"; 233 | std::string str = oss.str(); 234 | ValkeyModule_Assert(str.length() <= buf_size); 235 | memcpy(buf, str.c_str(), str.length()); 236 | buf[str.length()] = '\0'; 237 | } 238 | 239 | void jsonstats_sprint_doc_hist(char *buf, const size_t buf_size) { 240 | sprint_hist(doc_hist, NUM_BUCKETS, buf, buf_size); 241 | } 242 | 243 | void jsonstats_sprint_read_hist(char *buf, const size_t buf_size) { 244 | sprint_hist(read_hist, NUM_BUCKETS, buf, buf_size); 245 | } 246 | 247 | void jsonstats_sprint_insert_hist(char *buf, const size_t buf_size) { 248 | sprint_hist(insert_hist, NUM_BUCKETS, buf, buf_size); 249 | } 250 | 251 | void jsonstats_sprint_update_hist(char *buf, const size_t buf_size) { 252 | sprint_hist(update_hist, NUM_BUCKETS, buf, buf_size); 253 | } 254 | 255 | void jsonstats_sprint_delete_hist(char *buf, const size_t buf_size) { 256 | sprint_hist(delete_hist, NUM_BUCKETS, buf, buf_size); 257 | } 258 | 259 | void jsonstats_update_stats_on_read(const size_t fetched_val_size) { 260 | uint32_t bucket = jsonstats_find_bucket(fetched_val_size); 261 | read_hist[bucket]++; 262 | } 263 | 264 | void jsonstats_update_stats_on_insert(JDocument *doc, const bool is_delete_doc_key, const size_t orig_size, 265 | const size_t new_size, const size_t inserted_val_size) { 266 | if (is_delete_doc_key) jsonstats.num_doc_keys++; 267 | update_doc_hist(doc, orig_size, new_size, JSONSTATS_INSERT); 268 | uint32_t bucket = jsonstats_find_bucket(inserted_val_size); 269 | insert_hist[bucket]++; 270 | jsonstats_update_max_size_ever_seen(new_size); 271 | } 272 | 273 | void jsonstats_update_stats_on_update(JDocument *doc, const size_t orig_size, const size_t new_size, 274 | const size_t input_json_size) { 275 | update_doc_hist(doc, orig_size, new_size, JSONSTATS_UPDATE); 276 | uint32_t bucket = jsonstats_find_bucket(input_json_size); 277 | update_hist[bucket]++; 278 | jsonstats_update_max_size_ever_seen(new_size); 279 | } 280 | 281 | void jsonstats_update_stats_on_delete(JDocument *doc, const bool is_delete_doc_key, const size_t orig_size, 282 | const size_t new_size, const size_t deleted_val_size) { 283 | update_doc_hist(doc, orig_size, new_size, JSONSTATS_DELETE); 284 | if (is_delete_doc_key) { 285 | ValkeyModule_Assert(jsonstats.num_doc_keys > 0); 286 | jsonstats.num_doc_keys--; 287 | } 288 | uint32_t bucket = jsonstats_find_bucket(deleted_val_size); 289 | delete_hist[bucket]++; 290 | } 291 | 292 | -------------------------------------------------------------------------------- /tst/integration/data/truenull.json: -------------------------------------------------------------------------------- 1 | [true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null, true, null] -------------------------------------------------------------------------------- /src/json/memory.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include "json/memory.h" 8 | #include "json/dom.h" 9 | 10 | extern "C" { 11 | #define VALKEYMODULE_EXPERIMENTAL_API 12 | #include <./include/valkeymodule.h> 13 | } 14 | 15 | #define STATIC /* decorator for static functions, remove so that backtrace symbols include these */ 16 | 17 | void *(*memory_alloc)(size_t size); 18 | void (*memory_free)(void *ptr); 19 | void *(*memory_realloc)(void *orig_ptr, size_t new_size); 20 | size_t (*memory_allocsize)(void *ptr); 21 | 22 | bool memoryTrapsEnabled = false; 23 | 24 | static std::atomic totalMemoryUsage; 25 | 26 | size_t memory_usage() { 27 | return totalMemoryUsage; 28 | } 29 | 30 | /* 31 | * When Traps are disabled, The following code is used 32 | */ 33 | 34 | STATIC void *memory_alloc_without_traps(size_t size) { 35 | void *ptr = ValkeyModule_Alloc(size); 36 | totalMemoryUsage += ValkeyModule_MallocSize(ptr); 37 | return ptr; 38 | } 39 | 40 | STATIC void memory_free_without_traps(void *ptr) { 41 | if (!ptr) return; 42 | size_t sz = ValkeyModule_MallocSize(ptr); 43 | ValkeyModule_Assert(sz <= totalMemoryUsage); 44 | totalMemoryUsage -= sz; 45 | ValkeyModule_Free(ptr); 46 | } 47 | 48 | STATIC void *memory_realloc_without_traps(void *ptr, size_t new_size) { 49 | if (ptr) { 50 | size_t old_size = ValkeyModule_MallocSize(ptr); 51 | ValkeyModule_Assert(old_size <= totalMemoryUsage); 52 | totalMemoryUsage -= old_size; 53 | } 54 | ptr = ValkeyModule_Realloc(ptr, new_size); 55 | totalMemoryUsage += ValkeyModule_MallocSize(ptr); 56 | return ptr; 57 | } 58 | 59 | #define memory_allocsize_without_traps ValkeyModule_MallocSize 60 | 61 | // 62 | // Implementation of traps 63 | // 64 | 65 | // 66 | // This word of data precedes the memory allocation as seen by the client. 67 | // The presence of the length is redundant with calling the low-level allocators memory-size function, 68 | // but that function can be fairly expensive, so by duplicating here we optimize the run-time cost. 69 | // 70 | struct trap_prefix { 71 | mutable uint64_t length:40; 72 | mutable uint64_t valid_prefix:24; 73 | enum { VALID = 0xdeadbe, INVALID = 0xf00dad}; 74 | static trap_prefix *from_ptr( void *p) { return reinterpret_cast< trap_prefix *>(p) - 1; } 75 | static const trap_prefix *from_ptr(const void *p) { return reinterpret_cast(p) - 1; } 76 | }; 77 | 78 | // 79 | // Another word of data is added to end of each allocation. It's set to a known data pattern. 80 | // 81 | struct trap_suffix { 82 | mutable uint64_t valid_suffix; 83 | enum { VALID = 0xdeadfeedbeeff00dull, INVALID = ~VALID }; 84 | static trap_suffix *from_prefix(trap_prefix *p) { 85 | return reinterpret_cast(p + 1 + (p->length >> 3)); 86 | } 87 | static const trap_suffix *from_prefix(const trap_prefix *p) { 88 | return reinterpret_cast(p + 1 + (p->length >> 3)); 89 | } 90 | }; 91 | 92 | bool memory_validate_ptr(const void *ptr, bool crashOnError) { 93 | if (!ptr) return true; // Null pointers are valid. 94 | auto prefix = trap_prefix::from_ptr(ptr); 95 | if (prefix->valid_prefix != trap_prefix::VALID) { 96 | if (crashOnError) { 97 | ValkeyModule_Log(nullptr, "error", "Validation Failure memory Corrupted at:%p", ptr); 98 | ValkeyModule_Assert(nullptr == "Validate Prefix Corrupted"); 99 | } else { 100 | return false; 101 | } 102 | } 103 | auto suffix = trap_suffix::from_prefix(prefix); 104 | if (suffix->valid_suffix != trap_suffix::VALID) { 105 | if (!crashOnError) return false; 106 | // Dump the first N bytes. Hopefully this might give us a clue what's going wrong.... 107 | size_t malloc_size = ValkeyModule_MallocSize(const_cast(reinterpret_cast(prefix))); 108 | ValkeyModule_Assert(malloc_size >= (sizeof(trap_prefix) + sizeof(trap_suffix))); 109 | size_t available_size = malloc_size - (sizeof(trap_prefix) + sizeof(trap_suffix)); 110 | size_t dump_size = available_size > 256 ? 256 : available_size; 111 | ValkeyModule_Log(nullptr, "error", "Validation Failure memory overrun @%p size:%zu", ptr, available_size); 112 | auto data = static_cast(ptr); 113 | while (dump_size > (4 * sizeof(void *))) { 114 | ValkeyModule_Log(nullptr, "error", "Memory[%p]: %p %p %p %p", 115 | static_cast(data), data[0], data[1], data[2], data[3]); 116 | data += 4; 117 | dump_size -= 4 * sizeof(void *); 118 | } 119 | while (dump_size) { 120 | ValkeyModule_Log(nullptr, "error", "Memory[%p]: %p", 121 | static_cast(data), data[0]); 122 | data++; 123 | dump_size -= sizeof(void *); 124 | } 125 | ValkeyModule_Assert(nullptr == "Validate Suffix Corrupted"); 126 | } 127 | return true; 128 | } 129 | 130 | STATIC void *memory_alloc_with_traps(size_t size) { 131 | size_t requested_bytes = ~7 & (size + 7); // Round up 132 | size_t alloc_bytes = requested_bytes + sizeof(trap_prefix) + sizeof(trap_suffix); 133 | auto prefix = reinterpret_cast(ValkeyModule_Alloc(alloc_bytes)); 134 | totalMemoryUsage += ValkeyModule_MallocSize(prefix); 135 | prefix->valid_prefix = trap_prefix::VALID; 136 | prefix->length = requested_bytes; 137 | auto suffix = trap_suffix::from_prefix(prefix); 138 | suffix->valid_suffix = trap_suffix::VALID; 139 | return reinterpret_cast(prefix + 1); 140 | } 141 | 142 | STATIC void memory_free_with_traps(void *ptr) { 143 | if (!ptr) return; 144 | memory_validate_ptr(ptr); 145 | auto prefix = trap_prefix::from_ptr(ptr); 146 | prefix->valid_prefix = 0; 147 | size_t sz = ValkeyModule_MallocSize(prefix); 148 | ValkeyModule_Assert(sz <= totalMemoryUsage); 149 | totalMemoryUsage -= sz; 150 | ValkeyModule_Free(prefix); 151 | } 152 | 153 | STATIC size_t memory_allocsize_with_traps(void *ptr) { 154 | if (!ptr) return 0; 155 | memory_validate_ptr(ptr); 156 | auto prefix = trap_prefix::from_ptr(ptr); 157 | return prefix->length; 158 | } 159 | 160 | // 161 | // Do a realloc, but this is rare, so we do it suboptimally, i.e., with a copy 162 | // 163 | STATIC void *memory_realloc_with_traps(void *orig_ptr, size_t new_size) { 164 | if (!orig_ptr) return memory_alloc_with_traps(new_size); 165 | memory_validate_ptr(orig_ptr); 166 | auto new_ptr = memory_alloc_with_traps(new_size); 167 | memcpy(new_ptr, orig_ptr, memory_allocsize_with_traps(orig_ptr)); 168 | memory_free_with_traps(orig_ptr); 169 | return new_ptr; 170 | } 171 | 172 | // 173 | // Enable/Disable traps 174 | // 175 | bool memory_traps_control(bool enable) { 176 | if (totalMemoryUsage != 0) { 177 | ValkeyModule_Log(nullptr, "warning", 178 | "Attempt to enable/disable memory traps ignored, %zu outstanding memory.", totalMemoryUsage.load()); 179 | return false; 180 | } 181 | if (enable) { 182 | memory_alloc = memory_alloc_with_traps; 183 | memory_free = memory_free_with_traps; 184 | memory_realloc = memory_realloc_with_traps; 185 | memory_allocsize = memory_allocsize_with_traps; 186 | } else { 187 | memory_alloc = memory_alloc_without_traps; 188 | memory_free = memory_free_without_traps; 189 | memory_realloc = memory_realloc_without_traps; 190 | memory_allocsize = memory_allocsize_without_traps; 191 | } 192 | memoryTrapsEnabled = enable; 193 | return true; 194 | } 195 | 196 | void memory_corrupt_memory(const void *ptr, memTrapsCorruption_t corruption) { 197 | memory_validate_ptr(ptr); 198 | auto prefix = trap_prefix::from_ptr(ptr); 199 | auto suffix = trap_suffix::from_prefix(prefix); 200 | switch (corruption) { 201 | case CORRUPT_PREFIX: 202 | prefix->valid_prefix = trap_prefix::INVALID; 203 | break; 204 | case CORRUPT_LENGTH: 205 | prefix->length--; 206 | break; 207 | case CORRUPT_SUFFIX: 208 | suffix->valid_suffix = trap_suffix::INVALID; 209 | break; 210 | default: 211 | ValkeyModule_Assert(0); 212 | break; 213 | } 214 | } 215 | 216 | void memory_uncorrupt_memory(const void *ptr, memTrapsCorruption_t corruption) { 217 | auto prefix = trap_prefix::from_ptr(ptr); 218 | auto suffix = trap_suffix::from_prefix(prefix); 219 | switch (corruption) { 220 | case CORRUPT_PREFIX: 221 | ValkeyModule_Assert(prefix->valid_prefix == trap_prefix::INVALID); 222 | prefix->valid_prefix = trap_prefix::VALID; 223 | break; 224 | case CORRUPT_LENGTH: 225 | prefix->length++; 226 | break; 227 | case CORRUPT_SUFFIX: 228 | ValkeyModule_Assert(suffix->valid_suffix == trap_suffix::INVALID); 229 | suffix->valid_suffix = trap_suffix::VALID; 230 | break; 231 | default: 232 | ValkeyModule_Assert(0); 233 | break; 234 | } 235 | memory_validate_ptr(ptr); 236 | } 237 | 238 | // 239 | // Helper functions for JSON validation 240 | // 241 | // true => Valid. 242 | // false => NOT VALID 243 | // 244 | bool ValidateJValue(JValue &v) { 245 | auto p = v.trap_GetMallocPointer(false); 246 | if (p && !memory_validate_ptr(p, false)) return false; 247 | if (v.IsObject()) { 248 | for (auto m = v.MemberBegin(); m != v.MemberEnd(); ++m) { 249 | if (!ValidateJValue(m->value)) return false; 250 | } 251 | } else if (v.IsArray()) { 252 | for (size_t i = 0; i < v.Size(); ++i) { 253 | if (!ValidateJValue(v[i])) return false; 254 | } 255 | } 256 | return true; 257 | } 258 | 259 | // 260 | // Dump a JValue with Redaction and memory Validation. 261 | // 262 | // Typical use case: 263 | // 264 | // std::ostringstream os; 265 | // DumpRedactedJValue(os, ); 266 | // 267 | void DumpRedactedJValue(std::ostream& os, const JValue &v, size_t level, int index) { 268 | for (size_t i = 0; i < (3 * level); ++i) os << ' '; // Indent 269 | os << "@" << reinterpret_cast(&v) << " "; 270 | if (index != -1) os << '[' << index << ']' << ' '; 271 | if (v.IsDouble()) { 272 | os << "double string of length " << v.GetDoubleStringLength(); 273 | if (!IS_VALID_MEMORY(v.trap_GetMallocPointer(false))) { 274 | os << " <*INVALID*>\n"; 275 | } else if (v.trap_GetMallocPointer(false)) { 276 | os << " @" << v.trap_GetMallocPointer(false) << "\n"; 277 | } else { 278 | os << "\n"; 279 | } 280 | } else if (v.IsString()) { 281 | os << "String of length " << v.GetStringLength(); 282 | if (!IS_VALID_MEMORY(v.trap_GetMallocPointer(false))) { 283 | os << " <*INVALID*>\n"; 284 | } else if (v.trap_GetMallocPointer(false)) { 285 | os << " @" << v.trap_GetMallocPointer(false) << "\n"; 286 | } else { 287 | os << "\n"; 288 | } 289 | } else if (v.IsObject()) { 290 | os << " Object with " << v.MemberCount() << " Members"; 291 | if (!IS_VALID_MEMORY(v.trap_GetMallocPointer(false))) { 292 | os << " *INVALID*\n"; 293 | } else { 294 | os << " @" << v.trap_GetMallocPointer(false) << '\n'; 295 | index = 0; 296 | for (auto m = v.MemberBegin(); m != v.MemberEnd(); ++m) { 297 | DumpRedactedJValue(os, m->value, level+1, index); 298 | index++; 299 | } 300 | } 301 | } else if (v.IsArray()) { 302 | os << "Array with " << v.Size() << " Members"; 303 | if (!IS_VALID_MEMORY(v.trap_GetMallocPointer(false))) { 304 | os << " *INVALID*\n"; 305 | } else { 306 | os << " @" << v.trap_GetMallocPointer(false) << "\n"; 307 | for (size_t index = 0; index < v.Size(); ++index) { 308 | DumpRedactedJValue(os, v[index], level+1, int(index)); 309 | } 310 | } 311 | } else { 312 | os << "\n"; 313 | } 314 | } 315 | 316 | // 317 | // This class creates an ostream to the Valkey Log. Each line of output is a single call to the ValkeyLog function 318 | // 319 | class ValkeyLogStreamBuf : public std::streambuf { 320 | std::string line; 321 | ValkeyModuleCtx *ctx; 322 | const char *level; 323 | 324 | public: 325 | ValkeyLogStreamBuf(ValkeyModuleCtx *_ctx, const char *_level) : ctx(_ctx), level(_level) {} 326 | ~ValkeyLogStreamBuf() { 327 | if (!line.empty()) { 328 | ValkeyModule_Log(ctx, level, "%s", line.c_str()); 329 | } 330 | } 331 | std::streamsize xsputn(const char *p, std::streamsize n) { 332 | for (std::streamsize i = 0; i < n; ++i) { 333 | overflow(p[i]); 334 | } 335 | return n; 336 | } 337 | int overflow(int c) { 338 | if (c == '\n' || c == EOF) { 339 | ValkeyModule_Log(ctx, level, "%s", line.c_str()); 340 | line.resize(0); 341 | } else { 342 | line += c; 343 | } 344 | return c; 345 | } 346 | }; 347 | 348 | void DumpRedactedJValue(const JValue &v, ValkeyModuleCtx *ctx, const char *level) { 349 | ValkeyLogStreamBuf b(ctx, level); 350 | std::ostream buf(&b); 351 | DumpRedactedJValue(buf, v); 352 | } 353 | -------------------------------------------------------------------------------- /tst/unit/keytable_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "json/dom.h" 16 | #include "json/alloc.h" 17 | #include "json/stats.h" 18 | #include "json/keytable.h" 19 | #include "json/memory.h" 20 | #include "module_sim.h" 21 | 22 | class PtrWithMetaDataTest : public ::testing::Test { 23 | }; 24 | 25 | TEST_F(PtrWithMetaDataTest, t) { 26 | memory_traps_control(false); // Necessary so that MEMORY_VALIDATE buried in getPointer doesn't croak on bad memory 27 | EXPECT_EQ(0x7FFFF, PtrWithMetaData::METADATA_MASK); 28 | for (size_t i = 1; i & 0x7FFFF; i <<= 1) { 29 | size_t var = 0xdeadbeeffeedfeddull; 30 | PtrWithMetaData p(&var, i); 31 | EXPECT_EQ(&*p, &var); 32 | EXPECT_EQ(*p, var); 33 | EXPECT_EQ(size_t(p.getMetaData()), i); 34 | p.clear(); 35 | EXPECT_EQ(p.getMetaData(), 0); 36 | p.setMetaData(i); 37 | EXPECT_EQ(size_t(p.getMetaData()), i); 38 | } 39 | for (size_t i = 8; i & 0x0000FFFFFFFFFFF8ull; i <<= 1) { 40 | PtrWithMetaData p(reinterpret_cast(i), 0x7FFFF); 41 | EXPECT_EQ(size_t(&*p), i); 42 | EXPECT_EQ(p.getMetaData(), 0x7FFFF); 43 | } 44 | } 45 | 46 | // Cheap, predictable hash 47 | static size_t hash1(const char *ptr, size_t len) { 48 | (void)ptr; 49 | return len; 50 | } 51 | 52 | extern size_t MAX_FAST_TABLE_SIZE; // in keytable.cc 53 | 54 | class KeyTableTest : public ::testing::Test { 55 | protected: 56 | void SetUp() override { 57 | } 58 | 59 | void TearDown() override { 60 | if (t) { 61 | EXPECT_EQ(t->validate(), ""); 62 | } 63 | delete t; 64 | } 65 | 66 | void Setup1(size_t numShards = 1, size_t (*hf)(const char *, size_t) = hash1) { 67 | setupValkeyModulePointers(); 68 | KeyTable::Config c; 69 | c.malloc = dom_alloc; 70 | c.free = dom_free; 71 | c.hash = hf; 72 | c.numShards = numShards; 73 | t = new KeyTable(c); 74 | } 75 | 76 | KeyTable *t = nullptr; 77 | }; 78 | 79 | TEST_F(KeyTableTest, layoutTest) { 80 | Setup1(); 81 | 82 | size_t bias = 10; 83 | for (size_t slen : {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 84 | 0xFF, 0x100, 0xFFFF, 0x10000, 0xFFFFFF, 0x1000000}) { 85 | std::string s; 86 | s.resize(slen); 87 | for (size_t i = 0; i < slen; ++i) { 88 | s[i] = i + bias; 89 | } 90 | KeyTable_Layout *l = KeyTable_Layout::makeLayout(dom_alloc, s.data(), s.length(), 0, false); 91 | ASSERT_EQ(l->getLength(), slen); 92 | for (size_t i = 0; i < slen; ++i) { 93 | ASSERT_EQ(0xFF & (i + bias), 0xFF & (l->getText()[i])); 94 | } 95 | dom_free(l); 96 | bias++; 97 | } 98 | } 99 | 100 | TEST_F(KeyTableTest, testInitialization) { 101 | Setup1(); 102 | EXPECT_EQ(t->validate(), ""); 103 | EXPECT_GT(malloced, 0); 104 | EXPECT_EQ(t->validate(), ""); 105 | auto s = t->getStats(); 106 | EXPECT_EQ(s.size, 0); 107 | EXPECT_EQ(s.handles, 0); 108 | EXPECT_EQ(s.bytes, 0); 109 | EXPECT_GT(s.maxTableSize, 0); 110 | EXPECT_GT(s.totalTable, 0); 111 | EXPECT_EQ(s.rehashes, 0); 112 | EXPECT_EQ(s.maxSearch, 0); 113 | delete t; 114 | t = nullptr; 115 | EXPECT_EQ(malloced, 0); 116 | } 117 | 118 | TEST_F(KeyTableTest, testDuplication) { 119 | std::string e = "Empty"; 120 | Setup1(); 121 | auto f = t->getFactors(); 122 | f.maxLoad = 1.0; // No rehashes until we're full.... 123 | f.minLoad = std::numeric_limits::min(); 124 | t->setFactors(f); 125 | KeyTable_Handle h1 = t->makeHandle(e); 126 | EXPECT_TRUE(h1); 127 | EXPECT_EQ(t->validate(), ""); 128 | KeyTable_Handle h2 = t->makeHandle(e); 129 | EXPECT_EQ(t->validate(), ""); 130 | EXPECT_TRUE(h2); 131 | EXPECT_EQ(h1, h2); 132 | EXPECT_EQ(&*h1, &*h2); 133 | auto s = t->getStats(); 134 | EXPECT_EQ(s.size, 1); 135 | EXPECT_EQ(s.handles, 2); 136 | EXPECT_EQ(s.bytes, 5); 137 | t->destroyHandle(h1); 138 | EXPECT_TRUE(!h1); 139 | EXPECT_EQ(t->validate(), ""); 140 | s = t->getStats(); 141 | EXPECT_EQ(s.size, 1); 142 | EXPECT_EQ(s.handles, 1); 143 | EXPECT_EQ(s.bytes, 5); 144 | t->destroyHandle(h2); 145 | EXPECT_TRUE(!h2); 146 | EXPECT_EQ(t->validate(), ""); 147 | s = t->getStats(); 148 | EXPECT_EQ(s.rehashes, 0); 149 | } 150 | 151 | TEST_F(KeyTableTest, testClone) { 152 | std::string e = "Empty"; 153 | Setup1(); 154 | auto f = t->getFactors(); 155 | f.maxLoad = 1.0; // No rehashes until we're full.... 156 | f.minLoad = std::numeric_limits::min(); 157 | t->setFactors(f); 158 | KeyTable_Handle h1 = t->makeHandle(e); 159 | EXPECT_TRUE(h1); 160 | EXPECT_EQ(t->validate(), ""); 161 | KeyTable_Handle h2 = t->clone(h1); 162 | EXPECT_EQ(t->validate(), ""); 163 | EXPECT_TRUE(h2); 164 | EXPECT_EQ(h1, h2); 165 | EXPECT_EQ(&*h1, &*h2); 166 | auto s = t->getStats(); 167 | EXPECT_EQ(s.size, 1); 168 | EXPECT_EQ(s.handles, 2); 169 | EXPECT_EQ(s.bytes, 5); 170 | t->destroyHandle(h1); 171 | EXPECT_TRUE(!h1); 172 | EXPECT_EQ(t->validate(), ""); 173 | s = t->getStats(); 174 | EXPECT_EQ(s.size, 1); 175 | EXPECT_EQ(s.handles, 1); 176 | EXPECT_EQ(s.bytes, 5); 177 | t->destroyHandle(h2); 178 | EXPECT_TRUE(!h2); 179 | EXPECT_EQ(t->validate(), ""); 180 | s = t->getStats(); 181 | EXPECT_EQ(s.rehashes, 0); 182 | } 183 | 184 | TEST_F(KeyTableTest, SimpleRehash) { 185 | Setup1(1); // 4 element table is the minimum. 186 | auto f = t->getFactors(); 187 | f.maxLoad = 1.0; // No rehashes until we're full.... 188 | f.minLoad = std::numeric_limits::min(); 189 | f.grow = 1.0; 190 | f.shrink = 0.5; 191 | t->setFactors(f); 192 | std::vector h; 193 | std::vector keys; 194 | std::string k = ""; 195 | for (size_t i = 0; i < 4; ++i) { 196 | h.push_back(t->makeHandle(k)); 197 | keys.push_back(k); 198 | auto s = t->getStats(); 199 | EXPECT_EQ(s.size, i+1); 200 | EXPECT_EQ(s.rehashes, 0); 201 | EXPECT_EQ(t->validate(), ""); 202 | k += '*'; 203 | } 204 | for (size_t i = 4; i < 8; ++i) { 205 | auto f = t->getFactors(); 206 | f.maxLoad = i == 4 ? .5 : 1.0; // No rehashes until we're full.... 207 | t->setFactors(f); 208 | 209 | h.push_back(t->makeHandle(k)); 210 | keys.push_back(k); 211 | auto s = t->getStats(); 212 | EXPECT_EQ(s.size, i+1); 213 | EXPECT_EQ(s.rehashes, i == 4 ? 1 : 0); 214 | EXPECT_EQ(t->validate(), ""); 215 | k += '*'; 216 | } 217 | // 218 | // Now shrink 219 | // 220 | for (size_t i = 0; i < 4; ++i) { 221 | t->destroyHandle(h.back()); 222 | h.pop_back(); 223 | auto s = t->getStats(); 224 | EXPECT_EQ(s.rehashes, 0); 225 | EXPECT_EQ(t->validate(), ""); 226 | } 227 | // Next destroyHandle should case a rehash. 228 | for (size_t i = 0; i < 4; ++i) { 229 | auto f = t->getFactors(); 230 | f.minLoad = i == 0 ? .5f : std::numeric_limits::min(); // No rehashes until we're full.... 231 | t->setFactors(f); 232 | t->destroyHandle(h.back()); 233 | h.pop_back(); 234 | auto s = t->getStats(); 235 | EXPECT_EQ(s.maxTableSize, 4); 236 | EXPECT_EQ(s.rehashes, i == 0 ? 1 : 0); 237 | EXPECT_EQ(t->validate(), ""); 238 | } 239 | } 240 | 241 | // 242 | // Generate some strings, duplicates are ok. 243 | // Because the hash is the length + the last character the total number of unique strings 244 | // is only 10x of the max length (from the random distribution) 245 | // 246 | std::default_random_engine generator(0); 247 | std::uniform_int_distribution dice(0, 10000); // there are actually ~10x this number of unique strings 248 | size_t make_rand() { 249 | return dice(generator); 250 | } 251 | 252 | std::string make_key() { 253 | size_t len = make_rand(); 254 | size_t lastDigit = make_rand() % 10; 255 | std::string k; 256 | for (size_t i = 0; i < len; ++i) k += '*'; 257 | k += '0' + lastDigit; 258 | return k; 259 | } 260 | 261 | TEST_F(KeyTableTest, BigTest) { 262 | // 263 | // Make a zillion keys, Yes, there will be lots of duplicates -> Intentionally 264 | // 265 | for (size_t ft : { 1 << 8, 1 << 10, 1 << 12}) { 266 | MAX_FAST_TABLE_SIZE = ft; 267 | for (size_t numShards : {1, 2}) { 268 | for (size_t numKeys : {1000}) { 269 | Setup1(numShards); 270 | auto f = t->getFactors(); 271 | f.grow = 1.1; // Grow slowly 272 | f.maxLoad = .95; // Let the table get REALLY full between hashes 273 | t->setFactors(f); 274 | std::vector h; 275 | std::vector k; 276 | for (size_t i = 0; i < numKeys; ++i) { 277 | k.push_back(make_key()); 278 | h.push_back(t->makeHandle(k.back().c_str(), k.back().length())); 279 | if (0 == (i & 0xFF)) { 280 | EXPECT_EQ(t->validate(), ""); 281 | } 282 | } 283 | auto s = t->getStats(); 284 | EXPECT_EQ(s.handles, k.size()); 285 | EXPECT_LT(s.size, k.size()); // must have at least one duplicate 286 | EXPECT_GT(s.rehashes, 5); // should have had several rehashes 287 | // 288 | // now delete them SLOWLY with lots of rehashes 289 | // 290 | f = t->getFactors(); 291 | f.shrink = .05; // Shrink slowly 292 | f.minLoad = .9; // Let the table get REALLY full between hashes 293 | t->setFactors(f); 294 | for (size_t i = 0; i < numKeys; ++i) { 295 | t->destroyHandle(h[i]); 296 | if (0 == (i & 0xFF)) { 297 | EXPECT_EQ(t->validate(), ""); 298 | } 299 | } 300 | // 301 | // Teardown. 302 | // 303 | EXPECT_EQ(t->validate(), ""); 304 | s = t->getStats(); 305 | EXPECT_GT(s.rehashes, 10); 306 | EXPECT_EQ(s.size, 0); 307 | delete t; 308 | t = nullptr; 309 | } 310 | } 311 | } 312 | } 313 | 314 | TEST_F(KeyTableTest, StuckKeys) { 315 | Setup1(1); 316 | KeyTable_Layout::setMaxRefCount(3); 317 | std::string e = "Empty"; 318 | KeyTable_Handle h1 = t->makeHandle(e); 319 | KeyTable_Handle h2 = t->makeHandle(e); 320 | KeyTable_Handle h3 = t->makeHandle(e); 321 | KeyTable_Handle h4 = t->makeHandle(e); 322 | EXPECT_EQ(t->validate(), ""); 323 | auto s = t->getStats(); 324 | EXPECT_EQ(s.size, 1); 325 | EXPECT_EQ(s.stuckKeys, 1); 326 | EXPECT_EQ(s.handles, 4); 327 | t->destroyHandle(h1); 328 | t->destroyHandle(h2); 329 | t->destroyHandle(h3); 330 | t->destroyHandle(h4); 331 | s = t->getStats(); 332 | EXPECT_EQ(s.stuckKeys, 1); 333 | EXPECT_EQ(s.size, 1); 334 | EXPECT_EQ(s.handles, 0); 335 | } 336 | 337 | // 338 | // Make a very large shard, check some stats, delete the elements and see if it shrinks 339 | // 340 | extern size_t hash_function(const char *, size_t); 341 | 342 | TEST_F(KeyTableTest, BigShard) { 343 | memory_traps_control(false); 344 | Setup1(1, hash_function); 345 | enum { TABLE_SIZE_BITS = 22 }; // LOG2(Table Size) 346 | enum { TABLE_SIZE = 1ull << TABLE_SIZE_BITS }; 347 | std::vector handles1; 348 | std::vector handles2; 349 | // 350 | // Fill up the table 351 | // 352 | for (size_t i = 0; i < TABLE_SIZE; ++i) { 353 | handles1.push_back(t->makeHandle(std::to_string(i))); 354 | } 355 | auto s = t->getStats(); 356 | EXPECT_EQ(s.size, TABLE_SIZE); 357 | EXPECT_EQ(s.handles, TABLE_SIZE); 358 | EXPECT_LE(s.rehashes, TABLE_SIZE_BITS); 359 | // 360 | // Check hash table distribution 361 | // 362 | auto ls = t->getLongStats(2); 363 | EXPECT_EQ(ls.runs.size(), 2); 364 | EXPECT_LT(ls.runs.rbegin()->first, 100); // Only look at second longest run 365 | // 366 | // Duplicate add of Handle 367 | // 368 | for (size_t i = 0; i < TABLE_SIZE; ++i) { 369 | handles2.push_back(t->makeHandle(std::to_string(i))); 370 | EXPECT_EQ(handles1[i], handles2[i]); 371 | } 372 | s = t->getStats(); 373 | EXPECT_EQ(s.size, TABLE_SIZE); 374 | EXPECT_LE(s.rehashes, 0); 375 | EXPECT_EQ(s.handles, 2*TABLE_SIZE); 376 | // 377 | // Now, delete each handle once. Basically nothing about the table should change 378 | // 379 | for (auto& h : handles1) { t->destroyHandle(h); } 380 | s = t->getStats(); 381 | EXPECT_EQ(s.size, TABLE_SIZE); 382 | EXPECT_EQ(s.handles, TABLE_SIZE); 383 | EXPECT_EQ(s.maxSearch, 0); 384 | EXPECT_EQ(s.rehashes, 0); 385 | // 386 | // Now empty the table 387 | // 388 | for (auto& h : handles2) { t->destroyHandle(h); } 389 | s = t->getStats(); 390 | EXPECT_EQ(s.size, 0); 391 | EXPECT_EQ(s.handles, 0); 392 | EXPECT_GT(s.rehashes, TABLE_SIZE_BITS - 3); // Minimum table size 393 | } 394 | -------------------------------------------------------------------------------- /src/rapidjson/prettywriter.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_PRETTYWRITER_H_ 16 | #define RAPIDJSON_PRETTYWRITER_H_ 17 | 18 | #include 19 | #include "json/json.h" 20 | 21 | #ifdef __GNUC__ 22 | RAPIDJSON_DIAG_PUSH 23 | RAPIDJSON_DIAG_OFF(effc++) 24 | #endif 25 | 26 | #if defined(__clang__) 27 | RAPIDJSON_DIAG_PUSH 28 | RAPIDJSON_DIAG_OFF(c++98-compat) 29 | #endif 30 | 31 | RAPIDJSON_NAMESPACE_BEGIN 32 | 33 | //! Combination of PrettyWriter format flags. 34 | /*! \see PrettyWriter::SetFormatOptions 35 | */ 36 | enum PrettyFormatOptions { 37 | kFormatDefault = 0, //!< Default pretty formatting. 38 | kFormatSingleLineArray = 1 //!< Format arrays on a single line. 39 | }; 40 | 41 | //! Writer with indentation and spacing. 42 | /*! 43 | \tparam OutputStream Type of output os. 44 | \tparam SourceEncoding Encoding of source string. 45 | \tparam TargetEncoding Encoding of output stream. 46 | \tparam StackAllocator Type of allocator for allocating memory of stack. 47 | */ 48 | template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> 49 | class PrettyWriter : public Writer { 50 | public: 51 | typedef Writer Base; 52 | typedef typename Base::Ch Ch; 53 | 54 | //! Constructor 55 | /*! \param os Output stream. 56 | \param allocator User supplied allocator. If it is null, it will create a private one. 57 | \param levelDepth Initial capacity of stack. 58 | */ 59 | explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : 60 | Base(os, allocator, levelDepth), formatOptions_(kFormatDefault), initialLevel(0), curDepth(0), maxDepth(0) {} 61 | 62 | 63 | explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : 64 | Base(allocator, levelDepth), formatOptions_(kFormatDefault), initialLevel(0), curDepth(0), maxDepth(0) {} 65 | 66 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 67 | PrettyWriter(PrettyWriter&& rhs) : 68 | Base(std::forward(rhs)), formatOptions_(rhs.formatOptions_), 69 | newline_(rhs.newline_), indent_(rhs.indent_), space_(rhs.space_), initialLevel(rhs.initialLevel_) {} 70 | #endif 71 | 72 | //! Set pretty writer formatting options. 73 | /*! \param options Formatting options. 74 | */ 75 | PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { 76 | formatOptions_ = options; 77 | return *this; 78 | } 79 | PrettyWriter& SetNewline(const std::string_view &newline) { 80 | newline_ = newline; 81 | return *this; 82 | } 83 | PrettyWriter& SetIndent(const std::string_view &indent) { 84 | indent_ = indent; 85 | return *this; 86 | } 87 | PrettyWriter& SetSpace(const std::string_view &space) { 88 | space_ = space; 89 | return *this; 90 | } 91 | PrettyWriter& SetInitialLevel(size_t il) { 92 | initialLevel = il; 93 | return *this; 94 | } 95 | 96 | /*! @name Implementation of Handler 97 | \see Handler 98 | */ 99 | //@{ 100 | 101 | bool Null() { PrettyPrefix(kNullType); return Base::EndValue(Base::WriteNull()); } 102 | bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::EndValue(Base::WriteBool(b)); } 103 | bool Int(int i) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt(i)); } 104 | bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint(u)); } 105 | bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt64(i64)); } 106 | bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint64(u64)); } 107 | bool Double(double d) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteDouble(d)); } 108 | 109 | bool RawNumber(const Ch* str, SizeType length, bool copy = false) { 110 | RAPIDJSON_ASSERT(str != 0); 111 | (void)copy; 112 | PrettyPrefix(kNumberType); 113 | return Base::EndValue(Base::WriteDouble(str, length)); 114 | } 115 | 116 | bool String(const Ch* str, SizeType length, bool copy = false) { 117 | RAPIDJSON_ASSERT(str != 0); 118 | (void)copy; 119 | PrettyPrefix(kStringType); 120 | return Base::EndValue(Base::WriteString(str, length)); 121 | } 122 | 123 | #if RAPIDJSON_HAS_STDSTRING 124 | bool String(const std::basic_string& str) { 125 | return String(str.data(), SizeType(str.size())); 126 | } 127 | #endif 128 | 129 | size_t GetMaxDepth() { 130 | return maxDepth; 131 | } 132 | 133 | bool StartObject() { 134 | IncrDepth(); 135 | PrettyPrefix(kObjectType); 136 | new (Base::level_stack_.template Push()) typename Base::Level(false); 137 | return Base::WriteStartObject(); 138 | } 139 | 140 | bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } 141 | 142 | #if RAPIDJSON_HAS_STDSTRING 143 | bool Key(const std::basic_string& str) { 144 | return Key(str.data(), SizeType(str.size())); 145 | } 146 | #endif 147 | 148 | bool EndObject(SizeType memberCount = 0) { 149 | (void)memberCount; 150 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object 151 | RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object 152 | RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value 153 | 154 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 155 | 156 | if (!empty) { 157 | WriteNewline(); 158 | WriteIndent(); 159 | } 160 | bool ret = Base::EndValue(Base::WriteEndObject()); 161 | (void)ret; 162 | RAPIDJSON_ASSERT(ret == true); 163 | if (Base::level_stack_.Empty()) // end of json text 164 | Base::Flush(); 165 | DecrDepth(); 166 | return true; 167 | } 168 | 169 | bool StartArray() { 170 | IncrDepth(); 171 | PrettyPrefix(kArrayType); 172 | new (Base::level_stack_.template Push()) typename Base::Level(true); 173 | return Base::WriteStartArray(); 174 | } 175 | 176 | bool EndArray(SizeType memberCount = 0) { 177 | (void)memberCount; 178 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); 179 | RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); 180 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 181 | 182 | if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { 183 | WriteNewline(); 184 | WriteIndent(); 185 | } 186 | bool ret = Base::EndValue(Base::WriteEndArray()); 187 | (void)ret; 188 | RAPIDJSON_ASSERT(ret == true); 189 | if (Base::level_stack_.Empty()) // end of json text 190 | Base::Flush(); 191 | DecrDepth(); 192 | return true; 193 | } 194 | 195 | //@} 196 | 197 | /*! @name Convenience extensions */ 198 | //@{ 199 | 200 | //! Simpler but slower overload. 201 | bool String(const Ch* str) { return String(str, internal::StrLen(str)); } 202 | bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } 203 | 204 | //@} 205 | 206 | //! Write a raw JSON value. 207 | /*! 208 | For user to write a stringified JSON as a value. 209 | 210 | \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. 211 | \param length Length of the json. 212 | \param type Type of the root of json. 213 | \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. 214 | */ 215 | bool RawValue(const Ch* json, size_t length, Type type) { 216 | RAPIDJSON_ASSERT(json != 0); 217 | PrettyPrefix(type); 218 | return Base::EndValue(Base::WriteRawValue(json, length)); 219 | } 220 | 221 | protected: 222 | void PrettyPrefix(Type type) { 223 | (void)type; 224 | if (Base::level_stack_.GetSize() != 0) { // this value is not at root 225 | typename Base::Level* level = Base::level_stack_.template Top(); 226 | 227 | if (level->inArray) { 228 | if (level->valueCount > 0) { 229 | Base::os_->Put(','); // add comma if it is not the first element in array 230 | if (formatOptions_ & kFormatSingleLineArray) 231 | WriteSpace(); 232 | } 233 | 234 | if (!(formatOptions_ & kFormatSingleLineArray)) { 235 | WriteNewline(); 236 | WriteIndent(); 237 | } 238 | } 239 | else { // in object 240 | if (level->valueCount > 0) { 241 | if (level->valueCount % 2 == 0) { 242 | Base::os_->Put(','); 243 | WriteNewline(); 244 | } 245 | else { 246 | Base::os_->Put(':'); 247 | WriteSpace(); 248 | } 249 | } 250 | else 251 | WriteNewline(); 252 | 253 | if (level->valueCount % 2 == 0) 254 | WriteIndent(); 255 | } 256 | if (!level->inArray && level->valueCount % 2 == 0) 257 | RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name 258 | level->valueCount++; 259 | } 260 | else { 261 | RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. 262 | Base::hasRoot_ = true; 263 | } 264 | } 265 | void WriteStringView(const std::string_view& v) { 266 | if (!v.empty()) { 267 | size_t sz = v.size(); 268 | char *buf = Base::os_->Push(sz); 269 | v.copy(buf, sz); 270 | } 271 | } 272 | void WriteString(const char *ptr, size_t len, bool noescape) { 273 | if (noescape) { 274 | char *p = Base::os_->Push(len + 2); 275 | p[0] = '"'; 276 | std::memcpy(p + 1, ptr, len); 277 | p[len + 1] = '"'; 278 | } else { 279 | Base::WriteString(ptr, len); 280 | } 281 | } 282 | void WriteNewline() { WriteStringView(newline_); } 283 | void WriteSpace() { WriteStringView(space_); } 284 | void WriteIndent() { 285 | size_t count = initialLevel + (Base::level_stack_.GetSize() / sizeof(typename Base::Level)); 286 | for (size_t i = 0; i < count; ++i) WriteStringView(indent_); 287 | } 288 | 289 | public: 290 | // 291 | // Accelerated write when there's definitely no format 292 | // 293 | template 294 | void FastWrite(JValue &value, size_t *max_depth) { 295 | *max_depth = 0; 296 | FastWrite_internal(value, 0, max_depth); 297 | } 298 | 299 | PrettyFormatOptions formatOptions_; 300 | std::string_view newline_; 301 | std::string_view indent_; 302 | std::string_view space_; 303 | size_t initialLevel; 304 | 305 | private: 306 | // Prohibit copy constructor & assignment operator. 307 | PrettyWriter(const PrettyWriter&); 308 | PrettyWriter& operator=(const PrettyWriter&); 309 | size_t curDepth; 310 | size_t maxDepth; 311 | 312 | void IncrDepth() { 313 | curDepth++; 314 | if (curDepth > maxDepth) maxDepth = curDepth; 315 | } 316 | 317 | void DecrDepth() { 318 | RAPIDJSON_ASSERT(curDepth > 0); 319 | curDepth--; 320 | } 321 | 322 | template 323 | void FastWrite_internal(JValue &value, const size_t level, size_t *max_depth) { 324 | if (level > *max_depth) *max_depth = level; 325 | 326 | bool firstElement; 327 | switch (value.GetType()) { 328 | case kStringType: 329 | WriteString(value.GetString(), value.GetStringLength(), value.IsNoescape()); 330 | break; 331 | case kNullType: 332 | Base::WriteNull(); 333 | break; 334 | case kFalseType: 335 | Base::WriteBool(false); 336 | break; 337 | case kTrueType: 338 | Base::WriteBool(true); 339 | break; 340 | case kObjectType: 341 | Base::os_->Put('{'); 342 | firstElement = true; 343 | for (typename JValue::ConstMemberIterator m = value.MemberBegin(); m != value.MemberEnd(); ++m) { 344 | if (!firstElement) { 345 | Base::os_->Put(','); 346 | } else { 347 | firstElement = false; 348 | } 349 | WriteString(m->name.GetString(), m->name.GetStringLength(), m->name.IsNoescape()); 350 | Base::os_->Put(':'); 351 | FastWrite_internal(m->value, level + 1, max_depth); 352 | } 353 | Base::os_->Put('}'); 354 | break; 355 | case kArrayType: 356 | Base::os_->Put('['); 357 | firstElement = true; 358 | for (typename JValue::ConstValueIterator v = value.Begin(); v != value.End(); ++v) { 359 | if (!firstElement) { 360 | Base::os_->Put(','); 361 | } else { 362 | firstElement = false; 363 | } 364 | FastWrite_internal(*v, level + 1, max_depth); 365 | } 366 | Base::os_->Put(']'); 367 | break; 368 | default: 369 | RAPIDJSON_ASSERT(value.GetType() == kNumberType); 370 | if (value.IsDouble()) { 371 | Base::WriteDouble(value.GetDoubleString(), value.GetDoubleStringLength()); 372 | } 373 | else if (value.IsInt()) Base::WriteInt(value.GetInt()); 374 | else if (value.IsUint()) Base::WriteUint(value.GetUint()); 375 | else if (value.IsInt64()) Base::WriteInt64(value.GetInt64()); 376 | else Base::WriteUint64(value.GetUint64()); 377 | break; 378 | } 379 | } 380 | 381 | }; 382 | 383 | RAPIDJSON_NAMESPACE_END 384 | 385 | #if defined(__clang__) 386 | RAPIDJSON_DIAG_POP 387 | #endif 388 | 389 | #ifdef __GNUC__ 390 | RAPIDJSON_DIAG_POP 391 | #endif 392 | 393 | #endif // RAPIDJSON_RAPIDJSON_H_ 394 | --------------------------------------------------------------------------------