├── .clang-format ├── .clang-tidy ├── .config └── typos.toml ├── .devcontainer ├── Dockerfile ├── devcontainer_base.json └── setup.sh ├── .github └── workflows │ ├── clang_tidy_format.yml │ ├── integration_tests-asan.yml │ ├── integration_tests.yml │ ├── spell_check.yml │ ├── trigger-search-release.yml │ ├── unittests-asan.yml │ └── unittests.yml ├── .gitignore ├── .vscode ├── cspell.json └── settings.json ├── 00-RELEASENOTES ├── CMakeLists.txt ├── COMMANDS.md ├── LICENSE ├── QUICK_START.md ├── README.md ├── build.sh ├── ci ├── asan.supp ├── build_ubuntu.sh ├── check_changes.sh ├── check_clang_format.sh ├── entrypoint.sh └── refresh_comp_db.sh ├── cmake └── Modules │ ├── linux_utils.cmake │ ├── protobuf_generate.cmake │ └── valkey_search.cmake ├── debs └── .placeholder ├── rfc ├── TEMPLATE.md └── rdb-format.md ├── src ├── CMakeLists.txt ├── acl.cc ├── acl.h ├── attribute.h ├── attribute_data_type.cc ├── attribute_data_type.h ├── commands │ ├── CMakeLists.txt │ ├── commands.h │ ├── filter_parser.cc │ ├── filter_parser.h │ ├── ft._list.json │ ├── ft.create.json │ ├── ft.dropindex.json │ ├── ft.info.json │ ├── ft.search.json │ ├── ft_create.cc │ ├── ft_create_parser.cc │ ├── ft_create_parser.h │ ├── ft_dropindex.cc │ ├── ft_info.cc │ ├── ft_list.cc │ ├── ft_search.cc │ ├── ft_search.h │ ├── ft_search_parser.cc │ └── ft_search_parser.h ├── coordinator │ ├── CMakeLists.txt │ ├── client.cc │ ├── client.h │ ├── client_pool.h │ ├── coordinator.proto │ ├── grpc_suspender.cc │ ├── grpc_suspender.h │ ├── metadata_manager.cc │ ├── metadata_manager.h │ ├── search_converter.cc │ ├── search_converter.h │ ├── server.cc │ ├── server.h │ └── util.h ├── index_schema.cc ├── index_schema.h ├── index_schema.proto ├── indexes │ ├── CMakeLists.txt │ ├── index_base.h │ ├── numeric.cc │ ├── numeric.h │ ├── tag.cc │ ├── tag.h │ ├── vector_base.cc │ ├── vector_base.h │ ├── vector_flat.cc │ ├── vector_flat.h │ ├── vector_hnsw.cc │ └── vector_hnsw.h ├── keyspace_event_manager.cc ├── keyspace_event_manager.h ├── metrics.h ├── module_loader.cc ├── query │ ├── CMakeLists.txt │ ├── fanout.cc │ ├── fanout.h │ ├── planner.cc │ ├── planner.h │ ├── predicate.cc │ ├── predicate.h │ ├── response_generator.cc │ ├── response_generator.h │ ├── search.cc │ └── search.h ├── rdb_section.proto ├── rdb_serialization.cc ├── rdb_serialization.h ├── schema_manager.cc ├── schema_manager.h ├── server_events.cc ├── server_events.h ├── utils │ ├── CMakeLists.txt │ ├── allocator.cc │ ├── allocator.h │ ├── intrusive_list.h │ ├── intrusive_ref_count.h │ ├── lru.h │ ├── patricia_tree.h │ ├── segment_tree.h │ ├── string_interning.cc │ └── string_interning.h ├── valkey_search.cc ├── valkey_search.h ├── valkey_search_options.cc ├── valkey_search_options.h ├── vector_externalizer.cc └── vector_externalizer.h ├── submodules ├── CMakeLists.txt └── package-submodules.sh ├── testing ├── CMakeLists.txt ├── acl_test.cc ├── attribute_data_type_test.cc ├── common.cc ├── common.h ├── coordinator │ ├── CMakeLists.txt │ ├── common.h │ └── metadata_manager_test.cc ├── filter_test.cc ├── ft_create_parser_test.cc ├── ft_create_test.cc ├── ft_dropindex_test.cc ├── ft_info_test.cc ├── ft_list_test.cc ├── ft_search_parser_test.cc ├── ft_search_test.cc ├── index_schema_test.cc ├── integration │ ├── .gitignore │ ├── CMakeLists.txt │ ├── requirements.txt │ ├── run.sh │ ├── stability_runner.py │ ├── stability_test.py │ ├── utils.py │ └── vector_search_integration_test.py ├── keyspace_event_manager_test.cc ├── multi_exec_test.cc ├── numeric_index_test.cc ├── query │ ├── CMakeLists.txt │ ├── fanout_test.cc │ └── response_generator_test.cc ├── rdb_serialization_test.cc ├── schema_manager_test.cc ├── search_test.cc ├── segment_tree_test.cc ├── server_events_test.cc ├── tag_index_test.cc ├── utils │ ├── CMakeLists.txt │ ├── allocator_test.cc │ ├── intrusive_list_test.cc │ ├── intrusive_ref_count_test.cc │ ├── lru_test.cc │ ├── patricia_tree_test.cc │ ├── segment_tree_test.cc │ └── string_interning_test.cc ├── valkey_search_test.cc ├── vector_externalizer_test.cc └── vector_test.cc ├── third_party ├── CMakeLists.txt ├── hdrhistogram_c │ ├── CMakeLists.txt │ ├── COPYING.txt │ ├── LICENSE │ ├── README.md │ ├── examples │ │ ├── CMakeLists.txt │ │ ├── hdr_decoder.c │ │ └── hiccup.c │ └── src │ │ ├── CMakeLists.txt │ │ ├── hdr_atomic.h │ │ ├── hdr_encoding.c │ │ ├── hdr_encoding.h │ │ ├── hdr_endian.h │ │ ├── hdr_histogram.c │ │ ├── hdr_histogram.h │ │ ├── hdr_interval_recorder.c │ │ ├── hdr_interval_recorder.h │ │ ├── hdr_tests.h │ │ ├── hdr_thread.c │ │ ├── hdr_thread.h │ │ ├── hdr_time.c │ │ ├── hdr_time.h │ │ ├── hdr_writer_reader_phaser.c │ │ └── hdr_writer_reader_phaser.h ├── hnswlib │ ├── CMakeLists.txt │ ├── LICENSE │ ├── OWNERS │ ├── bruteforce.h │ ├── hnswalg.h │ ├── hnswlib.h │ ├── index.proto │ ├── iostream.h │ ├── simsimd.h │ ├── space_ip.h │ ├── space_l2.h │ ├── stop_condition.h │ └── visited_list_pool.h └── simsimd │ ├── CMakeLists.txt │ ├── LICENSE │ ├── METADATA │ ├── OWNERS │ ├── VERSION │ ├── bluze.textproto │ ├── c │ ├── CMakeLists.txt │ └── lib.c │ ├── include │ └── simsimd │ │ ├── binary.h │ │ ├── dot.h │ │ ├── geospatial.h │ │ ├── probability.h │ │ ├── simsimd.h │ │ ├── spatial.h │ │ └── types.h │ └── third_party.simsimd.blueprint └── vmsdk ├── CMakeLists.txt ├── LICENSE ├── README.md ├── src ├── CMakeLists.txt ├── blocked_client.cc ├── blocked_client.h ├── command_parser.h ├── concurrency.cc ├── concurrency.h ├── latency_sampler.h ├── log.cc ├── log.h ├── managed_pointers.h ├── memory_allocation.cc ├── memory_allocation.h ├── memory_allocation_overrides.cc ├── memory_allocation_overrides.h ├── module.cc ├── module.h ├── module_config.cc ├── module_config.h ├── module_type.cc ├── module_type.h ├── status │ ├── CMakeLists.txt │ ├── source_location.h │ ├── status_builder.cc │ ├── status_builder.h │ └── status_macros.h ├── testing_infra │ ├── CMakeLists.txt │ ├── module.h │ ├── utils.cc │ └── utils.h ├── thread_pool.cc ├── thread_pool.h ├── thread_safe_vector.h ├── time_sliced_mrmw_mutex.cc ├── time_sliced_mrmw_mutex.h ├── type_conversions.h ├── utils.cc ├── utils.h └── valkey_module_api │ ├── .clang-tidy │ ├── CMakeLists.txt │ └── valkey_module.h ├── testing ├── CMakeLists.txt ├── blocked_client_test.cc ├── command_parser_test.cc ├── concurrency_test.cc ├── configuration_test.cc ├── log_test.cc ├── memory_allocation_test.cc ├── module_test.cc ├── module_type_test.cc ├── mrmw_mutex_test.cc ├── status_macros_test.cc ├── thread_pool_test.cc └── utils_test.cc └── versionscript.lds /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | IndentWidth: 2 3 | ColumnLimit: 80 4 | -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: > 2 | -clang-analyzer-core.NonNullParamChecker, 3 | -clang-analyzer-optin.cplusplus.UninitializedObject, 4 | -clang-diagnostic-builtin-macro-redefined, 5 | -clang-analyzer-core.uninitialized.Assign, 6 | abseil-duration-*, 7 | abseil-faster-strsplit-delimiter, 8 | abseil-no-namespace, 9 | abseil-redundant-strcat-calls, 10 | abseil-str-cat-append, 11 | abseil-string-find-startswith, 12 | abseil-upgrade-duration-conversions, 13 | bugprone-assert-side-effect, 14 | bugprone-unused-raii, 15 | bugprone-use-after-move, 16 | clang-analyzer-core.DivideZero, 17 | misc-unused-using-decls, 18 | modernize-deprecated-headers, 19 | modernize-loop-convert, 20 | modernize-make-shared, 21 | modernize-make-unique, 22 | modernize-return-braced-init-list, 23 | modernize-use-default-member-init, 24 | modernize-use-equals-default, 25 | modernize-use-nullptr, 26 | modernize-use-override, 27 | modernize-use-using, 28 | performance-faster-string-find, 29 | performance-for-range-copy, 30 | performance-inefficient-algorithm, 31 | performance-inefficient-vector-operation, 32 | performance-noexcept-move-constructor, 33 | performance-move-constructor-init, 34 | performance-type-promotion-in-math-fn, 35 | performance-unnecessary-copy-initialization, 36 | readability-braces-around-statements, 37 | readability-container-size-empty, 38 | readability-identifier-naming, 39 | readability-redundant-control-flow, 40 | readability-redundant-member-init, 41 | readability-redundant-smartptr-get, 42 | readability-redundant-string-cstr 43 | 44 | 45 | CheckOptions: 46 | - key: cppcoreguidelines-unused-variable.IgnorePattern 47 | value: "^_$" 48 | - key: bugprone-assert-side-effect.AssertMacros 49 | value: 'ASSERT' 50 | - key: modernize-use-auto.MinTypeNameLength 51 | value: '10' 52 | - key: readability-identifier-naming.ClassCase 53 | value: 'CamelCase' 54 | - key: readability-identifier-naming.EnumCase 55 | value: 'CamelCase' 56 | - key: readability-identifier-naming.EnumConstantPrefix 57 | value: k 58 | - key: readability-identifier-naming.EnumConstantCase 59 | value: 'CamelCase' 60 | - key: readability-identifier-naming.ParameterCase 61 | value: 'lower_case' 62 | - key: readability-identifier-naming.ParameterIgnoredRegexp 63 | value: (^cname_ttl_$) 64 | - key: readability-identifier-naming.PrivateMemberSuffix 65 | value: '_' 66 | - key: readability-identifier-naming.StructCase 67 | value: 'CamelCase' 68 | - key: readability-identifier-naming.TypeAliasCase 69 | value: 'CamelCase' 70 | - key: readability-identifier-naming.TypeAliasIgnoredRegexp 71 | value: '(result_type)' 72 | - key: readability-identifier-naming.UnionCase 73 | value: 'CamelCase' 74 | - key: readability-identifier-naming.FunctionCase 75 | value: 'CamelCase' 76 | - key: readability-identifier-naming.FunctionIgnoredRegexp 77 | value: '^_[A-Z]+.*$' 78 | - key: readability-identifier-naming.GlobalConstantCase 79 | value: 'CamelCase' 80 | - key: readability-identifier-naming.GlobalConstantPrefix 81 | value: k 82 | - key: readability-identifier-naming.MacroDefinitionCase 83 | value: UPPER_CASE 84 | - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp 85 | value: '^[A-Z]+(_[A-Z]+)*_$' 86 | - key: readability-identifier-naming.FunctionIgnoredRegexp 87 | # To have the regex chomped correctly fence all items with `|` (other than first/last) 88 | value: >- 89 | (^FRIEND_TEST$) 90 | 91 | 92 | HeaderFilterRegex: '^./(src|testing)\/.*\.(cc|h)$|^./vmsdk/src/[a-zA-Z0-9_]+(\.cc|\.h)$|^./vmsdk/testing/.*' 93 | #HeaderFilterRegex: '.*' 94 | FormatStyle: file 95 | 96 | WarningsAsErrors: '*' -------------------------------------------------------------------------------- /.config/typos.toml: -------------------------------------------------------------------------------- 1 | # See https://github.com/crate-ci/typos/blob/master/docs/reference.md to configure typos 2 | 3 | [files] 4 | extend-exclude = [ 5 | ] 6 | 7 | 8 | [type.cpp] 9 | extend-ignore-re = [ 10 | "baNAna", 11 | "eXIst", 12 | ] 13 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer_base.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ValkeySearch Development Environment", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "context": ".", 6 | "args": { 7 | "USER_UID": "1000", 8 | "USER_NAME": "ubuntu", 9 | "USER_GID": "1000", 10 | "USER_GNAME": "ubuntu" 11 | } 12 | }, 13 | "customizations": { 14 | "vscode": { 15 | "settings": { 16 | "bazel.buildifierFixOnFormat": true 17 | }, 18 | "extensions": [ 19 | "ms-vscode.cpptools", 20 | "llvm-vs-code-extensions.vscode-clangd", 21 | "xaver.clang-format", 22 | "ms-vscode-remote.remote-containers", 23 | "ms-vscode-remote.remote-ssh", 24 | "ms-vscode-remote.remote-ssh-edit", 25 | "ms-vscode.remote-explorer", 26 | "ms-azuretools.vscode-docker", 27 | "ms-vscode.cmake-tools", 28 | "ms-python.python" 29 | ] 30 | } 31 | }, 32 | "remoteUser": "ubuntu", 33 | "postCreateCommand": "export USER=ubuntu && ./ci/entrypoint.sh && /bin/bash" 34 | } -------------------------------------------------------------------------------- /.devcontainer/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Constants 4 | BOLD_PINK='\e[35;1m' 5 | RESET='\e[0m' 6 | GREEN='\e[32;1m' 7 | RED='\e[31;1m' 8 | BLUE='\e[34;1m' 9 | 10 | user_id=$(id -u) 11 | user_name=$(whoami) 12 | group_id=$(id -g) 13 | group_name=$(id -gn) 14 | 15 | # Define source and destination file paths 16 | source_file=".devcontainer/devcontainer_base.json" 17 | destination_file=".devcontainer/devcontainer.json" 18 | destination_file_tmp=".devcontainer/devcontainer.json.tmp" 19 | 20 | # Check if the source file exists 21 | if [[ ! -f $source_file ]]; then 22 | printf "\n${RED}Source file ${source_file} does not exist. Exiting.${RESET}\n\n" >&2 23 | exit 1 24 | fi 25 | 26 | # Create the destination file with the updated content 27 | sed -e "s/\"USER_UID\": \"1000\"/\"USER_UID\": \"$user_id\"/g" \ 28 | -e "s/\"USER_GID\": \"1000\"/\"USER_GID\": \"$group_id\"/g" \ 29 | -e "s/\"remoteUser\": \"ubuntu\"/\"remoteUser\": \"$user_name\"/g" \ 30 | -e "s/\"USER_NAME\": \"ubuntu\"/\"USER_NAME\": \"$user_name\"/g" \ 31 | -e "s/\"USER_GNAME\": \"ubuntu\"/\"USER_GNAME\": \"$group_name\"/g" \ 32 | -e "s/USER=ubuntu/USER=$user_name/g" \ 33 | "$source_file" > "$destination_file_tmp" 34 | 35 | if [[ $? -eq 0 ]]; then 36 | if [[ ! -f "$destination_file" ]] || ! cmp -s "$destination_file" "$destination_file_tmp"; then 37 | cp $destination_file_tmp $destination_file 38 | fi 39 | rm $destination_file_tmp 40 | printf "\n${GREEN}Success!${RESET}\n\n" 41 | echo " USER_UID: $user_id" 42 | echo " USER_UNAME: $user_name" 43 | echo " USER_GID: $group_id" 44 | echo " USER_GNAME: $group_name" 45 | else 46 | printf "\n${RED}Failed to create the file ${destination_file}.${RESET}\n\n" >&2 47 | exit 1 48 | fi 49 | -------------------------------------------------------------------------------- /.github/workflows/clang_tidy_format.yml: -------------------------------------------------------------------------------- 1 | name: Clang Tidy and Format 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'src/**' 9 | - 'testing/**' 10 | - 'vmsdk/src/**' 11 | - 'vmsdk/testing/**' 12 | pull_request: 13 | paths: 14 | - 'src/**' 15 | - 'testing/**' 16 | - 'vmsdk/src/**' 17 | - 'vmsdk/testing/**' 18 | workflow_dispatch: # allow manual triggering 19 | concurrency: 20 | group: clang-${{ github.head_ref || github.ref }} 21 | cancel-in-progress: true 22 | jobs: 23 | process-files: 24 | runs-on: ubuntu-latest 25 | 26 | steps: 27 | - name: Checkout code 28 | uses: actions/checkout@v3 29 | 30 | # Build the Docker image from the Dockerfile in your repo 31 | - name: Build Docker Image 32 | run: .devcontainer/setup.sh && docker build -t presubmit-image -f .devcontainer/Dockerfile . 33 | 34 | - name: Build Comp Database 35 | id: build-comp-db 36 | run: | 37 | echo "Building CompDB" 38 | docker run --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" presubmit-image bash -c "sudo su ubuntu && export USERNAME='ubuntu' && sudo chown -R ubuntu:ubuntu /workspace && export HOME='/home/ubuntu' && export USERNAME='ubuntu' && ci/refresh_comp_db.sh " 39 | 40 | - name: Determine base branch 41 | id: determine-base 42 | run: | 43 | # If it's a pull request, compare against the base branch. 44 | # If it's a push to main, compare against the previous commit. 45 | if [[ "${{ github.ref }}" != "refs/heads/main" ]]; then 46 | echo "BASE_BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV 47 | else 48 | echo "BASE_BRANCH=$(git rev-parse HEAD~1)" >> $GITHUB_ENV 49 | fi 50 | 51 | - name: Find and process modified files 52 | run: | 53 | echo "Finding modified or added .cc and .h files..." 54 | MODIFIED_FILES=$(git diff --name-only --diff-filter=AM $BASE_BRANCH | grep -E '\.(cc|h)$' || echo "") 55 | 56 | if [[ -z "$MODIFIED_FILES" ]]; then 57 | echo "No .cc or .h files modified." 58 | exit 0 59 | fi 60 | 61 | echo "Processing the following files:" 62 | echo "$MODIFIED_FILES" 63 | 64 | for FILE in $MODIFIED_FILES; do 65 | echo "Running script on $FILE" 66 | docker run --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" clang-tidy-format-image \ 67 | bash -c ' clang-tidy --quiet -p compile_commands.json "$FILE" 2>&1 | tail -n +3 && ci/check_clang_format.sh "$FILE"' 68 | done 69 | 70 | - name: Ensure script execution success 71 | run: echo "Workflow completed successfully." 72 | -------------------------------------------------------------------------------- /.github/workflows/integration_tests-asan.yml: -------------------------------------------------------------------------------- 1 | name: Integration tests ASAN 2 | 3 | on: 4 | pull_request: # This triggers the workflow for pull requests 5 | branches: 6 | - main # Run tests for pull requests targeting the "main" branch 7 | push: 8 | branches: 9 | - main # Optional: Run tests for direct pushes to "main" 10 | workflow_dispatch: # allow manual triggering 11 | 12 | concurrency: 13 | group: integration-tests-${{ github.head_ref || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | integration-tests-asan: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | # Checkout the code from the repository 22 | - name: Checkout Code 23 | uses: actions/checkout@v3 24 | 25 | # Set up Docker to build and run the container 26 | - name: Set up Docker 27 | uses: docker/setup-buildx-action@v2 28 | 29 | # Build the Docker image from the Dockerfile in your repo 30 | - name: Build Docker Image 31 | run: docker build -t presubmit-image -f .devcontainer/Dockerfile . 32 | 33 | - name: Run Integration Tests 34 | run: docker run --privileged --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" presubmit-image sudo ci/build_ubuntu.sh --asan --run-integration-tests 35 | -------------------------------------------------------------------------------- /.github/workflows/integration_tests.yml: -------------------------------------------------------------------------------- 1 | name: Integration tests 2 | 3 | on: 4 | pull_request: # This triggers the workflow for pull requests 5 | branches: 6 | - main # Run tests for pull requests targeting the "main" branch 7 | push: 8 | branches: 9 | - main # Optional: Run tests for direct pushes to "main" 10 | workflow_dispatch: # allow manual triggering 11 | 12 | concurrency: 13 | group: integration-tests-${{ github.head_ref || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | integration-tests: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | # Checkout the code from the repository 22 | - name: Checkout Code 23 | uses: actions/checkout@v3 24 | 25 | # Set up Docker to build and run the container 26 | - name: Set up Docker 27 | uses: docker/setup-buildx-action@v2 28 | 29 | # Build the Docker image from the Dockerfile in your repo 30 | - name: Build Docker Image 31 | run: docker build -t presubmit-image -f .devcontainer/Dockerfile . 32 | 33 | - name: Run Integration Tests 34 | run: docker run --privileged --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" presubmit-image sudo ci/build_ubuntu.sh --run-integration-tests 35 | -------------------------------------------------------------------------------- /.github/workflows/spell_check.yml: -------------------------------------------------------------------------------- 1 | name: Spellcheck 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | concurrency: 8 | group: spellcheck-${{ github.head_ref || github.ref }} 9 | cancel-in-progress: true 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | build: 16 | name: Spellcheck 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v3 22 | 23 | - name: Install typos 24 | uses: taiki-e/install-action@fe9759bf4432218c779595708e80a1aadc85cedc # v2.46.10 25 | with: 26 | tool: typos 27 | 28 | - name: Spell check 29 | run: typos --config=./.config/typos.toml 30 | -------------------------------------------------------------------------------- /.github/workflows/trigger-search-release.yml: -------------------------------------------------------------------------------- 1 | name: Trigger Extension Update - Search 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | inputs: 8 | version: 9 | description: Version of Valkey Search module that was released 10 | required: true 11 | 12 | jobs: 13 | trigger: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Determine version 17 | id: determine-vars 18 | run: | 19 | if [[ "${{ github.event_name }}" == "release" ]]; then 20 | echo "Triggered by a release event." 21 | VERSION=${{ github.event.release.tag_name }} 22 | else 23 | echo "Triggered manually." 24 | VERSION=${{ inputs.version }} 25 | fi 26 | 27 | echo "version=$VERSION" >> $GITHUB_OUTPUT 28 | 29 | - name: Trigger extension update 30 | uses: peter-evans/repository-dispatch@v3 31 | with: 32 | token: ${{ secrets.EXTENSION_PAT }} 33 | repository: ${{ github.repository_owner }}/valkey-extensions 34 | event-type: search-release 35 | client-payload: > 36 | { 37 | "version": "${{ steps.determine-vars.outputs.version }}", 38 | "module": "search" 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/unittests-asan.yml: -------------------------------------------------------------------------------- 1 | name: unittests-asan 2 | 3 | on: 4 | pull_request: # This triggers the workflow for pull requests 5 | branches: 6 | - main # Run tests for pull requests targeting the "main" branch 7 | push: 8 | branches: 9 | - main # Optional: Run tests for direct pushes to "main" 10 | workflow_dispatch: # allow manual triggering 11 | 12 | concurrency: 13 | group: unittests-asan-${{ github.head_ref || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | cmake-tests: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | # Checkout the code from the repository 22 | - name: Checkout Code 23 | uses: actions/checkout@v3 24 | 25 | # Set up Docker to build and run the container 26 | - name: Set up Docker 27 | uses: docker/setup-buildx-action@v2 28 | 29 | # Build the Docker image from the Dockerfile in your repo 30 | - name: Build Docker Image 31 | run: docker build -t presubmit-image -f .devcontainer/Dockerfile . 32 | 33 | - name: Run Tests ASAN 34 | run: docker run --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" presubmit-image sudo ci/build_ubuntu.sh --run-tests --asan 35 | -------------------------------------------------------------------------------- /.github/workflows/unittests.yml: -------------------------------------------------------------------------------- 1 | name: unittests 2 | 3 | on: 4 | pull_request: # This triggers the workflow for pull requests 5 | branches: 6 | - main # Run tests for pull requests targeting the "main" branch 7 | push: 8 | branches: 9 | - main # Optional: Run tests for direct pushes to "main" 10 | workflow_dispatch: # allow manual triggering 11 | 12 | concurrency: 13 | group: unittests-${{ github.head_ref || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | cmake-tests: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | # Checkout the code from the repository 22 | - name: Checkout Code 23 | uses: actions/checkout@v3 24 | 25 | # Set up Docker to build and run the container 26 | - name: Set up Docker 27 | uses: docker/setup-buildx-action@v2 28 | 29 | # Build the Docker image from the Dockerfile in your repo 30 | - name: Build Docker Image 31 | run: docker build -t presubmit-image -f .devcontainer/Dockerfile . 32 | 33 | - name: Run Tests 34 | run: docker run --rm -v "$(pwd):/workspace" --user "ubuntu:ubuntu" presubmit-image sudo ci/build_ubuntu.sh --run-tests 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | resolved_deps.py 3 | *.swp 4 | compile_commands.json 5 | .devcontainer/devcontainer.json 6 | .devcontainer/._devcontainer.json 7 | .codelite/ 8 | .cache/ 9 | build-* 10 | *.workspace 11 | build/ 12 | .build-* 13 | debs/*.deb 14 | -------------------------------------------------------------------------------- /.vscode/cspell.json: -------------------------------------------------------------------------------- 1 | { 2 | // Version of the setting file. Always 0.2 3 | "version": "0.2", 4 | // language - current active spelling language 5 | "language": "en", 6 | "ignorePaths": [ 7 | ".devcontainer/*", 8 | "valkey_module.h", 9 | "BUILD", 10 | "*.yml" 11 | ], 12 | "allowCompoundWords": true, 13 | // words - list of words to be always considered correct 14 | "words": [ 15 | "absl", 16 | "vmsdk", 17 | "redis", 18 | "Valkey", 19 | "nonexistentkey", 20 | "valkeysearch", 21 | "hnsw", 22 | "hnswlib", 23 | "MRMW", 24 | "Externalizer", 25 | "highwayhash", 26 | "mstime", 27 | "NOLINTNEXTLINE", 28 | "synchronistically", 29 | "bazel", 30 | ], 31 | // flagWords - list of words to be always considered incorrect 32 | // This is useful for offensive words and common spelling errors. 33 | // For example "hte" should be "the" 34 | "flagWords": [] 35 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "C_Cpp.default.compileCommands": "compile_commands.json", 3 | "terminal.integrated.defaultProfile.linux": "bash", 4 | "terminal.integrated.defaultProfile.osx": "bash", 5 | "clangd.path": "/usr/bin/clangd", 6 | "C_Cpp.codeAnalysis.runAutomatically": true, 7 | "editor.formatOnSave": true, 8 | "C_Cpp.intelliSenseEngine": "disabled", 9 | "C_Cpp.codeAnalysis.clangTidy.enabled": true, 10 | "C_Cpp.codeAnalysis.clangTidy.useBuildPath": true, 11 | "clangd.arguments": [ 12 | "--clang-tidy", 13 | "--all-scopes-completion", 14 | "--background-index", 15 | "--header-insertion=never", 16 | "--compile-commands-dir=${workspaceFolder}/", 17 | "--query-driver=**" 18 | ], 19 | "clangd.inactiveRegions.useBackgroundHighlight": true, 20 | "clangd.enableCodeCompletion": true 21 | } 22 | -------------------------------------------------------------------------------- /00-RELEASENOTES: -------------------------------------------------------------------------------- 1 | Valkey Search 1.0.0 release notes 2 | ================================= 3 | 4 | ================================================================================ 5 | Valkey Search 1.0.0 RC1 - Released Fri 28 Mar 2025 6 | ================================================================================ 7 | This is the first release candidate of valkey-search 1.0 that is a high-performance Vector Similarity Search engine optimized for AI-driven workloads. It delivers single-digit millisecond latency and high QPS, capable of handling billions of vectors with over 99% recall. 8 | 9 | Valkey-Search allows users to create indexes and perform similarity searches, incorporating complex filters. It supports Approximate Nearest Neighbor (ANN) search with HNSW and exact matching using K-Nearest Neighbors (KNN). Users can index data using either Valkey Hash or Valkey-JSON data types. 10 | 11 | Major API and Functionality 12 | ========================= 13 | * Add the search module data type which can handle RDB load, RDB save, free, and memory usage 14 | * Add the following search module commands: 15 | ** FT.CREATE 16 | ** FT.DROPINDEX 17 | ** FT.INFO 18 | ** FT._LIST 19 | ** FT.SEARCH 20 | * Supported indexes 21 | ** Vector: HNSW and Flat 22 | ** Non-vector: Numeric and Tag 23 | * Index data types include Valkey Hash and Valkey JSON. 24 | * Cluster support 25 | * Linear scaling of keyspace and compute 26 | * ACL support 27 | * RDB serialization of metadata including the search index 28 | * Hybrid queries combining vector and non vector indexes 29 | * Handle key space events for data mutation 30 | * Expose statistics and reporting memory usage to the core valkey engine 31 | 32 | New configurations 33 | =========================== 34 | * Add support for the following configurations: reader-threads, writer-threads, use-coordinator, log-level 35 | 36 | We appreciate the efforts of all who contributed code to this release! 37 | 38 | Yair Gottdenker (yairgott), Allen Samuels (allenss-amazon), Eran Ifrah (eifrah-aws), Jacob Murphy (murphyjacob4), Chanil Jeon (jeon1226), Qu Chen (QuChen88), DP (dpbnasika) 39 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.16) 2 | 3 | # Project definition 4 | 5 | project(ValkeySearch VERSION 1.0.0 LANGUAGES C CXX) 6 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules/") 7 | 8 | # Options 9 | option(BUILD_TESTS "Build valkey-search tests" ON) 10 | option(WITH_SUBMODULES_SYSTEM "Use system submodules" OFF) 11 | option(ASAN_BUILD "Enable address sanitizer build" OFF) 12 | 13 | # Initialise submodules (gRPC, Protobuf, GTest & Abseil) 14 | add_subdirectory(submodules) 15 | 16 | if(ASAN_BUILD) 17 | message(STATUS "Address sanitizer is enabled") 18 | endif() 19 | 20 | message(STATUS "GRPC_CPP_PLUGIN_PATH is set to ${GRPC_CPP_PLUGIN_PATH}") 21 | message(STATUS "protoc_EXE is set to ${protoc_EXE}") 22 | message(STATUS "highwayhash include path: ${HIGHWAY_HASH_INCLUDE_PATH}") 23 | 24 | set(CMAKE_CXX_STANDARD 20 REQUIRED) 25 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 26 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 27 | 28 | include_directories(${CMAKE_SOURCE_DIR}) 29 | include_directories(${CMAKE_BINARY_DIR}/.deps/install/include) 30 | 31 | include(valkey_search) 32 | 33 | add_subdirectory(vmsdk) 34 | add_subdirectory(third_party) 35 | add_subdirectory(src) 36 | 37 | if(BUILD_TESTS) 38 | message(STATUS "Building tests") 39 | add_subdirectory(testing) 40 | endif() 41 | 42 | # Create a symbolic link to the root directory for compile_commands.json 43 | execute_process( 44 | COMMAND 45 | ${CMAKE_COMMAND} -E create_symlink 46 | ${CMAKE_BINARY_DIR}/compile_commands.json 47 | ${CMAKE_SOURCE_DIR}/compile_commands.json) 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024-present, valkey-search contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /QUICK_START.md: -------------------------------------------------------------------------------- 1 | # Quick Start 2 | 3 | Follow these steps to set up, build, and run the Valkey server with vector search capabilities. This guide will walk you through creating a vector index, inserting vectors, and issuing queries. 4 | 5 | ## Step 1: Install Valkey and valkey-search 6 | 7 | 1. Build Valkey from source by following the instructions [here](https://github.com/valkey-io/valkey?tab=readme-ov-file#building-valkey-using-makefile). Make sure to use Valkey version 7.2.6 or later as the previous versions have Valkey module API bugs. 8 | 2. Build valkey-search module from source by following the instructions [here](https://github.com/valkey-io/valkey-search/tree/main?tab=readme-ov-file#build-instructions). 9 | 10 | ## Step 2: Run the Valkey Server 11 | 12 | Once valkey-search is built, run the Valkey server with the valkey-search module loaded: 13 | 14 | ```bash 15 | ./valkey-server "--loadmodule libsearch.so --reader-threads 64 --writer-threads 64" 16 | ``` 17 | 18 | You should see the Valkey server start, and it will be ready to accept commands. 19 | 20 | ## Step 3: Create a Vector Index 21 | 22 | To enable vector search functionality, you need to create an index for storing vector data. 23 | Start a Valkey CLI session: 24 | 25 | ```bash 26 | valkey-cli 27 | ``` 28 | 29 | Create a vector field using the FT.CREATE command. For example: 30 | 31 | ```bash 32 | FT.CREATE myIndex SCHEMA vector VECTOR HNSW 6 TYPE FLOAT32 DIM 3 DISTANCE_METRIC COSINE 33 | ``` 34 | 35 | - `vector` is the vector field for storing the vectors. 36 | - `VECTOR HNSW` specifies the use of the HNSW (Hierarchical Navigable Small World) algorithm for vector search. 37 | - `DIM 3` sets the vector dimensionality to 3. 38 | - `DISTANCE_METRIC COSINE` sets the distance metric to cosine similarity. 39 | 40 | ## Step 4: Insert Some Vectors 41 | 42 | To insert vectors, use the `HSET` command: 43 | 44 | ```bash 45 | HSET my_hash_key_1 vector "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80?" 46 | HSET my_hash_key_2 vector "\x00\xaa\x00\x00\x00\x00\x00\x00\x00\x00\x80?" 47 | ``` 48 | 49 | Replace the `\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80?` and `\x00\xaa\x00\x00\x00\x00\x00\x00\x00\x00\x80?` with actual vectors. 50 | 51 | ## Step 5: Issue Vector Queries 52 | 53 | Now that you've created an index and inserted vectors, you can perform a vector search. Use the `FT.SEARCH` command to find similar vectors: 54 | 55 | ```bash 56 | FT.SEARCH myIndex "*=>[KNN 5 @vector $query_vector]" PARAMS 2 query_vector "\xcd\xccL?\x00\x00\x00\x00\x00\x00\x00\x00" 57 | ``` 58 | 59 | This command performs a K-nearest neighbors search and returns the top 5 closest vectors to the provided query vector. 60 | -------------------------------------------------------------------------------- /ci/asan.supp: -------------------------------------------------------------------------------- 1 | leak:luaM_realloc_ 2 | -------------------------------------------------------------------------------- /ci/build_ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | CI_DIR=$(readlink -f $(dirname $0)) 4 | ROOT_DIR=$(readlink -f ${CI_DIR}/..) 5 | BUILD_SH_ARGS=$@ 6 | WGET="wget -q --show-progress" 7 | HOSTADDR="https://github.com/valkey-io/valkey-search/releases/download/1.0.0-rc1" 8 | 9 | # Constants 10 | RESET='\e[0m' 11 | GREEN='\e[32;1m' 12 | RED='\e[31;1m' 13 | 14 | ## Search for --asan 15 | while [ $# -gt 0 ] 16 | do 17 | arg=$1 18 | case $arg in 19 | --asan) 20 | ASAN_BUILD="yes" 21 | shift || true 22 | ;; 23 | *) 24 | shift || true 25 | ;; 26 | esac 27 | done 28 | 29 | asan_suffix="" 30 | if [[ "${ASAN_BUILD}" == "yes" ]]; then 31 | asan_suffix="-asan" 32 | echo "Building with ASAN enabled" 33 | fi 34 | 35 | function LOG_INFO() { 36 | printf "${GREEN}INFO ${RESET} $1\n" 37 | } 38 | 39 | function LOG_ERROR() { 40 | printf "${RED}ERROR${RESET} $1\n" 41 | } 42 | 43 | function get_deb_suffix() { 44 | local ARCH="" 45 | local uname_m=$(uname -m) 46 | if [[ "${uname_m}" == "x86_64" ]]; then 47 | ARCH="amd64" 48 | else 49 | ARCH="arm64" 50 | fi 51 | 52 | if [ ! -f /usr/bin/lsb_release ]; then 53 | DISTRO="linux" 54 | else 55 | CODENAME=$(lsb_release -c|cut -d":" -f2) 56 | DISTRO_ID=$(lsb_release -i|cut -d":" -f2) 57 | CODENAME=${CODENAME#[$'\r\t\n ']} 58 | DISTRO_ID=${DISTRO_ID#[$'\r\t\n ']} 59 | DISTRO=${DISTRO_ID}-${CODENAME} 60 | DISTRO=$(echo "$DISTRO" | tr '[:upper:]' '[:lower:]') 61 | fi 62 | echo valkey-search-deps-${DISTRO}${asan_suffix}-${ARCH}.deb 63 | } 64 | 65 | function download_deb() { 66 | local deb_package=$1 67 | LOG_INFO "Downloading ${HOSTADDR}/${deb_package}" 68 | ${WGET} ${HOSTADDR}/${deb_package} -O ${ROOT_DIR}/debs/${deb_package} 69 | } 70 | 71 | # Prepare the environment before getting started 72 | function prepare_env() { 73 | local deb_package=$(get_deb_suffix) 74 | if [ ! -d /opt/valkey-search-deps${asan_suffix}/ ]; then 75 | # Fetch the deb from github 76 | download_deb ${deb_package} 77 | LOG_INFO "Installing ${ROOT_DIR}/debs/${deb_package}" 78 | sudo dpkg -i ${ROOT_DIR}/debs/${deb_package} 79 | else 80 | LOG_INFO "Debian file: '${deb_package}' is already installed" 81 | fi 82 | } 83 | 84 | function cleanup() { 85 | # This method is called just before the script exits 86 | local exit_code=$? 87 | LOG_INFO "Cleaning up before exit" 88 | 89 | if [[ $exit_code -ne 0 ]]; then 90 | LOG_ERROR "Script ended with error code ${exit_code}" 91 | else 92 | LOG_INFO "Script completed successfully" 93 | fi 94 | } 95 | 96 | function build_and_run_tests() { 97 | local DEPS_DIR=/opt/valkey-search-deps${asan_suffix} 98 | local CMAKE_DIR=${DEPS_DIR}/lib/cmake 99 | # Let CMake find -config.cmake files by updating the CMAKE_PREFIX_PATH variable 100 | export CMAKE_PREFIX_PATH=${CMAKE_DIR}/protobuf:${CMAKE_DIR}/absl:${CMAKE_DIR}/grpc:${CMAKE_DIR}/GTest:${CMAKE_DIR}/utf8_range:${DEPS_DIR} 101 | (cd ${ROOT_DIR} && ./build.sh --use-system-modules --test-errors-stdout ${BUILD_SH_ARGS}) 102 | } 103 | 104 | # Write a success or error message on exit 105 | trap cleanup EXIT 106 | 107 | cd ${CI_DIR} 108 | 109 | prepare_env 110 | build_and_run_tests 111 | -------------------------------------------------------------------------------- /ci/check_changes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set +e 3 | 4 | git_args="" 5 | for arg in "$@"; do 6 | git_args+=" \"$arg\"" 7 | done 8 | # Get the list of modified or new files 9 | files=$(eval "git diff --name-only --diff-filter=AM $git_args"| grep -E '\.cc$|\.h$' || echo "") 10 | 11 | # Check if there are any files to process 12 | if [ -z "$files" ]; then 13 | exit 0 14 | fi 15 | 16 | execute_command() { 17 | local command="$1" # The command to run 18 | local output # Variable to store the command output 19 | 20 | # Run the command and capture the output 21 | output=$(eval "$command" 2>&1) 22 | 23 | # Check if the command was successful 24 | if [ $? -eq 0 ]; then 25 | # Print OK in green if the command succeeded 26 | echo -e "\033[0;32mOK\033[0m" 27 | else 28 | # Print Error in red and the command output if it failed 29 | echo -e "\033[0;31mError\033[0m" 30 | echo "$output" 31 | fi 32 | } 33 | 34 | for file in $files; do 35 | echo "Checking $file" 36 | echo -n "clang-tidy: " 37 | execute_command "clang-tidy --quiet -p compile_commands.json $file 2>&1 | tail -n +3" 38 | echo -n "clang-format: " 39 | execute_command "ci/check_clang_format.sh $file" 40 | done 41 | 42 | -------------------------------------------------------------------------------- /ci/check_clang_format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | if [ "$#" -ne 1 ]; then 4 | echo "Usage: $0 " >&2 5 | exit 1 6 | fi 7 | 8 | FILE=$1 9 | 10 | if [ ! -f "$FILE" ]; then 11 | echo "Error: File '$FILE' does not exist." >&2 12 | exit 1 13 | fi 14 | 15 | # Create a temporary file for formatted output 16 | TEMP_FILE=$(mktemp) 17 | 18 | # Run clang-format and save output to the temp file 19 | clang-format "$FILE" > "$TEMP_FILE" 20 | 21 | # Perform a diff between the original file and the formatted file 22 | if diff -u "$FILE" "$TEMP_FILE" > /dev/null; then 23 | rm -f "$TEMP_FILE" 24 | exit 0 25 | else 26 | echo "Formatting issues detected in $FILE:" >&2 27 | diff -U 0 "$FILE" "$TEMP_FILE" | tail -n +3 >&2 28 | rm -f "$TEMP_FILE" 29 | exit 1 30 | fi 31 | -------------------------------------------------------------------------------- /ci/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # git shortcut: `git pushb` -> `git push origin ` 5 | git config --global alias.pushb '!git push origin $(git rev-parse --abbrev-ref HEAD)' 6 | 7 | export MOUNTED_DIR=$(pwd) 8 | echo "export MOUNTED_DIR=$MOUNTED_DIR" >> /home/$USER/.bashrc 9 | if [ "$ENABLE_COMP_DB_REFRESH" = "true" ]; then 10 | echo "$(pwd)/ci/refresh_comp_db.sh &> /dev/null &" >> /home/$USER/.bashrc 11 | fi 12 | ./build.sh 13 | .devcontainer/setup.sh &> /dev/null 14 | -------------------------------------------------------------------------------- /ci/refresh_comp_db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Locate the top level folder 4 | ROOT_DIR=$(readlink -f $(dirname $(readlink -f $0))/..) 5 | 6 | # Regenerating compile_commands.json is done by running `cmake` (build is not required) 7 | cd ${ROOT_DIR} 8 | ${ROOT_DIR}/ci/build_ubuntu.sh --no-build --configure 9 | -------------------------------------------------------------------------------- /debs/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valkey-io/valkey-search/cd26f4d1dae82d7b110f9ea0b1aff7208695276c/debs/.placeholder -------------------------------------------------------------------------------- /rfc/TEMPLATE.md: -------------------------------------------------------------------------------- 1 | --- 2 | RFC: (PR number) 3 | Status: (Change to Proposed when it's ready for review) 4 | --- 5 | 6 | # Title (Required) 7 | 8 | ## Abstract (Required) 9 | 10 | A few sentences describing the feature. 11 | 12 | ## Motivation (Required) 13 | 14 | What the feature solves and why the existing functionality is not enough. 15 | 16 | ## Design considerations (Required) 17 | 18 | A description of the design constraints and requirements for the proposal, along with comparisons to similar features in other projects. 19 | 20 | ## Specification (Required) 21 | 22 | A more detailed description of the feature, including the reasoning behind the design choices. 23 | 24 | ### Commands (Optional) 25 | 26 | If any new commands are introduced: 27 | 28 | 1. Command name 29 | - **Request** 30 | - **Response** 31 | 32 | ### Authentication and Authorization (Optional) 33 | 34 | If there are any changes around introducing new ACL command/categories for user access control. 35 | 36 | ### Append-only file (Optional) 37 | 38 | If there are any changes around the persistence mechanism of every write operation. 39 | 40 | ### RDB (Optional) 41 | 42 | If there are any changes in snapshotting mechanisms like new data type, version, etc. 43 | 44 | ### Configuration (Optional) 45 | 46 | If there are any configuration changes introduced to enable/disable/modify the behavior of the feature. 47 | 48 | ### Keyspace notifications (Optional) 49 | 50 | If there are any events to be introduced or modified to observe activity around the dataset. 51 | 52 | ### Cluster mode (Optional) 53 | 54 | If there is any special handling for this feature (e.g., client redirection, Sharded PubSub, etc) in cluster mode or if there are any new cluster bus extensions or messages introduced, list out the changes. 55 | 56 | ### Module API (Optional) 57 | 58 | If any new module APIs are needed to implement or support this feature. 59 | 60 | ### Replication (Optional) 61 | 62 | If there are any changes required in the replication mechanism between a primary and replica. 63 | 64 | ### Networking (Optional) 65 | 66 | If there are any changes introduced in the RESP protocol (RESP), client behavior, new server-client interaction mechanism (TCP, RDMA), etc. 67 | 68 | ### Dependencies (Optional) 69 | 70 | If there are any new dependency libraries required to support the feature. Existing dependencies are jemalloc, lua, etc. If the library needs to be vendored into the project, please add supporting reason for it. 71 | 72 | ### Benchmarking (Optional) 73 | 74 | If there are any benchmarks performed and preliminary results (add the hardware/software setup) are available to share or a set of scenarios identified to measure the feature's performance. 75 | 76 | ### Testing (Optional) 77 | 78 | If there are any test scenarios planned to ensure the feature's stability and validate its behavior. 79 | 80 | ### Observability (Optional) 81 | 82 | If there are any new metrics/stats to be introduced to observe behavior or measure the performance of the feature. 83 | 84 | ### Debug mechanism (Optional) 85 | 86 | If there is any debug mechanism introduced to support admin/operators for maintaining the feature. 87 | 88 | ## Appendix (Optional) 89 | 90 | Links to related material such as issues, pull requests, papers, or other references. 91 | -------------------------------------------------------------------------------- /src/acl.h: -------------------------------------------------------------------------------- 1 | #ifndef VALKEYSEARCH_SRC_COMMANDS_ACL_H_ 2 | #define VALKEYSEARCH_SRC_COMMANDS_ACL_H_ 3 | 4 | #include "absl/container/flat_hash_set.h" 5 | #include "absl/status/status.h" 6 | #include "src/index_schema.pb.h" 7 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 8 | 9 | namespace valkey_search { 10 | namespace acl { 11 | 12 | struct ValkeyAclGetUserReplyView { 13 | absl::string_view cmds; 14 | absl::string_view keys; 15 | }; 16 | 17 | /* Expose this function for tests */ 18 | bool StringEndsWithWildCardMatch(const char *pattern, int pattern_len, 19 | const char *string, int string_len); 20 | } // namespace acl 21 | 22 | /* 23 | Check if 24 | * the user behind the connection in the context `ctx`, 25 | * can run a command with command categories defined in 26 | `module_allowed_cmds`, which are composed of commands and commands categories, 27 | * against ALL POSSIBLE keys with key prefixes, defined as 28 | `module_prefixes`, 29 | * according to the ACL rules defined in the server. 30 | */ 31 | absl::Status AclPrefixCheck( 32 | RedisModuleCtx *ctx, 33 | const absl::flat_hash_set &module_allowed_cmds, 34 | const std::vector &module_prefixes); 35 | 36 | absl::Status AclPrefixCheck( 37 | RedisModuleCtx *ctx, 38 | const absl::flat_hash_set &module_allowed_cmds, 39 | const data_model::IndexSchema &index_schema_proto); 40 | 41 | } // namespace valkey_search 42 | #endif // VALKEYSEARCH_SRC_COMMANDS_ACL_H_ -------------------------------------------------------------------------------- /src/attribute.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, ValkeySearch contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_ATTRIBUTE_H_ 31 | #define VALKEYSEARCH_SRC_ATTRIBUTE_H_ 32 | 33 | #include 34 | #include 35 | 36 | #include "absl/strings/str_cat.h" 37 | #include "absl/strings/string_view.h" 38 | #include "src/index_schema.pb.h" 39 | #include "src/indexes/index_base.h" 40 | #include "vmsdk/src/managed_pointers.h" 41 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 42 | 43 | namespace valkey_search { 44 | 45 | class Attribute { 46 | public: 47 | Attribute(absl::string_view alias, absl::string_view identifier, 48 | std::shared_ptr index) 49 | : alias_(alias), identifier_(identifier), index_(index) {} 50 | inline const std::string& GetAlias() const { return alias_; } 51 | inline const std::string& GetIdentifier() const { return identifier_; } 52 | std::shared_ptr GetIndex() const { return index_; } 53 | std::unique_ptr ToProto() const { 54 | auto attribute_proto = std::make_unique(); 55 | attribute_proto->set_alias(alias_); 56 | attribute_proto->set_identifier(identifier_); 57 | attribute_proto->set_allocated_index(index_->ToProto().release()); 58 | return attribute_proto; 59 | } 60 | inline int RespondWithInfo(RedisModuleCtx* ctx) const { 61 | RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_LEN); 62 | RedisModule_ReplyWithSimpleString(ctx, "identifier"); 63 | RedisModule_ReplyWithSimpleString(ctx, GetIdentifier().c_str()); 64 | RedisModule_ReplyWithSimpleString(ctx, "attribute"); 65 | RedisModule_ReplyWithSimpleString(ctx, GetAlias().c_str()); 66 | int added_fields = index_->RespondWithInfo(ctx); 67 | RedisModule_ReplySetArrayLength(ctx, added_fields + 4); 68 | return 1; 69 | } 70 | 71 | inline vmsdk::UniqueRedisString DefaultReplyScoreAs() const { 72 | if (!cached_score_as_) { 73 | cached_score_as_ = 74 | vmsdk::MakeUniqueRedisString(absl::StrCat("__", alias_, "_score")); 75 | } 76 | return vmsdk::RetainUniqueRedisString(cached_score_as_.get()); 77 | } 78 | 79 | private: 80 | std::string alias_; 81 | std::string identifier_; 82 | std::shared_ptr index_; 83 | // Maintaining a cached version 84 | mutable vmsdk::UniqueRedisString cached_score_as_; 85 | }; 86 | 87 | } // namespace valkey_search 88 | 89 | #endif // VALKEYSEARCH_SRC_ATTRIBUTE_H_ 90 | -------------------------------------------------------------------------------- /src/commands/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_COMMANDS 2 | ${CMAKE_CURRENT_LIST_DIR}/ft_create.cc 3 | ${CMAKE_CURRENT_LIST_DIR}/ft_dropindex.cc 4 | ${CMAKE_CURRENT_LIST_DIR}/ft_info.cc ${CMAKE_CURRENT_LIST_DIR}/ft_list.cc 5 | ${CMAKE_CURRENT_LIST_DIR}/ft_search.cc ${CMAKE_CURRENT_LIST_DIR}/commands.h 6 | ${CMAKE_CURRENT_LIST_DIR}/ft_search.h) 7 | 8 | valkey_search_add_static_library(commands "${SRCS_COMMANDS}") 9 | target_include_directories(commands PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 10 | target_link_libraries(commands PUBLIC ft_create_parser) 11 | target_link_libraries(commands PUBLIC ft_search_parser) 12 | target_link_libraries(commands PUBLIC index_schema) 13 | target_link_libraries(commands PUBLIC index_schema_cc_proto) 14 | target_link_libraries(commands PUBLIC metrics) 15 | target_link_libraries(commands PUBLIC schema_manager) 16 | target_link_libraries(commands PUBLIC valkey_search) 17 | target_link_libraries(commands PUBLIC vector_base) 18 | target_link_libraries(commands PUBLIC fanout) 19 | target_link_libraries(commands PUBLIC response_generator) 20 | target_link_libraries(commands PUBLIC search) 21 | target_link_libraries(commands PUBLIC command_parser) 22 | target_link_libraries(commands PUBLIC managed_pointers) 23 | target_link_libraries(commands PUBLIC blocked_client) 24 | target_link_libraries(commands PUBLIC type_conversions) 25 | target_link_libraries(commands PUBLIC utils) 26 | target_link_libraries(commands PUBLIC status_macros) 27 | target_link_libraries(commands PUBLIC valkey_module) 28 | 29 | set(SRCS_FT_SEARCH_PARSER ${CMAKE_CURRENT_LIST_DIR}/ft_search_parser.cc 30 | ${CMAKE_CURRENT_LIST_DIR}/ft_search_parser.h) 31 | 32 | valkey_search_add_static_library(ft_search_parser "${SRCS_FT_SEARCH_PARSER}") 33 | target_include_directories(ft_search_parser PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 34 | target_link_libraries(ft_search_parser PUBLIC filter_parser) 35 | target_link_libraries(ft_search_parser PUBLIC index_schema) 36 | target_link_libraries(ft_search_parser PUBLIC metrics) 37 | target_link_libraries(ft_search_parser PUBLIC schema_manager) 38 | target_link_libraries(ft_search_parser PUBLIC index_base) 39 | target_link_libraries(ft_search_parser PUBLIC search) 40 | target_link_libraries(ft_search_parser PUBLIC command_parser) 41 | target_link_libraries(ft_search_parser PUBLIC managed_pointers) 42 | target_link_libraries(ft_search_parser PUBLIC type_conversions) 43 | target_link_libraries(ft_search_parser PUBLIC status_macros) 44 | target_link_libraries(ft_search_parser PUBLIC valkey_module) 45 | 46 | set(SRCS_FT_CREATE_PARSER ${CMAKE_CURRENT_LIST_DIR}/ft_create_parser.cc 47 | ${CMAKE_CURRENT_LIST_DIR}/ft_create_parser.h) 48 | 49 | valkey_search_add_static_library(ft_create_parser "${SRCS_FT_CREATE_PARSER}") 50 | target_include_directories(ft_create_parser PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 51 | target_link_libraries(ft_create_parser PUBLIC attribute_data_type) 52 | target_link_libraries(ft_create_parser PUBLIC index_schema_cc_proto) 53 | target_link_libraries(ft_create_parser PUBLIC index_base) 54 | target_link_libraries(ft_create_parser PUBLIC vector_base) 55 | target_link_libraries(ft_create_parser PUBLIC command_parser) 56 | target_link_libraries(ft_create_parser PUBLIC type_conversions) 57 | target_link_libraries(ft_create_parser PUBLIC status_macros) 58 | target_link_libraries(ft_create_parser PUBLIC valkey_module) 59 | 60 | set(SRCS_FILTER_PARSER ${CMAKE_CURRENT_LIST_DIR}/filter_parser.cc 61 | ${CMAKE_CURRENT_LIST_DIR}/filter_parser.h) 62 | 63 | valkey_search_add_static_library(filter_parser "${SRCS_FILTER_PARSER}") 64 | target_include_directories(filter_parser PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 65 | target_link_libraries(filter_parser PUBLIC index_schema) 66 | target_link_libraries(filter_parser PUBLIC index_base) 67 | target_link_libraries(filter_parser PUBLIC numeric) 68 | target_link_libraries(filter_parser PUBLIC tag) 69 | target_link_libraries(filter_parser PUBLIC predicate) 70 | target_link_libraries(filter_parser PUBLIC status_macros) 71 | -------------------------------------------------------------------------------- /src/commands/filter_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COMMANDS_FILTER_PARSER_H_ 31 | #define VALKEYSEARCH_SRC_COMMANDS_FILTER_PARSER_H_ 32 | #include 33 | #include 34 | #include 35 | 36 | #include "absl/container/flat_hash_set.h" 37 | #include "absl/status/statusor.h" 38 | #include "absl/strings/string_view.h" 39 | #include "src/index_schema.h" 40 | #include "src/indexes/tag.h" 41 | #include "src/query/predicate.h" 42 | 43 | namespace valkey_search { 44 | namespace indexes { 45 | class Tag; 46 | } // namespace indexes 47 | struct FilterParseResults { 48 | std::unique_ptr root_predicate; 49 | absl::flat_hash_set filter_identifiers; 50 | }; 51 | class FilterParser { 52 | public: 53 | FilterParser(const IndexSchema& index_schema, absl::string_view expression); 54 | 55 | absl::StatusOr Parse(); 56 | 57 | private: 58 | const IndexSchema& index_schema_; 59 | absl::string_view expression_; 60 | size_t pos_{0}; 61 | absl::flat_hash_set filter_identifiers_; 62 | 63 | absl::StatusOr IsMatchAllExpression(); 64 | absl::StatusOr> ParseExpression(); 65 | absl::StatusOr> 66 | ParseNumericPredicate(const std::string& attribute_alias); 67 | absl::StatusOr> ParseTagPredicate( 68 | const std::string& attribute_alias); 69 | void SkipWhitespace(); 70 | 71 | char Peek() const { return expression_[pos_]; } 72 | 73 | bool IsEnd() const { return pos_ >= expression_.length(); } 74 | bool Match(char expected, bool skip_whitespace = true); 75 | bool MatchInsensitive(const std::string& expected); 76 | absl::StatusOr ParseFieldName(); 77 | 78 | absl::StatusOr ParseNumber(); 79 | 80 | absl::StatusOr ParseTagString(); 81 | 82 | absl::StatusOr> ParseTags( 83 | absl::string_view tag_string, indexes::Tag* tag_index) const; 84 | }; 85 | 86 | } // namespace valkey_search 87 | #endif // VALKEYSEARCH_SRC_COMMANDS_FILTER_PARSER_H_ 88 | -------------------------------------------------------------------------------- /src/commands/ft._list.json: -------------------------------------------------------------------------------- 1 | { 2 | "FT._LIST": { 3 | "acl_categories": [ 4 | "ADMIN", 5 | "SLOW", 6 | "READ", 7 | "SEARCH" 8 | ], 9 | "arguments": [], 10 | "arity": 1, 11 | "complexity": "O(1)", 12 | "group": "search", 13 | "module_since": "1.0.0", 14 | "summary": "Lists the currently defined indexes" 15 | } 16 | } -------------------------------------------------------------------------------- /src/commands/ft.create.json: -------------------------------------------------------------------------------- 1 | { 2 | "FT.CREATE": { 3 | "acl_categories": [ 4 | "FAST", 5 | "WRITE", 6 | "SEARCH" 7 | ], 8 | "arguments": [], 9 | "complexity": "Construction time O(N log N), where N is the number of indexed items", 10 | "group": "search", 11 | "module_since": "1.0.0", 12 | "summary": "Creates an empty search index and initiates the backfill process" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/commands/ft.dropindex.json: -------------------------------------------------------------------------------- 1 | { 2 | "FT.DROPINDEX": { 3 | "acl_categories": [ 4 | "FAST", 5 | "WRITE", 6 | "SEARCH" 7 | ], 8 | "arguments": [ 9 | { 10 | "key_spec_index": 0, 11 | "name": "key", 12 | "type": "key" 13 | } 14 | ], 15 | "arity": 2, 16 | "complexity": "O(N)", 17 | "group": "search", 18 | "module_since": "1.0.0", 19 | "summary": "Drop the index created by FT.CREATE command. It is an error if the index doesn't exist" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/commands/ft.info.json: -------------------------------------------------------------------------------- 1 | { 2 | "FT.INFO": { 3 | "acl_categories": [ 4 | "READ", 5 | "FAST", 6 | "SEARCH" 7 | ], 8 | "arguments": [ 9 | { 10 | "key_spec_index": 0, 11 | "name": "key", 12 | "type": "key" 13 | } 14 | ], 15 | "arity": 2, 16 | "complexity": "O(1)", 17 | "group": "search", 18 | "module_since": "1.0.0", 19 | "summary": "Detailed information about the specified index is returned" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/commands/ft.search.json: -------------------------------------------------------------------------------- 1 | { 2 | "FT.SEARCH": { 3 | "acl_categories": [ 4 | "READ", 5 | "SLOW", 6 | "SEARCH" 7 | ], 8 | "arguments": [ 9 | { 10 | "key_spec_index": 0, 11 | "name": "index", 12 | "type": "key" 13 | }, 14 | { 15 | "key_spec_index": 1, 16 | "name": "query", 17 | "type": "string" 18 | } 19 | ], 20 | "arity": -3, 21 | "complexity": "O(log N)", 22 | "group": "search", 23 | "module_since": "1.0.0", 24 | "summary": "Performs a search of the specified index. The keys which match the query expression are returned" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/commands/ft_create.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "absl/status/status.h" 31 | #include "src/acl.h" 32 | #include "src/commands/commands.h" 33 | #include "src/commands/ft_create_parser.h" 34 | #include "src/schema_manager.h" 35 | #include "vmsdk/src/status/status_macros.h" 36 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 37 | 38 | namespace valkey_search { 39 | 40 | absl::Status FTCreateCmd(RedisModuleCtx *ctx, RedisModuleString **argv, 41 | int argc) { 42 | VMSDK_ASSIGN_OR_RETURN(auto index_schema_proto, 43 | ParseFTCreateArgs(ctx, argv + 1, argc - 1)); 44 | index_schema_proto.set_db_num(RedisModule_GetSelectedDb(ctx)); 45 | static const auto permissions = 46 | PrefixACLPermissions(kCreateCmdPermissions, kCreateCommand); 47 | VMSDK_RETURN_IF_ERROR(AclPrefixCheck(ctx, permissions, index_schema_proto)); 48 | VMSDK_RETURN_IF_ERROR( 49 | SchemaManager::Instance().CreateIndexSchema(ctx, index_schema_proto)); 50 | 51 | RedisModule_ReplyWithSimpleString(ctx, "OK"); 52 | RedisModule_ReplicateVerbatim(ctx); 53 | return absl::OkStatus(); 54 | } 55 | } // namespace valkey_search 56 | -------------------------------------------------------------------------------- /src/commands/ft_create_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COMMANDS_FT_CREATE_PARSER_H_ 31 | #define VALKEYSEARCH_SRC_COMMANDS_FT_CREATE_PARSER_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | #include "absl/status/status.h" 39 | #include "absl/status/statusor.h" 40 | #include "absl/strings/string_view.h" 41 | #include "src/index_schema.pb.h" 42 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 43 | 44 | namespace valkey_search { 45 | 46 | struct FTCreateTagParameters { 47 | absl::string_view separator{","}; 48 | bool case_sensitive{false}; 49 | }; 50 | 51 | constexpr int kDefaultInitialCap{10 * 1024}; 52 | 53 | struct FTCreateVectorParameters { 54 | std::optional dimensions; 55 | data_model::DistanceMetric distance_metric{ 56 | data_model::DistanceMetric::DISTANCE_METRIC_UNSPECIFIED}; 57 | data_model::VectorDataType vector_data_type{ 58 | data_model::VectorDataType::VECTOR_DATA_TYPE_UNSPECIFIED}; 59 | int initial_cap{kDefaultInitialCap}; 60 | absl::Status Verify() const; 61 | std::unique_ptr ToProto() const; 62 | }; 63 | 64 | constexpr int kDefaultBlockSize{1024}; 65 | constexpr int kDefaultM{16}; 66 | constexpr int kDefaultEFConstruction{200}; 67 | constexpr int kDefaultEFRuntime{10}; 68 | 69 | struct HNSWParameters : public FTCreateVectorParameters { 70 | // Tightly connected with internal dimensionality of the data 71 | // strongly affects the memory consumption 72 | int m{kDefaultM}; 73 | int ef_construction{kDefaultEFConstruction}; 74 | size_t ef_runtime{kDefaultEFRuntime}; 75 | absl::Status Verify() const; 76 | std::unique_ptr ToProto() const; 77 | }; 78 | 79 | struct FlatParameters : public FTCreateVectorParameters { 80 | // Block size holds the amount of vectors in a contiguous array. This is 81 | // useful when the index is dynamic with respect to addition and deletion. 82 | uint32_t block_size{kDefaultBlockSize}; 83 | std::unique_ptr ToProto() const; 84 | }; 85 | 86 | absl::StatusOr ParseFTCreateArgs( 87 | RedisModuleCtx *ctx, RedisModuleString **argv, int argc); 88 | } // namespace valkey_search 89 | #endif // VALKEYSEARCH_SRC_COMMANDS_FT_CREATE_PARSER_H_ 90 | -------------------------------------------------------------------------------- /src/commands/ft_dropindex.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "absl/status/status.h" 31 | #include "absl/strings/string_view.h" 32 | #include "src/acl.h" 33 | #include "src/commands/commands.h" 34 | #include "src/schema_manager.h" 35 | #include "vmsdk/src/status/status_macros.h" 36 | #include "vmsdk/src/type_conversions.h" 37 | #include "vmsdk/src/utils.h" 38 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 39 | 40 | namespace valkey_search { 41 | 42 | absl::Status FTDropIndexCmd(RedisModuleCtx *ctx, RedisModuleString **argv, 43 | int argc) { 44 | if (argc != 2) { 45 | return absl::InvalidArgumentError(vmsdk::WrongArity(kDropIndexCommand)); 46 | } 47 | auto index_schema_name = vmsdk::ToStringView(argv[1]); 48 | 49 | VMSDK_ASSIGN_OR_RETURN( 50 | auto index_schema, 51 | SchemaManager::Instance().GetIndexSchema(RedisModule_GetSelectedDb(ctx), 52 | index_schema_name)); 53 | static const auto permissions = 54 | PrefixACLPermissions(kDropIndexCmdPermissions, kDropIndexCommand); 55 | VMSDK_RETURN_IF_ERROR( 56 | AclPrefixCheck(ctx, permissions, index_schema->GetKeyPrefixes())); 57 | 58 | VMSDK_RETURN_IF_ERROR(SchemaManager::Instance().RemoveIndexSchema( 59 | RedisModule_GetSelectedDb(ctx), index_schema_name)); 60 | RedisModule_ReplyWithSimpleString(ctx, "OK"); 61 | RedisModule_ReplicateVerbatim(ctx); 62 | return absl::OkStatus(); 63 | } 64 | } // namespace valkey_search 65 | -------------------------------------------------------------------------------- /src/commands/ft_info.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "absl/status/status.h" 31 | #include "absl/strings/string_view.h" 32 | #include "src/acl.h" 33 | #include "src/commands/commands.h" 34 | #include "src/schema_manager.h" 35 | #include "vmsdk/src/command_parser.h" 36 | #include "vmsdk/src/status/status_macros.h" 37 | #include "vmsdk/src/type_conversions.h" 38 | #include "vmsdk/src/utils.h" 39 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 40 | 41 | namespace valkey_search { 42 | 43 | absl::Status FTInfoCmd(RedisModuleCtx *ctx, RedisModuleString **argv, 44 | int argc) { 45 | if (argc < 2) { 46 | return absl::InvalidArgumentError(vmsdk::WrongArity(kInfoCommand)); 47 | } 48 | 49 | vmsdk::ArgsIterator itr{argv, argc}; 50 | itr.Next(); 51 | VMSDK_ASSIGN_OR_RETURN(auto itr_arg, itr.Get()); 52 | auto index_schema_name = vmsdk::ToStringView(itr_arg); 53 | 54 | VMSDK_ASSIGN_OR_RETURN( 55 | auto index_schema, 56 | SchemaManager::Instance().GetIndexSchema(RedisModule_GetSelectedDb(ctx), 57 | index_schema_name)); 58 | static const auto permissions = 59 | PrefixACLPermissions(kInfoCmdPermissions, kInfoCommand); 60 | VMSDK_RETURN_IF_ERROR( 61 | AclPrefixCheck(ctx, permissions, index_schema->GetKeyPrefixes())); 62 | index_schema->RespondWithInfo(ctx); 63 | 64 | return absl::OkStatus(); 65 | } 66 | 67 | } // namespace valkey_search 68 | -------------------------------------------------------------------------------- /src/commands/ft_list.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include 31 | 32 | #include "absl/container/flat_hash_set.h" 33 | #include "absl/status/status.h" 34 | #include "absl/strings/string_view.h" 35 | #include "src/commands/commands.h" 36 | #include "src/schema_manager.h" 37 | #include "vmsdk/src/utils.h" 38 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 39 | 40 | namespace valkey_search { 41 | 42 | absl::Status FTListCmd(RedisModuleCtx *ctx, RedisModuleString **argv, 43 | int argc) { 44 | if (argc > 1) { 45 | return absl::InvalidArgumentError(vmsdk::WrongArity(kListCommand)); 46 | } 47 | absl::flat_hash_set names = 48 | SchemaManager::Instance().GetIndexSchemasInDB( 49 | RedisModule_GetSelectedDb(ctx)); 50 | RedisModule_ReplyWithArray(ctx, names.size()); 51 | for (const auto &name : names) { 52 | RedisModule_ReplyWithSimpleString(ctx, name.c_str()); 53 | } 54 | return absl::OkStatus(); 55 | } 56 | } // namespace valkey_search 57 | -------------------------------------------------------------------------------- /src/commands/ft_search.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_H_ 31 | #define VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_H_ 32 | 33 | #include 34 | #include 35 | 36 | #include "absl/status/statusor.h" 37 | #include "src/index_schema.h" 38 | #include "src/indexes/vector_base.h" 39 | #include "src/query/search.h" 40 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 41 | 42 | namespace valkey_search { 43 | class ValkeySearch; 44 | // Declared here to support testing 45 | void SendReply(RedisModuleCtx *ctx, std::deque &neighbors, 46 | const query::VectorSearchParameters ¶meters); 47 | namespace async { 48 | 49 | struct Result { 50 | absl::StatusOr> neighbors; 51 | std::unique_ptr parameters; 52 | }; 53 | 54 | int Reply(RedisModuleCtx *ctx, [[maybe_unused]] RedisModuleString **argv, 55 | [[maybe_unused]] int argc); 56 | 57 | void Free(RedisModuleCtx * /*ctx*/, void *privdata); 58 | 59 | } // namespace async 60 | 61 | } // namespace valkey_search 62 | #endif // VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_H_ 63 | -------------------------------------------------------------------------------- /src/commands/ft_search_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_PARSER_H_ 31 | #define VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_PARSER_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | #include "absl/status/statusor.h" 38 | #include "src/query/search.h" 39 | #include "src/schema_manager.h" 40 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 41 | 42 | namespace valkey_search { 43 | 44 | constexpr int64_t kTimeoutMS{50000}; 45 | const size_t kMaxTimeoutMs{60000}; 46 | 47 | struct LimitParameter { 48 | uint64_t first_index{0}; 49 | uint64_t number{10}; 50 | }; 51 | 52 | absl::StatusOr> 53 | ParseVectorSearchParameters(RedisModuleCtx *ctx, RedisModuleString **argv, 54 | int argc, const SchemaManager &schema_manager); 55 | 56 | } // namespace valkey_search 57 | #endif // VALKEYSEARCH_SRC_COMMANDS_FT_SEARCH_PARSER_H_ 58 | -------------------------------------------------------------------------------- /src/coordinator/client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COORDINATOR_CLIENT_H_ 31 | #define VALKEYSEARCH_SRC_COORDINATOR_CLIENT_H_ 32 | 33 | #include 34 | #include 35 | 36 | #include "absl/functional/any_invocable.h" 37 | #include "absl/strings/string_view.h" 38 | #include "grpcpp/support/status.h" 39 | #include "src/coordinator/coordinator.grpc.pb.h" 40 | #include "src/coordinator/coordinator.pb.h" 41 | #include "vmsdk/src/managed_pointers.h" 42 | 43 | namespace valkey_search::coordinator { 44 | 45 | using GetGlobalMetadataCallback = 46 | absl::AnyInvocable; 47 | using SearchIndexPartitionCallback = 48 | absl::AnyInvocable; 49 | 50 | class Client { 51 | public: 52 | virtual ~Client() = default; 53 | virtual void GetGlobalMetadata(GetGlobalMetadataCallback done) = 0; 54 | virtual void SearchIndexPartition( 55 | std::unique_ptr request, 56 | SearchIndexPartitionCallback done) = 0; 57 | }; 58 | 59 | class ClientImpl : public Client { 60 | public: 61 | ClientImpl(vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx, 62 | absl::string_view address, 63 | std::unique_ptr stub); 64 | static std::shared_ptr MakeInsecureClient( 65 | vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx, 66 | absl::string_view address); 67 | 68 | ClientImpl(const ClientImpl&) = delete; 69 | ClientImpl& operator=(const ClientImpl&) = delete; 70 | 71 | void GetGlobalMetadata(GetGlobalMetadataCallback done) override; 72 | void SearchIndexPartition( 73 | std::unique_ptr request, 74 | SearchIndexPartitionCallback done) override; 75 | 76 | private: 77 | vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx_; 78 | std::string address_; 79 | std::unique_ptr stub_; 80 | }; 81 | 82 | } // namespace valkey_search::coordinator 83 | 84 | #endif // VALKEYSEARCH_SRC_COORDINATOR_CLIENT_H_ 85 | -------------------------------------------------------------------------------- /src/coordinator/client_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COORDINATOR_CLIENT_POOL_H_ 31 | #define VALKEYSEARCH_SRC_COORDINATOR_CLIENT_POOL_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | #include "absl/base/thread_annotations.h" 38 | #include "absl/container/flat_hash_map.h" 39 | #include "absl/strings/string_view.h" 40 | #include "absl/synchronization/mutex.h" 41 | #include "src/coordinator/client.h" 42 | #include "vmsdk/src/managed_pointers.h" 43 | 44 | namespace valkey_search::coordinator { 45 | 46 | class ClientPool { 47 | public: 48 | ClientPool(vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx) 49 | : detached_ctx_(std::move(detached_ctx)) {} 50 | virtual ~ClientPool() = default; 51 | 52 | virtual std::shared_ptr GetClient(absl::string_view address) { 53 | auto mutex = absl::MutexLock(&client_pool_mutex_); 54 | auto itr = client_pool_.find(address); 55 | if (itr == client_pool_.end()) { 56 | auto client = ClientImpl::MakeInsecureClient( 57 | vmsdk::MakeUniqueRedisDetachedThreadSafeContext(detached_ctx_.get()), 58 | address); 59 | client_pool_[address] = std::move(client); 60 | } 61 | return client_pool_[address]; 62 | } 63 | 64 | private: 65 | vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx_; 66 | absl::Mutex client_pool_mutex_; 67 | absl::flat_hash_map> client_pool_ 68 | ABSL_GUARDED_BY(client_pool_mutex_); 69 | }; 70 | 71 | } // namespace valkey_search::coordinator 72 | 73 | #endif // VALKEYSEARCH_SRC_COORDINATOR_CLIENT_POOL_H_ 74 | -------------------------------------------------------------------------------- /src/coordinator/coordinator.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package valkey_search.coordinator; 18 | 19 | import "google/protobuf/any.proto"; 20 | 21 | service Coordinator { 22 | // Get the current schema metadata. 23 | rpc GetGlobalMetadata(GetGlobalMetadataRequest) 24 | returns (GetGlobalMetadataResponse) { 25 | } 26 | // Search the index partition for documents. 27 | rpc SearchIndexPartition(SearchIndexPartitionRequest) 28 | returns (SearchIndexPartitionResponse) { 29 | } 30 | } 31 | 32 | message LimitParameter { 33 | uint64 first_index = 1; 34 | uint64 number = 2; 35 | } 36 | 37 | message TagPredicate { 38 | string attribute_alias = 1; 39 | string raw_tag_string = 2; 40 | } 41 | 42 | message NumericPredicate { 43 | string attribute_alias = 1; 44 | double start = 2; 45 | bool is_inclusive_start = 3; 46 | double end = 4; 47 | bool is_inclusive_end = 5; 48 | } 49 | 50 | message AndPredicate { 51 | Predicate lhs = 1; 52 | Predicate rhs = 2; 53 | } 54 | 55 | message OrPredicate { 56 | Predicate lhs = 1; 57 | Predicate rhs = 2; 58 | } 59 | 60 | message NegatePredicate { 61 | Predicate predicate = 1; 62 | } 63 | 64 | message Predicate { 65 | oneof predicate { 66 | TagPredicate tag = 1; 67 | NumericPredicate numeric = 2; 68 | AndPredicate and = 3; 69 | OrPredicate or = 4; 70 | NegatePredicate negate = 5; 71 | } 72 | } 73 | 74 | message ReturnParameter { 75 | string identifier = 1; 76 | string alias = 2; 77 | } 78 | 79 | message SearchIndexPartitionRequest { 80 | string index_schema_name = 2; 81 | string attribute_alias = 3; 82 | optional string score_as = 4; 83 | bytes query = 5; 84 | uint32 dialect = 6; 85 | uint32 k = 7; 86 | uint64 ef = 8; 87 | LimitParameter limit = 9; 88 | uint64 timeout_ms = 10; 89 | bool no_content = 11; 90 | optional Predicate root_filter_predicate = 12; 91 | repeated ReturnParameter return_parameters = 13; 92 | } 93 | 94 | message NeighborEntry { 95 | string key = 1; 96 | float score = 2; 97 | repeated AttributeContentEntry attribute_contents = 3; 98 | } 99 | 100 | message SearchIndexPartitionResponse { 101 | repeated NeighborEntry neighbors = 1; 102 | } 103 | 104 | message AttributeContentEntry { 105 | string identifier = 1; 106 | bytes content = 2; 107 | } 108 | 109 | message GlobalMetadataVersionHeader { 110 | uint64 top_level_fingerprint = 1; 111 | uint32 top_level_version = 2; 112 | } 113 | 114 | message GlobalMetadata { 115 | GlobalMetadataVersionHeader version_header = 1; 116 | // Nested map. Outer map is namespace, inner map is ID to entry. 117 | map type_namespace_map = 2; 118 | } 119 | 120 | // GlobalMetadataEntryMap is a wrapper around a map from ID to metadata entry. 121 | message GlobalMetadataEntryMap { 122 | map entries = 1; 123 | } 124 | 125 | message GlobalMetadataEntry { 126 | uint32 version = 1; 127 | uint64 fingerprint = 2; 128 | uint32 encoding_version = 3; 129 | optional google.protobuf.Any content = 4; 130 | } 131 | 132 | message GetGlobalMetadataRequest {} 133 | message GetGlobalMetadataResponse { 134 | GlobalMetadata metadata = 1; 135 | } 136 | -------------------------------------------------------------------------------- /src/coordinator/grpc_suspender.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "src/coordinator/grpc_suspender.h" 31 | 32 | #include "absl/log/check.h" 33 | #include "absl/status/status.h" 34 | #include "absl/synchronization/mutex.h" 35 | 36 | namespace valkey_search::coordinator { 37 | 38 | absl::Status GRPCSuspender::Suspend() { 39 | absl::MutexLock lock(&mutex_); 40 | if (suspended_) { 41 | return absl::FailedPreconditionError("Already suspended"); 42 | } 43 | suspended_ = true; 44 | while (count_ > 0) { 45 | in_flight_tasks_completed_.Wait(&mutex_); 46 | } 47 | return absl::OkStatus(); 48 | } 49 | 50 | absl::Status GRPCSuspender::Resume() { 51 | absl::MutexLock lock(&mutex_); 52 | if (!suspended_) { 53 | return absl::FailedPreconditionError("Not suspended"); 54 | } 55 | CHECK_EQ(count_, 0); 56 | suspended_ = false; 57 | resume_.SignalAll(); 58 | return absl::OkStatus(); 59 | } 60 | 61 | void GRPCSuspender::Increment() { 62 | absl::MutexLock lock(&mutex_); 63 | while (suspended_) { 64 | resume_.Wait(&mutex_); 65 | } 66 | ++count_; 67 | } 68 | 69 | void GRPCSuspender::Decrement() { 70 | absl::MutexLock lock(&mutex_); 71 | --count_; 72 | if (count_ == 0) { 73 | in_flight_tasks_completed_.SignalAll(); 74 | } 75 | } 76 | 77 | } // namespace valkey_search::coordinator 78 | -------------------------------------------------------------------------------- /src/coordinator/grpc_suspender.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COORDINATOR_GRPC_SUSPENDER_H_ 31 | #define VALKEYSEARCH_SRC_COORDINATOR_GRPC_SUSPENDER_H_ 32 | 33 | #include 34 | 35 | #include "absl/base/thread_annotations.h" 36 | #include "absl/log/check.h" 37 | #include "absl/status/status.h" 38 | #include "absl/synchronization/mutex.h" 39 | 40 | namespace valkey_search::coordinator { 41 | 42 | // GRPCSuspender is used to suspend and resume gRPC callbacks in combination 43 | // with GRPCSuspensionGuard. This is used to ensure that gRPC callbacks do not 44 | // access shared mutexes used by the child process during fork. 45 | class GRPCSuspender { 46 | public: 47 | static GRPCSuspender& Instance() { 48 | static GRPCSuspender instance; 49 | return instance; 50 | } 51 | absl::Status Suspend(); 52 | absl::Status Resume(); 53 | void Increment(); 54 | void Decrement(); 55 | 56 | private: 57 | GRPCSuspender() = default; 58 | 59 | absl::Mutex mutex_; 60 | int64_t count_ ABSL_GUARDED_BY(mutex_) = 0; 61 | bool suspended_ ABSL_GUARDED_BY(mutex_) = false; 62 | absl::CondVar in_flight_tasks_completed_ ABSL_GUARDED_BY(mutex_); 63 | absl::CondVar resume_ ABSL_GUARDED_BY(mutex_); 64 | }; 65 | 66 | // gRPC runs server callbacks and client-provided callbacks on a background 67 | // thread. This guard ensures that these threads do not access any shared 68 | // mutexes used by the child process during fork. It should be acquired by each 69 | // gRPC callback so that new callbacks can be suspended prior to forking. 70 | class GRPCSuspensionGuard { 71 | public: 72 | explicit GRPCSuspensionGuard(GRPCSuspender& grpc_suspender) 73 | : grpc_suspender_(grpc_suspender) { 74 | grpc_suspender_.Increment(); 75 | } 76 | ~GRPCSuspensionGuard() { grpc_suspender_.Decrement(); } 77 | 78 | private: 79 | GRPCSuspender& grpc_suspender_; 80 | }; 81 | 82 | } // namespace valkey_search::coordinator 83 | 84 | #endif // VALKEYSEARCH_SRC_COORDINATOR_GRPC_SUSPENDER_H_ 85 | -------------------------------------------------------------------------------- /src/coordinator/search_converter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COORDINATOR_SEARCH_CONVERTER_H_ 31 | #define VALKEYSEARCH_SRC_COORDINATOR_SEARCH_CONVERTER_H_ 32 | 33 | #include 34 | 35 | #include "absl/status/statusor.h" 36 | #include "src/coordinator/coordinator.pb.h" 37 | #include "src/query/search.h" 38 | 39 | namespace valkey_search::coordinator { 40 | 41 | absl::StatusOr> 42 | GRPCSearchRequestToParameters(const SearchIndexPartitionRequest& request); 43 | 44 | std::unique_ptr ParametersToGRPCSearchRequest( 45 | const query::VectorSearchParameters& parameters); 46 | 47 | } // namespace valkey_search::coordinator 48 | 49 | #endif // VALKEYSEARCH_SRC_COORDINATOR_SEARCH_CONVERTER_H_ 50 | -------------------------------------------------------------------------------- /src/coordinator/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_COORDINATOR_UTIL_H_ 31 | #define VALKEYSEARCH_SRC_COORDINATOR_UTIL_H_ 32 | 33 | #include 34 | 35 | #include "absl/status/status.h" 36 | #include "grpcpp/support/status.h" 37 | #include "src/coordinator/coordinator.pb.h" 38 | 39 | namespace valkey_search { 40 | inline grpc::Status ToGrpcStatus(const absl::Status& status) { 41 | if (status.ok()) { 42 | return grpc::Status::OK; 43 | } 44 | return {static_cast(status.code()), 45 | std::string(status.message())}; 46 | } 47 | namespace coordinator { 48 | // This offset results in 26673 for Redis default port 6379 - which is COORD 49 | // on a telephone keypad. 50 | static constexpr int kCoordinatorPortOffset = 20294; 51 | 52 | inline int GetCoordinatorPort(int redis_port) { 53 | // TODO Make handling of TLS more robust 54 | if (redis_port == 6378) { 55 | return redis_port + kCoordinatorPortOffset + 1; 56 | } 57 | return redis_port + kCoordinatorPortOffset; 58 | } 59 | } // namespace coordinator 60 | 61 | } // namespace valkey_search 62 | 63 | #endif // VALKEYSEARCH_SRC_COORDINATOR_UTIL_H_ 64 | -------------------------------------------------------------------------------- /src/index_schema.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package valkey_search.data_model; 4 | 5 | enum AttributeDataType { 6 | ATTRIBUTE_DATA_TYPE_UNSPECIFIED = 0; 7 | ATTRIBUTE_DATA_TYPE_HASH = 1; 8 | ATTRIBUTE_DATA_TYPE_JSON = 2; 9 | } 10 | 11 | enum Language { 12 | LANGUAGE_UNSPECIFIED = 0; 13 | LANGUAGE_ENGLISH = 1; 14 | } 15 | 16 | message IndexSchema { 17 | message Stats { 18 | // Total number of hash keys indexed 19 | uint32 documents_count = 1; 20 | // Sum of vectors indexed in all the indexes for the IndexSchema 21 | uint32 records_count = 2 [deprecated = true]; 22 | } 23 | 24 | string name = 1; 25 | repeated string subscribed_key_prefixes = 2; 26 | AttributeDataType attribute_data_type = 3; 27 | Language language = 4; 28 | float score = 5; 29 | repeated Attribute attributes = 6; 30 | Stats stats = 7; 31 | optional uint32 db_num = 8; 32 | } 33 | 34 | enum AttributeIndexType { 35 | ATTRIBUTE_INDEX_TYPE_UNSPECIFIED = 0; 36 | ATTRIBUTE_INDEX_TYPE_VECTOR = 1; 37 | } 38 | 39 | message Attribute { 40 | string alias = 1; 41 | string identifier = 2; 42 | Index index = 3; 43 | } 44 | 45 | message Index { 46 | oneof index_type { 47 | VectorIndex vector_index = 1; 48 | NumericIndex numeric_index = 2; 49 | TagIndex tag_index = 3; 50 | } 51 | } 52 | 53 | message NumericIndex {} 54 | 55 | message TagIndex { 56 | string separator = 1; 57 | bool case_sensitive = 2; 58 | } 59 | 60 | message TrackedKeyMetadata { 61 | string key = 1; 62 | uint64 internal_id = 2; 63 | float magnitude = 3; 64 | } 65 | 66 | message VectorIndex { 67 | uint32 dimension_count = 1; 68 | bool normalize = 2; 69 | DistanceMetric distance_metric = 3; 70 | VectorDataType vector_data_type = 4; 71 | uint32 initial_cap = 5; 72 | oneof algorithm { 73 | HNSWAlgorithm hnsw_algorithm = 6; 74 | FlatAlgorithm flat_algorithm = 7; 75 | } 76 | } 77 | 78 | enum DistanceMetric { 79 | DISTANCE_METRIC_UNSPECIFIED = 0; 80 | DISTANCE_METRIC_L2 = 1; 81 | DISTANCE_METRIC_IP = 2; 82 | DISTANCE_METRIC_COSINE = 3; 83 | } 84 | 85 | enum VectorDataType { 86 | VECTOR_DATA_TYPE_UNSPECIFIED = 0; 87 | VECTOR_DATA_TYPE_FLOAT32 = 1; 88 | } 89 | 90 | message HNSWAlgorithm { 91 | uint32 m = 1; 92 | uint32 ef_construction = 2; 93 | uint32 ef_runtime = 3; 94 | } 95 | 96 | message FlatAlgorithm { 97 | uint32 block_size = 1; 98 | } 99 | -------------------------------------------------------------------------------- /src/query/fanout.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_QUERY_FANOUT_H_ 31 | #define VALKEYSEARCH_SRC_QUERY_FANOUT_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | #include "absl/status/status.h" 39 | #include "src/coordinator/client_pool.h" 40 | #include "src/coordinator/coordinator.pb.h" 41 | #include "src/index_schema.h" 42 | #include "src/query/search.h" 43 | #include "vmsdk/src/thread_pool.h" 44 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 45 | 46 | namespace valkey_search::query::fanout { 47 | 48 | struct FanoutSearchTarget { 49 | enum Type { 50 | kLocal, 51 | kRemote, 52 | }; 53 | Type type; 54 | // Empty string if type is kLocal. 55 | std::string address; 56 | 57 | bool operator==(const FanoutSearchTarget& other) const { 58 | return type == other.type && address == other.address; 59 | } 60 | 61 | friend std::ostream& operator<<(std::ostream& os, 62 | const FanoutSearchTarget& target) { 63 | os << "FanoutSearchTarget{type: " << target.type 64 | << ", address: " << target.address << "}"; 65 | return os; 66 | } 67 | }; 68 | 69 | absl::Status PerformSearchFanoutAsync( 70 | RedisModuleCtx* ctx, std::vector& search_targets, 71 | coordinator::ClientPool* coordinator_client_pool, 72 | std::unique_ptr parameters, 73 | vmsdk::ThreadPool* thread_pool, query::SearchResponseCallback callback); 74 | 75 | std::vector GetSearchTargetsForFanout(RedisModuleCtx* ctx); 76 | 77 | } // namespace valkey_search::query::fanout 78 | 79 | #endif // VALKEYSEARCH_SRC_QUERY_FANOUT_H_ 80 | -------------------------------------------------------------------------------- /src/query/planner.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "src/query/planner.h" 31 | 32 | #include 33 | 34 | #include "absl/log/check.h" 35 | #include "src/indexes/index_base.h" 36 | #include "src/indexes/vector_base.h" 37 | 38 | namespace valkey_search::query { 39 | // TODO: Tune this parameter. 40 | constexpr double kPreFilteringThresholdRatio = 0.001; // 0.1% 41 | // The query planner decides whether to use pre or inline filtering based on 42 | // heuristics. 43 | bool UsePreFiltering(size_t estimated_num_of_keys, 44 | indexes::VectorBase *vector_index) { 45 | if (vector_index->GetIndexerType() == indexes::IndexerType::kFlat) { 46 | /* With a flat index, the search needs to go through all the vectors, 47 | taking O(N*log(k)). With pre-filtering, we can do the same search on the 48 | reduced space, taking O(n*log(k)). Therefore we should always choose 49 | pre-filtering */ 50 | return true; 51 | } 52 | if (vector_index->GetIndexerType() == indexes::IndexerType::kHNSW) { 53 | // TODO: Come up with a formulation accounting for various 54 | // other factors like ef_construction, M, size of vectors, ef_runtime, k 55 | // etc. Also benchmark various combinations to tune the hyperparameters. 56 | size_t N = vector_index->GetCapacity(); 57 | // We choose pre-filtering if the size of the filtered space is below a 58 | // certain threshold (relative to the total size). 59 | return estimated_num_of_keys <= kPreFilteringThresholdRatio * N; 60 | } 61 | CHECK(false) << "Unsupported indexer type: " 62 | << (int)vector_index->GetIndexerType(); 63 | } 64 | } // namespace valkey_search::query 65 | -------------------------------------------------------------------------------- /src/query/planner.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_QUERY_PLANNER_H_ 31 | #define VALKEYSEARCH_SRC_QUERY_PLANNER_H_ 32 | 33 | #include "src/indexes/vector_base.h" 34 | 35 | namespace valkey_search::query { 36 | 37 | // Returns whether to use pre-filtering as opposed to inline filtering based on 38 | // heuristics. 39 | bool UsePreFiltering(size_t estimated_num_of_keys, 40 | indexes::VectorBase *vector_index); 41 | } // namespace valkey_search::query 42 | 43 | #endif // VALKEYSEARCH_SRC_QUERY_PLANNER_H_ 44 | -------------------------------------------------------------------------------- /src/query/response_generator.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_QUERY_RESPONSE_GENERATOR_H_ 31 | #define VALKEYSEARCH_SRC_QUERY_RESPONSE_GENERATOR_H_ 32 | 33 | #include 34 | #include 35 | 36 | #include "src/attribute_data_type.h" 37 | #include "src/coordinator/coordinator.pb.h" 38 | #include "src/indexes/vector_base.h" 39 | #include "src/query/search.h" 40 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 41 | 42 | namespace valkey_search::query { 43 | 44 | // Adds all local content for neighbors to the list of neighbors. 45 | // Skipping neighbors if one of the following: 46 | // Neighbor already contained in the attribute content map. 47 | // Neighbor without any attribute content. 48 | // Neighbor not comply to the pre-filter expression. 49 | void ProcessNeighborsForReply(RedisModuleCtx *ctx, 50 | const AttributeDataType &attribute_data_type, 51 | std::deque &neighbors, 52 | const query::VectorSearchParameters ¶meters, 53 | const std::string &identifier); 54 | 55 | } // namespace valkey_search::query 56 | 57 | #endif // VALKEYSEARCH_SRC_QUERY_RESPONSE_GENERATOR_H_ 58 | -------------------------------------------------------------------------------- /src/rdb_section.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | import "src/index_schema.proto"; 4 | import "src/coordinator/coordinator.proto"; 5 | 6 | package valkey_search.data_model; 7 | 8 | // The ValkeySearch RDB format is split into a series of protocol buffers that 9 | // are dumped to the RDB aux section. The beginning of the RDB format is marked 10 | // by a minimum semantic version and a count of RDBSection protos. Following 11 | // this, we have each RDBSection which each can be followed by any number of 12 | // supplemental content sections. A supplemental content section allows for 13 | // storing chunked data to the RDB without needing to serialize it in memory 14 | // first. Supplemental content begins with a SupplementalContentHeader, and is 15 | // followed by any number of SupplementalContentChunks, terminated by an empty 16 | // SupplementalContentChunk. 17 | // 18 | // See rfc/rdb-format.md for more details. 19 | // 20 | // To add a new RDBSection, add the type to the RDBSectionType enum and add the 21 | // content to RDBSection contents field. SupplementalContent can be added in a 22 | // similar fashion. 23 | 24 | enum RDBSectionType { 25 | RDB_SECTION_UNSET = 0; 26 | RDB_SECTION_INDEX_SCHEMA = 1; 27 | RDB_SECTION_GLOBAL_METADATA = 2; 28 | } 29 | 30 | message RDBSection { 31 | RDBSectionType type = 1; 32 | uint32 supplemental_count = 2; 33 | oneof contents { 34 | IndexSchema index_schema_contents = 3; 35 | coordinator.GlobalMetadata global_metadata_contents = 4; 36 | } 37 | } 38 | 39 | enum SupplementalContentType { 40 | SUPPLEMENTAL_CONTENT_UNSPECIFIED = 0; 41 | SUPPLEMENTAL_CONTENT_INDEX_CONTENT = 1; 42 | SUPPLEMENTAL_CONTENT_KEY_TO_ID_MAP = 2; 43 | } 44 | 45 | message IndexContentHeader { 46 | Attribute attribute = 1; 47 | } 48 | 49 | message KeyToIDMappingHeader { 50 | Attribute attribute = 1; 51 | } 52 | 53 | message SupplementalContentHeader { 54 | SupplementalContentType type = 1; 55 | oneof header { 56 | IndexContentHeader index_content_header = 2; 57 | KeyToIDMappingHeader key_to_id_map_header = 3; 58 | }; 59 | } 60 | 61 | message SupplementalContentChunk { 62 | // When not present, indicates EOF for this SupplementalContent 63 | optional bytes binary_content = 1; 64 | } -------------------------------------------------------------------------------- /src/server_events.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef THIRD_PARTY_SRC_SERVER_EVENT_SUBSCRIPTION_H_ 31 | #define THIRD_PARTY_SRC_SERVER_EVENT_SUBSCRIPTION_H_ 32 | 33 | namespace valkey_search::server_events { 34 | 35 | void SubscribeToServerEvents(); 36 | 37 | } 38 | 39 | #endif // THIRD_PARTY_SRC_SERVER_EVENT_SUBSCRIPTION_H_ 40 | -------------------------------------------------------------------------------- /src/utils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_PATRICIA_TREE ${CMAKE_CURRENT_LIST_DIR}/patricia_tree.h) 2 | 3 | add_library(patricia_tree INTERFACE ${SRCS_PATRICIA_TREE}) 4 | target_include_directories(patricia_tree INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 5 | 6 | set(SRCS_SEGMENT_TREE ${CMAKE_CURRENT_LIST_DIR}/segment_tree.h) 7 | 8 | add_library(segment_tree INTERFACE ${SRCS_SEGMENT_TREE}) 9 | target_include_directories(segment_tree INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 10 | 11 | set(SRCS_STRING_INTERNING ${CMAKE_CURRENT_LIST_DIR}/string_interning.cc 12 | ${CMAKE_CURRENT_LIST_DIR}/string_interning.h) 13 | 14 | valkey_search_add_static_library(string_interning "${SRCS_STRING_INTERNING}") 15 | target_include_directories(string_interning PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 16 | target_link_libraries(string_interning PUBLIC allocator) 17 | 18 | set(SRCS_ALLOCATOR ${CMAKE_CURRENT_LIST_DIR}/allocator.cc 19 | ${CMAKE_CURRENT_LIST_DIR}/allocator.h) 20 | 21 | valkey_search_add_static_library(allocator "${SRCS_ALLOCATOR}") 22 | target_include_directories(allocator PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 23 | target_link_libraries(allocator PUBLIC intrusive_list) 24 | target_link_libraries(allocator PUBLIC intrusive_ref_count) 25 | if(APPLE) 26 | target_link_libraries(allocator PUBLIC absl::base) 27 | else() 28 | target_link_libraries(allocator PUBLIC ${GRPC_LIB}) 29 | endif() 30 | 31 | set(SRCS_INTRUSIVE_LIST ${CMAKE_CURRENT_LIST_DIR}/intrusive_list.h) 32 | 33 | add_library(intrusive_list INTERFACE ${SRCS_INTRUSIVE_LIST}) 34 | target_include_directories(intrusive_list INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 35 | 36 | set(SRCS_LRU ${CMAKE_CURRENT_LIST_DIR}/lru.h) 37 | 38 | add_library(lru INTERFACE ${SRCS_LRU}) 39 | target_include_directories(lru INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 40 | target_link_libraries(lru INTERFACE intrusive_list) 41 | 42 | set(SRCS_INTRUSIVE_REF_COUNT ${CMAKE_CURRENT_LIST_DIR}/intrusive_ref_count.h) 43 | 44 | add_library(intrusive_ref_count INTERFACE ${SRCS_INTRUSIVE_REF_COUNT}) 45 | target_include_directories(intrusive_ref_count 46 | INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 47 | -------------------------------------------------------------------------------- /src/utils/intrusive_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_UTILS_INTRUSIVE_LIST_H_ 31 | #define VALKEYSEARCH_SRC_UTILS_INTRUSIVE_LIST_H_ 32 | 33 | #include 34 | 35 | #include "absl/log/check.h" 36 | 37 | namespace valkey_search { 38 | template 39 | class IntrusiveList { 40 | public: 41 | IntrusiveList() : head_(nullptr), tail_(nullptr) {} 42 | void PushBack(T *node); 43 | void Remove(T *node); 44 | T *Front() const { return head_; } 45 | bool Empty() const { return head_ == nullptr; } 46 | size_t Size() const { return size_; } 47 | 48 | private: 49 | T *head_; 50 | T *tail_; 51 | size_t size_{0}; 52 | }; 53 | 54 | template 55 | void IntrusiveList::PushBack(T *node) { 56 | CHECK(node->next == nullptr); 57 | CHECK(node->prev == nullptr); 58 | if (tail_) { 59 | tail_->next = node; 60 | node->prev = tail_; 61 | tail_ = node; 62 | } else { 63 | head_ = tail_ = node; 64 | } 65 | node->next = nullptr; 66 | ++size_; 67 | } 68 | 69 | template 70 | void IntrusiveList::Remove(T *node) { 71 | if (node->next == nullptr && node->prev == nullptr && tail_ != node && 72 | head_ != node) { 73 | return; 74 | } 75 | if (node->prev) { 76 | node->prev->next = node->next; 77 | } else { 78 | head_ = node->next; 79 | } 80 | if (node->next) { 81 | node->next->prev = node->prev; 82 | } else { 83 | tail_ = node->prev; 84 | } 85 | node->next = node->prev = nullptr; 86 | --size_; 87 | } 88 | 89 | } // namespace valkey_search 90 | #endif // VALKEYSEARCH_SRC_UTILS_INTRUSIVE_LIST_H_ 91 | -------------------------------------------------------------------------------- /src/utils/intrusive_ref_count.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_UTILS_INTRUSIVE_REF_COUNT_H_ 31 | #define VALKEYSEARCH_SRC_UTILS_INTRUSIVE_REF_COUNT_H_ 32 | #include 33 | #include 34 | 35 | #define DEFINE_UNIQUE_PTR_TYPE(Type) \ 36 | using Unique##Type##Ptr = std::unique_ptr; 37 | 38 | class IntrusiveRefCount { 39 | public: 40 | void IncrementRef() { ref_count_.fetch_add(1, std::memory_order_relaxed); } 41 | 42 | void DecrementRef() { 43 | if (ref_count_.fetch_sub(1, std::memory_order_acq_rel) == 1) { 44 | delete this; 45 | } 46 | } 47 | 48 | template 49 | static std::unique_ptr Create(Args&&... args) { 50 | T* obj = new T(std::forward(args)...); 51 | obj->IncrementRef(); // Start with a ref count of 1 52 | 53 | // Return a unique_ptr with a custom deleter 54 | return std::unique_ptr( 55 | obj, [](T* ptr) { ptr->DecrementRef(); }); 56 | } 57 | 58 | protected: 59 | virtual ~IntrusiveRefCount() = default; 60 | 61 | private: 62 | std::atomic ref_count_ = 0; 63 | }; 64 | 65 | #define CREATE_UNIQUE_PTR(Type, ...) \ 66 | IntrusiveRefCount::Create(__VA_ARGS__) 67 | 68 | #endif // VALKEYSEARCH_SRC_UTILS_INTRUSIVE_REF_COUNT_H_ 69 | -------------------------------------------------------------------------------- /src/utils/lru.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_SRC_UTILS_LRU_H_ 31 | #define VALKEYSEARCH_SRC_UTILS_LRU_H_ 32 | #include 33 | 34 | #include "absl/log/check.h" 35 | #include "src/utils/intrusive_list.h" 36 | 37 | namespace valkey_search { 38 | template 39 | class LRU { 40 | public: 41 | explicit LRU(size_t capacity) : capacity_(capacity){}; 42 | T *InsertAtTop(T *node); 43 | void Promote(T *node); 44 | void Remove(T *node); 45 | size_t Size() const { return list_.Size(); } 46 | 47 | private: 48 | size_t capacity_{0}; 49 | IntrusiveList list_; 50 | }; 51 | 52 | template 53 | T *LRU::InsertAtTop(T *node) { 54 | CHECK(node->next == nullptr && node->prev == nullptr); 55 | if (list_.Size() < capacity_) { 56 | list_.PushBack(node); 57 | return nullptr; 58 | } 59 | auto *last = list_.Front(); 60 | list_.Remove(last); 61 | list_.PushBack(node); 62 | return last; 63 | } 64 | 65 | template 66 | void LRU::Promote(T *node) { 67 | Remove(node); 68 | CHECK(!InsertAtTop(node)); 69 | } 70 | 71 | template 72 | void LRU::Remove(T *node) { 73 | list_.Remove(node); 74 | } 75 | 76 | } // namespace valkey_search 77 | 78 | #endif // VALKEYSEARCH_SRC_UTILS_LRU_H_ 79 | -------------------------------------------------------------------------------- /src/valkey_search_options.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | #pragma once 30 | 31 | #include "vmsdk/src/module_config.h" 32 | 33 | namespace valkey_search { 34 | namespace options { 35 | 36 | namespace config = vmsdk::config; 37 | 38 | /// Return a mutable reference to the HNSW resize configuration parameter 39 | config::Number& GetHNSWBlockSize(); 40 | 41 | /// Return the configuration entry that allows the caller to control the 42 | /// number of reader threads 43 | config::Number& GetReaderThreadCount(); 44 | 45 | /// Return the configuration entry that allows the caller to control the 46 | /// number of writer threads 47 | config::Number& GetWriterThreadCount(); 48 | 49 | /// Return an immutable reference to the "use-coordinator" flag 50 | const config::Boolean& GetUseCoordinator(); 51 | 52 | /// Return the log level 53 | config::Enum& GetLogLevel(); 54 | 55 | /// Reset the state of the options (mainly needed for testing) 56 | absl::Status Reset(); 57 | } // namespace options 58 | } // namespace valkey_search 59 | -------------------------------------------------------------------------------- /testing/coordinator/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_METADATA_MANAGER_TEST 2 | ${CMAKE_CURRENT_LIST_DIR}/metadata_manager_test.cc) 3 | 4 | add_executable(metadata_manager_test ${SRCS_METADATA_MANAGER_TEST}) 5 | target_include_directories(metadata_manager_test 6 | PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 7 | target_link_libraries(metadata_manager_test PUBLIC coordinator_common) 8 | target_link_libraries(metadata_manager_test PUBLIC client) 9 | target_link_libraries(metadata_manager_test PUBLIC client_pool) 10 | target_link_libraries(metadata_manager_test PUBLIC coordinator_cc_proto) 11 | target_link_libraries(metadata_manager_test PUBLIC metadata_manager) 12 | target_link_libraries(metadata_manager_test PUBLIC util) 13 | target_link_libraries(metadata_manager_test PUBLIC testing_common) 14 | target_link_libraries(metadata_manager_test PUBLIC managed_pointers) 15 | target_link_libraries(metadata_manager_test PUBLIC module) 16 | target_link_libraries(metadata_manager_test PUBLIC utils) 17 | target_link_libraries(metadata_manager_test PUBLIC valkey_module) 18 | finalize_test_flags(metadata_manager_test) 19 | 20 | set(SRCS_COMMON ${CMAKE_CURRENT_LIST_DIR}/common.h) 21 | 22 | add_library(coordinator_common INTERFACE ${SRCS_COMMON}) 23 | target_include_directories(coordinator_common 24 | INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 25 | target_link_libraries(coordinator_common INTERFACE client) 26 | target_link_libraries(coordinator_common INTERFACE client_pool) 27 | target_link_libraries(coordinator_common INTERFACE server) 28 | -------------------------------------------------------------------------------- /testing/coordinator/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VALKEYSEARCH_TESTING_COORDINATOR_COMMON_H_ 31 | #define VALKEYSEARCH_TESTING_COORDINATOR_COMMON_H_ 32 | 33 | #include 34 | #include 35 | 36 | #include "absl/strings/string_view.h" 37 | #include "gmock/gmock.h" 38 | #include "src/coordinator/client.h" 39 | #include "src/coordinator/client_pool.h" 40 | #include "src/coordinator/server.h" 41 | 42 | namespace valkey_search::coordinator { 43 | 44 | class MockClientPool : public ClientPool { 45 | public: 46 | MockClientPool() : ClientPool(nullptr) {} 47 | MOCK_METHOD(std::shared_ptr, GetClient, (absl::string_view address), 48 | (override)); 49 | }; 50 | 51 | class MockClient : public Client { 52 | public: 53 | MOCK_METHOD(void, GetGlobalMetadata, (GetGlobalMetadataCallback done), 54 | (override)); 55 | MOCK_METHOD(void, SearchIndexPartition, 56 | (std::unique_ptr request, 57 | SearchIndexPartitionCallback done), 58 | (override)); 59 | }; 60 | 61 | class MockServer : public Server { 62 | public: 63 | MOCK_METHOD(uint16_t, GetPort, (), (const, override)); 64 | }; 65 | 66 | } // namespace valkey_search::coordinator 67 | 68 | #endif // VALKEYSEARCH_TESTING_COORDINATOR_COMMON_H_ 69 | -------------------------------------------------------------------------------- /testing/integration/.gitignore: -------------------------------------------------------------------------------- 1 | .build* 2 | __pycache__ 3 | Testing 4 | -------------------------------------------------------------------------------- /testing/integration/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.12) 2 | project(MyPythonTests) 3 | 4 | # Find Python interpreter 5 | find_package(Python REQUIRED COMPONENTS Interpreter) 6 | 7 | # Check if pip is installed 8 | execute_process( 9 | COMMAND ${Python_EXECUTABLE} -m pip --version 10 | RESULT_VARIABLE PIP_CHECK 11 | ) 12 | 13 | # Install pip if missing 14 | if(NOT PIP_CHECK EQUAL 0) 15 | message(FATAL_ERROR "pip not found.") 16 | endif() 17 | 18 | # Define virtual environment path 19 | set(VENV_DIR "${CMAKE_BINARY_DIR}/venv") 20 | 21 | # Step 1: Create virtual environment if it doesn't exist 22 | if(NOT EXISTS "${VENV_DIR}") 23 | message(STATUS "Creating virtual environment at ${VENV_DIR}") 24 | execute_process( 25 | COMMAND ${Python_EXECUTABLE} -m venv ${VENV_DIR} 26 | RESULT_VARIABLE VENV_RESULT 27 | ) 28 | if(NOT VENV_RESULT EQUAL 0) 29 | message(FATAL_ERROR "Failed to create virtual environment.") 30 | endif() 31 | endif() 32 | 33 | # Set paths to virtual environment executables 34 | set(PYTHON_VENV_EXECUTABLE "${VENV_DIR}/bin/python") 35 | set(PIP_VENV_EXECUTABLE "${VENV_DIR}/bin/pip") 36 | 37 | # Step 2: Install dependencies from requirements.txt 38 | if(NOT EXISTS "${CMAKE_SOURCE_DIR}/requirements.txt") 39 | message(FATAL_ERROR "requirements.txt not found! Python dependencies may be missing.") 40 | endif() 41 | 42 | message(STATUS "Installing dependencies in virtual environment...") 43 | execute_process( 44 | COMMAND ${PYTHON_VENV_EXECUTABLE} -m pip install -r ${CMAKE_SOURCE_DIR}/requirements.txt 45 | RESULT_VARIABLE PIP_RESULT 46 | ) 47 | if(NOT PIP_RESULT EQUAL 0) 48 | message(FATAL_ERROR "Failed to install dependencies from requirements.txt") 49 | endif() 50 | 51 | include(CTest) # Enables test support 52 | 53 | # Function to register Python tests 54 | function(add_pytest test_name test_script) 55 | add_test( 56 | NAME ${test_name} 57 | COMMAND ${PYTHON_VENV_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${test_script} 58 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 59 | ) 60 | endfunction() 61 | 62 | # Add tests 63 | add_pytest(vector_search_integration_test vector_search_integration_test.py) 64 | add_pytest(stability_test stability_test.py) 65 | 66 | # Set timeout for long-running tests 67 | set_tests_properties(stability_test PROPERTIES TIMEOUT 3600) 68 | -------------------------------------------------------------------------------- /testing/integration/requirements.txt: -------------------------------------------------------------------------------- 1 | valkey==6.0.2 2 | absl-py==2.1.0 3 | numpy==2.2.2 -------------------------------------------------------------------------------- /testing/query/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_FANOUT_TEST ${CMAKE_CURRENT_LIST_DIR}/fanout_test.cc) 2 | 3 | add_executable(fanout_test ${SRCS_FANOUT_TEST}) 4 | target_include_directories(fanout_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 5 | target_link_libraries(fanout_test PUBLIC index_schema_cc_proto) 6 | target_link_libraries(fanout_test PUBLIC valkey_search) 7 | target_link_libraries(fanout_test PUBLIC coordinator_cc_proto) 8 | target_link_libraries(fanout_test PUBLIC search_converter) 9 | target_link_libraries(fanout_test PUBLIC util) 10 | target_link_libraries(fanout_test PUBLIC numeric) 11 | target_link_libraries(fanout_test PUBLIC tag) 12 | target_link_libraries(fanout_test PUBLIC fanout) 13 | target_link_libraries(fanout_test PUBLIC testing_common) 14 | target_link_libraries(fanout_test PUBLIC coordinator_common) 15 | target_link_libraries(fanout_test PUBLIC module) 16 | target_link_libraries(fanout_test PUBLIC utils) 17 | target_link_libraries(fanout_test PUBLIC valkey_module) 18 | finalize_test_flags(fanout_test) 19 | 20 | set(SRCS_RESPONSE_GENERATOR_TEST 21 | ${CMAKE_CURRENT_LIST_DIR}/response_generator_test.cc) 22 | 23 | add_executable(response_generator_test ${SRCS_RESPONSE_GENERATOR_TEST}) 24 | target_include_directories(response_generator_test 25 | PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 26 | target_link_libraries(response_generator_test PUBLIC attribute_data_type) 27 | target_link_libraries(response_generator_test PUBLIC vector_base) 28 | target_link_libraries(response_generator_test PUBLIC predicate) 29 | target_link_libraries(response_generator_test PUBLIC response_generator) 30 | target_link_libraries(response_generator_test PUBLIC search) 31 | target_link_libraries(response_generator_test PUBLIC string_interning) 32 | target_link_libraries(response_generator_test PUBLIC testing_common) 33 | target_link_libraries(response_generator_test PUBLIC managed_pointers) 34 | target_link_libraries(response_generator_test PUBLIC type_conversions) 35 | target_link_libraries(response_generator_test PUBLIC module) 36 | target_link_libraries(response_generator_test PUBLIC utils) 37 | target_link_libraries(response_generator_test PUBLIC valkey_module) 38 | finalize_test_flags(response_generator_test) 39 | -------------------------------------------------------------------------------- /testing/utils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_ALLOCATOR_TEST ${CMAKE_CURRENT_LIST_DIR}/allocator_test.cc) 2 | 3 | add_executable(allocator_test ${SRCS_ALLOCATOR_TEST}) 4 | target_include_directories(allocator_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 5 | target_link_libraries(allocator_test PUBLIC allocator) 6 | target_link_libraries(allocator_test PUBLIC log) 7 | target_link_libraries(allocator_test PUBLIC intrusive_ref_count) 8 | target_link_libraries(allocator_test PUBLIC vmsdk_testing_infra) 9 | finalize_test_flags(allocator_test) 10 | 11 | set(SRCS_INTRUSIVE_LIST_TEST ${CMAKE_CURRENT_LIST_DIR}/intrusive_list_test.cc) 12 | 13 | add_executable(intrusive_list_test ${SRCS_INTRUSIVE_LIST_TEST}) 14 | target_include_directories(intrusive_list_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 15 | target_link_libraries(intrusive_list_test PUBLIC intrusive_list) 16 | finalize_test_flags(intrusive_list_test) 17 | 18 | set(SRCS_INTRUSIVE_REF_COUNT_TEST 19 | ${CMAKE_CURRENT_LIST_DIR}/intrusive_ref_count_test.cc) 20 | 21 | add_executable(intrusive_ref_count_test ${SRCS_INTRUSIVE_REF_COUNT_TEST}) 22 | target_include_directories(intrusive_ref_count_test 23 | PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 24 | target_link_libraries(intrusive_ref_count_test PUBLIC intrusive_ref_count) 25 | finalize_test_flags(intrusive_ref_count_test) 26 | 27 | set(SRCS_LRU_TEST ${CMAKE_CURRENT_LIST_DIR}/lru_test.cc) 28 | 29 | add_executable(lru_test ${SRCS_LRU_TEST}) 30 | target_include_directories(lru_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 31 | target_link_libraries(lru_test PUBLIC lru) 32 | finalize_test_flags(lru_test) 33 | 34 | set(SRCS_PATRICIA_TREE_TEST ${CMAKE_CURRENT_LIST_DIR}/patricia_tree_test.cc) 35 | 36 | add_executable(patricia_tree_test ${SRCS_PATRICIA_TREE_TEST}) 37 | target_include_directories(patricia_tree_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 38 | target_link_libraries(patricia_tree_test PUBLIC patricia_tree) 39 | finalize_test_flags(patricia_tree_test) 40 | 41 | set(SRCS_SEGMENT_TREE_TEST ${CMAKE_CURRENT_LIST_DIR}/segment_tree_test.cc) 42 | 43 | add_executable(segment_tree_test ${SRCS_SEGMENT_TREE_TEST}) 44 | target_include_directories(segment_tree_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 45 | target_link_libraries(segment_tree_test PUBLIC segment_tree) 46 | finalize_test_flags(segment_tree_test) 47 | 48 | set(SRCS_STRING_INTERNING_TEST 49 | ${CMAKE_CURRENT_LIST_DIR}/string_interning_test.cc) 50 | 51 | add_executable(string_interning_test ${SRCS_STRING_INTERNING_TEST}) 52 | target_include_directories(string_interning_test 53 | PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 54 | target_link_libraries(string_interning_test PUBLIC allocator) 55 | target_link_libraries(string_interning_test PUBLIC intrusive_ref_count) 56 | target_link_libraries(string_interning_test PUBLIC string_interning) 57 | target_link_libraries(string_interning_test PUBLIC utils) 58 | target_link_libraries(string_interning_test PUBLIC log) 59 | finalize_test_flags(string_interning_test) 60 | -------------------------------------------------------------------------------- /testing/utils/intrusive_list_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "src/utils/intrusive_list.h" 31 | 32 | #include 33 | #include 34 | 35 | #include "gtest/gtest.h" 36 | 37 | namespace valkey_search { 38 | 39 | namespace { 40 | struct TestNode { 41 | std::string key; 42 | TestNode *prev{nullptr}; 43 | TestNode *next{nullptr}; 44 | }; 45 | 46 | TEST(IntrusiveListTest, SimpleAddRemove) { 47 | IntrusiveList list; 48 | EXPECT_EQ(list.Size(), 0); 49 | std::vector nodes; 50 | nodes.push_back(TestNode{.key = "a"}); 51 | nodes.push_back(TestNode{.key = "b"}); 52 | nodes.push_back(TestNode{.key = "c"}); 53 | for (auto &node : nodes) { 54 | list.PushBack(&node); 55 | } 56 | EXPECT_EQ(list.Size(), 3); 57 | auto i = 0; 58 | while (!list.Empty()) { 59 | auto node = list.Front(); 60 | list.Remove(node); 61 | EXPECT_EQ(node->key, nodes[i].key); 62 | ++i; 63 | } 64 | EXPECT_EQ(i, 3); 65 | EXPECT_EQ(list.Size(), 0L); 66 | } 67 | 68 | } // namespace 69 | } // namespace valkey_search 70 | -------------------------------------------------------------------------------- /testing/utils/intrusive_ref_count_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "src/utils/intrusive_ref_count.h" 31 | 32 | #include // NOLINT(build/c++11) 33 | #include 34 | 35 | #include "gtest/gtest.h" 36 | 37 | namespace valkey_search { 38 | 39 | namespace { 40 | struct Tester : public IntrusiveRefCount { 41 | Tester(int a_value, bool &a_deleted) : value(a_value), deleted(a_deleted) {} 42 | ~Tester() override { deleted = true; } 43 | int value; 44 | bool &deleted; 45 | }; 46 | 47 | TEST(IntrusiveRefCountTest, SimpleRefCount) { 48 | bool deleted = false; 49 | { 50 | auto ptr = CREATE_UNIQUE_PTR(Tester, 10, deleted); 51 | EXPECT_EQ(ptr->value, 10); 52 | EXPECT_FALSE(deleted); 53 | ptr->IncrementRef(); 54 | ptr->IncrementRef(); 55 | ptr->IncrementRef(); 56 | EXPECT_EQ(ptr->value, 10); 57 | EXPECT_FALSE(deleted); 58 | ptr->DecrementRef(); 59 | ptr->DecrementRef(); 60 | ptr->DecrementRef(); 61 | EXPECT_EQ(ptr->value, 10); 62 | EXPECT_FALSE(deleted); 63 | EXPECT_EQ(ptr->value, 10); 64 | } 65 | EXPECT_TRUE(deleted); 66 | } 67 | 68 | DEFINE_UNIQUE_PTR_TYPE(Tester); 69 | 70 | void FunctionToRunInThread(int thread_id, Tester *ptr) { 71 | for (int i = 0; i < 1000 * thread_id; ++i) { 72 | ptr->IncrementRef(); 73 | ptr->DecrementRef(); 74 | } 75 | } 76 | 77 | TEST(IntrusiveRefCountTest, Concurrent) { 78 | const int num_threads = 5; 79 | std::vector threads; 80 | bool deleted = false; 81 | { 82 | auto ptr = CREATE_UNIQUE_PTR(Tester, 10, deleted); 83 | for (int i = 0; i < num_threads; ++i) { 84 | threads.emplace_back(FunctionToRunInThread, i, ptr.get()); 85 | } 86 | 87 | for (auto &thread : threads) { 88 | thread.join(); 89 | } 90 | } 91 | EXPECT_TRUE(deleted); 92 | } 93 | 94 | } // namespace 95 | } // namespace valkey_search 96 | -------------------------------------------------------------------------------- /testing/utils/lru_test.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "src/utils/lru.h" 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include "absl/strings/str_cat.h" 37 | #include "gtest/gtest.h" 38 | 39 | namespace valkey_search { 40 | 41 | namespace { 42 | struct TestNode { 43 | std::string key; 44 | TestNode *prev{nullptr}; 45 | TestNode *next{nullptr}; 46 | }; 47 | 48 | constexpr int kLruCapacity = 10; 49 | 50 | std::vector InitializeLRU(LRU &lru, int size) { 51 | EXPECT_EQ(lru.Size(), 0L); 52 | std::vector nodes; 53 | nodes.reserve(20); 54 | for (int i = 0; i < 20; ++i) { 55 | nodes.push_back(TestNode{.key = absl::StrCat(i)}); 56 | } 57 | for (size_t i = 0; i < nodes.size(); ++i) { 58 | TestNode *removed = lru.InsertAtTop(&nodes[i]); 59 | if (i < kLruCapacity) { 60 | EXPECT_EQ(removed, nullptr); 61 | EXPECT_EQ(lru.Size(), i + 1); 62 | } else { 63 | EXPECT_EQ(removed, &nodes[i - kLruCapacity]); 64 | EXPECT_EQ(lru.Size(), kLruCapacity); 65 | } 66 | } 67 | EXPECT_EQ(lru.Size(), kLruCapacity); 68 | return nodes; 69 | } 70 | 71 | TEST(LRUTest, SimpleAddRemove) { 72 | LRU lru(kLruCapacity); 73 | auto nodes = InitializeLRU(lru, 20); 74 | for (size_t i = 0; i < nodes.size(); ++i) { 75 | lru.Remove(&nodes[i]); 76 | if (i < kLruCapacity) { 77 | EXPECT_EQ(lru.Size(), kLruCapacity); 78 | } else { 79 | EXPECT_EQ(lru.Size(), nodes.size() - i - 1); 80 | } 81 | } 82 | } 83 | 84 | TEST(LRUTest, Promote) { 85 | LRU lru(kLruCapacity); 86 | auto nodes = InitializeLRU(lru, 15); 87 | for (size_t i = nodes.size() - 1; i >= nodes.size() - kLruCapacity; --i) { 88 | lru.Promote(&nodes[i]); 89 | } 90 | for (size_t i = 0; i < nodes.size() - kLruCapacity; ++i) { 91 | TestNode *removed = lru.InsertAtTop(&nodes[i]); 92 | EXPECT_EQ(removed, &nodes[nodes.size() - i - 1]); 93 | } 94 | } 95 | } // namespace 96 | } // namespace valkey_search 97 | -------------------------------------------------------------------------------- /third_party/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(simsimd) 2 | add_subdirectory(hdrhistogram_c) 3 | add_subdirectory(hnswlib) 4 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(src) 2 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/LICENSE: -------------------------------------------------------------------------------- 1 | The code in this repository code was Written by Gil Tene, Michael Barker, 2 | and Matt Warren, and released to the public domain, as explained at 3 | http://creativecommons.org/publicdomain/zero/1.0/ 4 | 5 | For users of this code who wish to consume it under the "BSD" license 6 | rather than under the public domain or CC0 contribution text mentioned 7 | above, the code found under this directory is *also* provided under the 8 | following license (commonly referred to as the BSD 2-Clause License). This 9 | license does not detract from the above stated release of the code into 10 | the public domain, and simply represents an additional license granted by 11 | the Author. 12 | 13 | ----------------------------------------------------------------------------- 14 | ** Beginning of "BSD 2-Clause License" text. ** 15 | 16 | Copyright (c) 2012, 2013, 2014 Gil Tene 17 | Copyright (c) 2014 Michael Barker 18 | Copyright (c) 2014 Matt Warren 19 | All rights reserved. 20 | 21 | Redistribution and use in source and binary forms, with or without 22 | modification, are permitted provided that the following conditions are met: 23 | 24 | 1. Redistributions of source code must retain the above copyright notice, 25 | this list of conditions and the following disclaimer. 26 | 27 | 2. Redistributions in binary form must reproduce the above copyright notice, 28 | this list of conditions and the following disclaimer in the documentation 29 | and/or other materials provided with the distribution. 30 | 31 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 32 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 35 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 36 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 37 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 38 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 39 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 40 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 41 | THE POSSIBILITY OF SUCH DAMAGE. 42 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/README.md: -------------------------------------------------------------------------------- 1 | HdrHistogram_c: 'C' port of High Dynamic Range (HDR) Histogram 2 | 3 | HdrHistogram 4 | ---------------------------------------------- 5 | 6 | [![Gitter chat](https://badges.gitter.im/HdrHistogram/HdrHistogram.png)](https://gitter.im/HdrHistogram/HdrHistogram) 7 | 8 | Windows Build: [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/HdrHistogram/HdrHistogram_c?svg=true)](https://ci.appveyor.com/project/mikeb01/hdrhistogram-c) 9 | 10 | Linux Build: [![Build Status](https://semaphoreci.com/api/v1/mikeb01/hdrhistogram_c/branches/master/badge.svg)](https://semaphoreci.com/mikeb01/hdrhistogram_c) 11 | 12 | This port contains a subset of the functionality supported by the Java 13 | implementation. The current supported features are: 14 | 15 | * Standard histogram with 64 bit counts (32/16 bit counts not supported) 16 | * All iterator types (all values, recorded, percentiles, linear, logarithmic) 17 | * Histogram serialisation (encoding version 1.2, decoding 1.0-1.2) 18 | * Reader/writer phaser and interval recorder 19 | 20 | Features not supported, but planned 21 | 22 | * Auto-resizing of histograms 23 | 24 | Features unlikely to be implemented 25 | 26 | * Double histograms 27 | * Atomic/Concurrent histograms 28 | * 16/32 bit histograms 29 | 30 | # Simple Tutorial 31 | 32 | ## Recording values 33 | 34 | ```C 35 | #include 36 | 37 | struct hdr_histogram* histogram; 38 | 39 | // Initialise the histogram 40 | hdr_init( 41 | 1, // Minimum value 42 | INT64_C(3600000000), // Maximum value 43 | 3, // Number of significant figures 44 | &histogram) // Pointer to initialise 45 | 46 | // Record value 47 | hdr_record_value( 48 | histogram, // Histogram to record to 49 | value) // Value to record 50 | 51 | // Record value n times 52 | hdr_record_values( 53 | histogram, // Histogram to record to 54 | value, // Value to record 55 | 10) // Record value 10 times 56 | 57 | // Record value with correction for co-ordinated omission. 58 | hdr_record_corrected_value( 59 | histogram, // Histogram to record to 60 | value, // Value to record 61 | 1000) // Record with expected interval of 1000. 62 | 63 | // Print out the values of the histogram 64 | hdr_percentiles_print( 65 | histogram, 66 | stdout, // File to write to 67 | 5, // Granularity of printed values 68 | 1.0, // Multiplier for results 69 | CLASSIC); // Format CLASSIC/CSV supported. 70 | ``` 71 | 72 | ## More examples 73 | 74 | For more detailed examples of recording and logging results look at the 75 | [hdr_decoder](examples/hdr_decoder.c) 76 | and [hiccup](examples/hiccup.c) 77 | examples. You can run hiccup and decoder 78 | and pipe the results of one into the other. 79 | 80 | ``` 81 | $ ./examples/hiccup | ./examples/hdr_decoder 82 | ``` 83 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(hdr_decoder hdr_decoder.c) 2 | if(WIN32) 3 | target_link_libraries(hdr_decoder hdr_histogram_static) 4 | else() 5 | target_link_libraries(hdr_decoder hdr_histogram m z) 6 | endif() 7 | 8 | check_library_exists(rt clock_gettime "" RT_EXISTS) 9 | if(RT_EXISTS) 10 | target_link_libraries(hdr_decoder rt) 11 | endif(RT_EXISTS) 12 | 13 | install(TARGETS hdr_decoder DESTINATION bin) 14 | 15 | if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") 16 | add_executable(hiccup hiccup.c) 17 | target_link_libraries(hiccup hdr_histogram m z pthread rt) 18 | install(TARGETS hiccup DESTINATION bin) 19 | endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux") 20 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/examples/hdr_decoder.c: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_decoder.c 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | #if defined(_MSC_VER) 18 | #pragma warning(push) 19 | #pragma warning(disable: 4996) 20 | #endif 21 | 22 | int main(int argc, char** argv) 23 | { 24 | int rc = 0; 25 | FILE* f; 26 | struct hdr_log_reader reader; 27 | struct hdr_histogram* h = NULL; 28 | hdr_timespec timestamp, interval; 29 | 30 | if (argc == 1) 31 | { 32 | f = stdin; 33 | } 34 | else 35 | { 36 | f = fopen(argv[1], "r"); 37 | } 38 | 39 | if (!f) 40 | { 41 | fprintf(stderr, "Failed to open file(%s):%s\n", argv[1], strerror(errno)); 42 | return -1; 43 | } 44 | 45 | if (hdr_log_reader_init(&reader)) 46 | { 47 | fprintf(stderr, "Failed to init reader\n"); 48 | return -1; 49 | } 50 | 51 | rc = hdr_log_read_header(&reader, f); 52 | if(rc) 53 | { 54 | fprintf(stderr, "Failed to read header: %s\n", hdr_strerror(rc)); 55 | return -1; 56 | } 57 | 58 | while (true) 59 | { 60 | rc = hdr_log_read(&reader, f, &h, ×tamp, &interval); 61 | 62 | if (0 == rc) 63 | { 64 | hdr_percentiles_print(h, stdout, 5, 1.0, CLASSIC); 65 | } 66 | else if (EOF == rc) 67 | { 68 | break; 69 | } 70 | else 71 | { 72 | fprintf(stderr, "Failed to print histogram: %s\n", hdr_strerror(rc)); 73 | return -1; 74 | } 75 | } 76 | 77 | return 0; 78 | } 79 | 80 | #if defined(_MSC_VER) 81 | #pragma warning(pop) 82 | #endif 83 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_HDRHISTOGRAM_C 2 | "${CMAKE_CURRENT_LIST_DIR}/hdr_encoding.c" 3 | "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.c" 4 | "${CMAKE_CURRENT_LIST_DIR}/hdr_interval_recorder.c" 5 | "${CMAKE_CURRENT_LIST_DIR}/hdr_thread.c" 6 | "${CMAKE_CURRENT_LIST_DIR}/hdr_time.c" 7 | "${CMAKE_CURRENT_LIST_DIR}/hdr_writer_reader_phaser.c" 8 | "${CMAKE_CURRENT_LIST_DIR}/hdr_atomic.h" 9 | "${CMAKE_CURRENT_LIST_DIR}/hdr_encoding.h" 10 | "${CMAKE_CURRENT_LIST_DIR}/hdr_endian.h" 11 | "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.h" 12 | "${CMAKE_CURRENT_LIST_DIR}/hdr_interval_recorder.h" 13 | "${CMAKE_CURRENT_LIST_DIR}/hdr_tests.h" 14 | "${CMAKE_CURRENT_LIST_DIR}/hdr_thread.h" 15 | "${CMAKE_CURRENT_LIST_DIR}/hdr_time.h" 16 | "${CMAKE_CURRENT_LIST_DIR}/hdr_writer_reader_phaser.h") 17 | 18 | valkey_search_add_static_library(hdrhistogram_c "${SRCS_HDRHISTOGRAM_C}") 19 | target_include_directories(hdrhistogram_c PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 20 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_encoding.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_encoding.h 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #ifndef HDR_ENCODING_H 8 | #define HDR_ENCODING_H 9 | 10 | #include 11 | 12 | #define MAX_BYTES_LEB128 9 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | /** 19 | * Writes a int64_t value to the given buffer in LEB128 ZigZag encoded format 20 | * 21 | * @param buffer the buffer to write to 22 | * @param signed_value the value to write to the buffer 23 | * @return the number of bytes written to the buffer 24 | */ 25 | int zig_zag_encode_i64(uint8_t* buffer, int64_t signed_value); 26 | 27 | /** 28 | * Read an LEB128 ZigZag encoded long value from the given buffer 29 | * 30 | * @param buffer the buffer to read from 31 | * @param retVal out value to capture the read value 32 | * @return the number of bytes read from the buffer 33 | */ 34 | int zig_zag_decode_i64(const uint8_t* buffer, int64_t* signed_value); 35 | 36 | /** 37 | * Gets the length in bytes of base64 data, given the input size. 38 | * 39 | * @param decoded_size the size of the unencoded values. 40 | * @return the encoded size 41 | */ 42 | size_t hdr_base64_encoded_len(size_t decoded_size); 43 | 44 | /** 45 | * Encode into base64. 46 | * 47 | * @param input the data to encode 48 | * @param input_len the length of the data to encode 49 | * @param output the buffer to write the output to 50 | * @param output_len the number of bytes to write to the output 51 | */ 52 | int hdr_base64_encode( 53 | const uint8_t* input, size_t input_len, char* output, size_t output_len); 54 | 55 | /** 56 | * Gets the length in bytes of decoded base64 data, given the size of the base64 encoded 57 | * data. 58 | * 59 | * @param encoded_size the size of the encoded value. 60 | * @return the decoded size 61 | */ 62 | size_t hdr_base64_decoded_len(size_t encoded_size); 63 | 64 | /** 65 | * Decode from base64. 66 | * 67 | * @param input the base64 encoded data 68 | * @param input_len the size in bytes of the encoded data 69 | * @param output the buffer to write the decoded data to 70 | * @param output_len the number of bytes to write to the output data 71 | */ 72 | int hdr_base64_decode( 73 | const char* input, size_t input_len, uint8_t* output, size_t output_len); 74 | 75 | #ifdef __cplusplus 76 | } 77 | #endif 78 | 79 | #endif /* HDR_HISTOGRAM_HDR_ENCODING_H */ 80 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_endian.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_time.h 3 | * Released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ 4 | */ 5 | 6 | #ifndef HDR_ENDIAN_H__ 7 | #define HDR_ENDIAN_H__ 8 | 9 | #if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) 10 | 11 | # define __WINDOWS__ 12 | 13 | #endif 14 | 15 | #if defined(__linux__) || defined(__CYGWIN__) 16 | 17 | # include 18 | 19 | #elif defined(__APPLE__) 20 | 21 | # include 22 | 23 | # define htobe16(x) OSSwapHostToBigInt16(x) 24 | # define htole16(x) OSSwapHostToLittleInt16(x) 25 | # define be16toh(x) OSSwapBigToHostInt16(x) 26 | # define le16toh(x) OSSwapLittleToHostInt16(x) 27 | 28 | # define htobe32(x) OSSwapHostToBigInt32(x) 29 | # define htole32(x) OSSwapHostToLittleInt32(x) 30 | # define be32toh(x) OSSwapBigToHostInt32(x) 31 | # define le32toh(x) OSSwapLittleToHostInt32(x) 32 | 33 | # define htobe64(x) OSSwapHostToBigInt64(x) 34 | # define htole64(x) OSSwapHostToLittleInt64(x) 35 | # define be64toh(x) OSSwapBigToHostInt64(x) 36 | # define le64toh(x) OSSwapLittleToHostInt64(x) 37 | 38 | # define __BYTE_ORDER BYTE_ORDER 39 | # define __BIG_ENDIAN BIG_ENDIAN 40 | # define __LITTLE_ENDIAN LITTLE_ENDIAN 41 | # define __PDP_ENDIAN PDP_ENDIAN 42 | 43 | #elif defined(__OpenBSD__) 44 | 45 | # include 46 | 47 | #elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) 48 | 49 | # include 50 | 51 | # define be16toh(x) betoh16(x) 52 | # define le16toh(x) letoh16(x) 53 | 54 | # define be32toh(x) betoh32(x) 55 | # define le32toh(x) letoh32(x) 56 | 57 | # define be64toh(x) betoh64(x) 58 | # define le64toh(x) letoh64(x) 59 | 60 | #elif defined(__WINDOWS__) 61 | 62 | # include 63 | 64 | # if BYTE_ORDER == LITTLE_ENDIAN 65 | 66 | # define htobe16(x) htons(x) 67 | # define htole16(x) (x) 68 | # define be16toh(x) ntohs(x) 69 | # define le16toh(x) (x) 70 | 71 | # define htobe32(x) htonl(x) 72 | # define htole32(x) (x) 73 | # define be32toh(x) ntohl(x) 74 | # define le32toh(x) (x) 75 | 76 | # define htobe64(x) htonll(x) 77 | # define htole64(x) (x) 78 | # define be64toh(x) ntohll(x) 79 | # define le64toh(x) (x) 80 | 81 | # elif BYTE_ORDER == BIG_ENDIAN 82 | 83 | /* that would be xbox 360 */ 84 | # define htobe16(x) (x) 85 | # define htole16(x) __builtin_bswap16(x) 86 | # define be16toh(x) (x) 87 | # define le16toh(x) __builtin_bswap16(x) 88 | 89 | # define htobe32(x) (x) 90 | # define htole32(x) __builtin_bswap32(x) 91 | # define be32toh(x) (x) 92 | # define le32toh(x) __builtin_bswap32(x) 93 | 94 | # define htobe64(x) (x) 95 | # define htole64(x) __builtin_bswap64(x) 96 | # define be64toh(x) (x) 97 | # define le64toh(x) __builtin_bswap64(x) 98 | 99 | # else 100 | 101 | # error byte order not supported 102 | 103 | # endif 104 | 105 | # define __BYTE_ORDER BYTE_ORDER 106 | # define __BIG_ENDIAN BIG_ENDIAN 107 | # define __LITTLE_ENDIAN LITTLE_ENDIAN 108 | # define __PDP_ENDIAN PDP_ENDIAN 109 | 110 | #else 111 | 112 | # error platform not supported 113 | 114 | #endif 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_interval_recorder.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_interval_recorder.h 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #ifndef HDR_INTERVAL_RECORDER_H 8 | #define HDR_INTERVAL_RECORDER_H 1 9 | 10 | #include "hdr_writer_reader_phaser.h" 11 | #include "hdr_histogram.h" 12 | 13 | HDR_ALIGN_PREFIX(8) 14 | struct hdr_interval_recorder 15 | { 16 | struct hdr_histogram* active; 17 | struct hdr_histogram* inactive; 18 | struct hdr_writer_reader_phaser phaser; 19 | } 20 | HDR_ALIGN_SUFFIX(8); 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | int hdr_interval_recorder_init(struct hdr_interval_recorder* r); 27 | 28 | int hdr_interval_recorder_init_all( 29 | struct hdr_interval_recorder* r, 30 | int64_t lowest_trackable_value, 31 | int64_t highest_trackable_value, 32 | int significant_figures); 33 | 34 | void hdr_interval_recorder_destroy(struct hdr_interval_recorder* r); 35 | 36 | int64_t hdr_interval_recorder_record_value( 37 | struct hdr_interval_recorder* r, 38 | int64_t value 39 | ); 40 | 41 | int64_t hdr_interval_recorder_record_values( 42 | struct hdr_interval_recorder* r, 43 | int64_t value, 44 | int64_t count 45 | ); 46 | 47 | int64_t hdr_interval_recorder_record_corrected_value( 48 | struct hdr_interval_recorder* r, 49 | int64_t value, 50 | int64_t expected_interval 51 | ); 52 | 53 | int64_t hdr_interval_recorder_record_corrected_values( 54 | struct hdr_interval_recorder* r, 55 | int64_t value, 56 | int64_t count, 57 | int64_t expected_interval 58 | ); 59 | 60 | int64_t hdr_interval_recorder_record_value_atomic( 61 | struct hdr_interval_recorder* r, 62 | int64_t value 63 | ); 64 | 65 | int64_t hdr_interval_recorder_record_values_atomic( 66 | struct hdr_interval_recorder* r, 67 | int64_t value, 68 | int64_t count 69 | ); 70 | 71 | int64_t hdr_interval_recorder_record_corrected_value_atomic( 72 | struct hdr_interval_recorder* r, 73 | int64_t value, 74 | int64_t expected_interval 75 | ); 76 | 77 | int64_t hdr_interval_recorder_record_corrected_values_atomic( 78 | struct hdr_interval_recorder* r, 79 | int64_t value, 80 | int64_t count, 81 | int64_t expected_interval 82 | ); 83 | 84 | /** 85 | * The is generally the preferred approach for recycling histograms through 86 | * the recorder as it is safe when used from callers in multiple threads and 87 | * the returned histogram won't automatically become active without being 88 | * passed back into this method. 89 | * 90 | * @param r 'this' recorder 91 | * @param histogram_to_recycle 92 | * @return the histogram that was previous being recorded to. 93 | */ 94 | struct hdr_histogram* hdr_interval_recorder_sample_and_recycle( 95 | struct hdr_interval_recorder* r, 96 | struct hdr_histogram* histogram_to_recycle); 97 | 98 | /** 99 | * @deprecated Prefer hdr_interval_recorder_sample_and_recycle 100 | * @param r 'this' recorder 101 | * @return the histogram that was previous being recorded to. 102 | */ 103 | struct hdr_histogram* hdr_interval_recorder_sample(struct hdr_interval_recorder* r); 104 | 105 | #ifdef __cplusplus 106 | } 107 | #endif 108 | 109 | #endif 110 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_tests.h: -------------------------------------------------------------------------------- 1 | #ifndef HDR_TESTS_H 2 | #define HDR_TESTS_H 3 | 4 | /* These are functions used in tests and are not intended for normal usage. */ 5 | 6 | #include "hdr_histogram.h" 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | int32_t counts_index_for(const struct hdr_histogram* h, int64_t value); 13 | int hdr_encode_compressed(struct hdr_histogram* h, uint8_t** compressed_histogram, size_t* compressed_len); 14 | int hdr_decode_compressed(uint8_t* buffer, size_t length, struct hdr_histogram** histogram); 15 | void hdr_base64_decode_block(const char* input, uint8_t* output); 16 | void hdr_base64_encode_block(const uint8_t* input, char* output); 17 | 18 | #ifdef __cplusplus 19 | } 20 | #endif 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_thread.c: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_thread.c 3 | * Written by Philip Orwig and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #include 8 | #include "hdr_thread.h" 9 | 10 | struct hdr_mutex* hdr_mutex_alloc(void) 11 | { 12 | return malloc(sizeof(hdr_mutex)); 13 | } 14 | 15 | void hdr_mutex_free(struct hdr_mutex* mutex) 16 | { 17 | free(mutex); 18 | } 19 | 20 | #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) 21 | 22 | #if !defined(WIN32_LEAN_AND_MEAN) 23 | #define WIN32_LEAN_AND_MEAN 24 | #endif 25 | 26 | #include 27 | #include 28 | 29 | int hdr_mutex_init(struct hdr_mutex* mutex) 30 | { 31 | InitializeCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); 32 | return 0; 33 | } 34 | 35 | void hdr_mutex_destroy(struct hdr_mutex* mutex) 36 | { 37 | DeleteCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); 38 | } 39 | 40 | void hdr_mutex_lock(struct hdr_mutex* mutex) 41 | { 42 | EnterCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); 43 | } 44 | 45 | void hdr_mutex_unlock(struct hdr_mutex* mutex) 46 | { 47 | LeaveCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); 48 | } 49 | 50 | void hdr_yield() 51 | { 52 | Sleep(0); 53 | } 54 | 55 | int hdr_usleep(unsigned int useconds) 56 | { 57 | struct timeval tv; 58 | 59 | tv.tv_sec = (long)useconds / 1000000; 60 | tv.tv_usec = useconds % 1000000; 61 | select(0, NULL, NULL, NULL, &tv); 62 | 63 | return 0; 64 | } 65 | 66 | 67 | #else 68 | #include 69 | #include 70 | 71 | int hdr_mutex_init(struct hdr_mutex* mutex) 72 | { 73 | return pthread_mutex_init(&mutex->_mutex, NULL); 74 | } 75 | 76 | void hdr_mutex_destroy(struct hdr_mutex* mutex) 77 | { 78 | pthread_mutex_destroy(&mutex->_mutex); 79 | } 80 | 81 | void hdr_mutex_lock(struct hdr_mutex* mutex) 82 | { 83 | pthread_mutex_lock(&mutex->_mutex); 84 | } 85 | 86 | void hdr_mutex_unlock(struct hdr_mutex* mutex) 87 | { 88 | pthread_mutex_unlock(&mutex->_mutex); 89 | } 90 | 91 | void hdr_yield() 92 | { 93 | sched_yield(); 94 | } 95 | 96 | int hdr_usleep(unsigned int useconds) 97 | { 98 | return usleep(useconds); 99 | } 100 | 101 | 102 | #endif 103 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_thread.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_thread.h 3 | * Written by Philip Orwig and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #ifndef HDR_THREAD_H__ 8 | #define HDR_THREAD_H__ 9 | 10 | #include 11 | 12 | #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) 13 | 14 | 15 | #define HDR_ALIGN_PREFIX(alignment) __declspec( align(alignment) ) 16 | #define HDR_ALIGN_SUFFIX(alignment) 17 | 18 | typedef struct hdr_mutex 19 | { 20 | uint8_t _critical_section[40]; 21 | } hdr_mutex; 22 | 23 | #else 24 | 25 | #include 26 | 27 | #define HDR_ALIGN_PREFIX(alignment) 28 | #define HDR_ALIGN_SUFFIX(alignment) __attribute__((aligned(alignment))) 29 | 30 | typedef struct hdr_mutex 31 | { 32 | pthread_mutex_t _mutex; 33 | } hdr_mutex; 34 | #endif 35 | 36 | #ifdef __cplusplus 37 | extern "C" { 38 | #endif 39 | 40 | struct hdr_mutex* hdr_mutex_alloc(void); 41 | void hdr_mutex_free(struct hdr_mutex*); 42 | 43 | int hdr_mutex_init(struct hdr_mutex* mutex); 44 | void hdr_mutex_destroy(struct hdr_mutex* mutex); 45 | 46 | void hdr_mutex_lock(struct hdr_mutex* mutex); 47 | void hdr_mutex_unlock(struct hdr_mutex* mutex); 48 | 49 | void hdr_yield(void); 50 | int hdr_usleep(unsigned int useconds); 51 | 52 | #ifdef __cplusplus 53 | } 54 | #endif 55 | #endif 56 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_time.c: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_time.h 3 | * Written by Michael Barker and Philip Orwig and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #include "hdr_time.h" 8 | 9 | #if defined(_WIN32) || defined(_WIN64) 10 | 11 | #if !defined(WIN32_LEAN_AND_MEAN) 12 | #define WIN32_LEAN_AND_MEAN 13 | #endif 14 | 15 | #include 16 | 17 | static int s_clockPeriodSet = 0; 18 | static double s_clockPeriod = 1.0; 19 | 20 | void hdr_gettime(hdr_timespec* t) 21 | { 22 | LARGE_INTEGER num; 23 | /* if this is distasteful, we can add in an hdr_time_init() */ 24 | if (!s_clockPeriodSet) 25 | { 26 | QueryPerformanceFrequency(&num); 27 | s_clockPeriod = 1.0 / (double) num.QuadPart; 28 | s_clockPeriodSet = 1; 29 | } 30 | 31 | QueryPerformanceCounter(&num); 32 | double seconds = num.QuadPart * s_clockPeriod; 33 | double integral; 34 | double remainder = modf(seconds, &integral); 35 | 36 | t->tv_sec = (long) integral; 37 | t->tv_nsec = (long) (remainder * 1000000000); 38 | } 39 | 40 | #elif defined(__APPLE__) 41 | 42 | #include 43 | #include 44 | 45 | 46 | void hdr_gettime(hdr_timespec* ts) 47 | { 48 | clock_serv_t cclock; 49 | mach_timespec_t mts; 50 | host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); 51 | clock_get_time(cclock, &mts); 52 | mach_port_deallocate(mach_task_self(), cclock); 53 | ts->tv_sec = mts.tv_sec; 54 | ts->tv_nsec = mts.tv_nsec; 55 | } 56 | 57 | 58 | void hdr_getnow(hdr_timespec* ts) 59 | { 60 | hdr_gettime(ts); 61 | } 62 | 63 | #elif defined(__linux__) || defined(__CYGWIN__) 64 | 65 | 66 | void hdr_gettime(hdr_timespec* t) 67 | { 68 | clock_gettime(CLOCK_MONOTONIC, (struct timespec*)t); 69 | } 70 | 71 | void hdr_getnow(hdr_timespec* t) 72 | { 73 | clock_gettime(CLOCK_REALTIME, (struct timespec*)t); 74 | } 75 | 76 | #else 77 | 78 | #warning "Platform not supported\n" 79 | 80 | #endif 81 | 82 | double hdr_timespec_as_double(const hdr_timespec* t) 83 | { 84 | double d = t->tv_sec; 85 | return d + (t->tv_nsec / 1000000000.0); 86 | } 87 | 88 | void hdr_timespec_from_double(hdr_timespec* t, double value) 89 | { 90 | int seconds = (int) value; 91 | int milliseconds = (int) round((value - seconds) * 1000); 92 | 93 | t->tv_sec = seconds; 94 | t->tv_nsec = milliseconds * 1000000; 95 | } 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_time.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_time.h 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #ifndef HDR_TIME_H__ 8 | #define HDR_TIME_H__ 9 | 10 | #include 11 | #include 12 | 13 | #if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) 14 | 15 | typedef struct hdr_timespec 16 | { 17 | long tv_sec; 18 | long tv_nsec; 19 | } hdr_timespec; 20 | 21 | #else 22 | 23 | typedef struct timespec hdr_timespec; 24 | 25 | #endif 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #if defined(_MSC_VER) 32 | void hdr_gettime(hdr_timespec* t); 33 | #else 34 | void hdr_gettime(hdr_timespec* t); 35 | #endif 36 | 37 | void hdr_getnow(hdr_timespec* t); 38 | 39 | double hdr_timespec_as_double(const hdr_timespec* t); 40 | 41 | /* Assumes only millisecond accuracy. */ 42 | void hdr_timespec_from_double(hdr_timespec* t, double value); 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif 49 | 50 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_writer_reader_phaser.c: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_writer_reader_phaser.h 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "hdr_atomic.h" 12 | #include "hdr_thread.h" 13 | 14 | #include "hdr_writer_reader_phaser.h" 15 | 16 | static int64_t _hdr_phaser_get_epoch(int64_t* field) 17 | { 18 | return hdr_atomic_load_64(field); 19 | } 20 | 21 | static void _hdr_phaser_set_epoch(int64_t* field, int64_t val) 22 | { 23 | hdr_atomic_store_64(field, val); 24 | } 25 | 26 | static int64_t _hdr_phaser_reset_epoch(int64_t* field, int64_t initial_value) 27 | { 28 | return hdr_atomic_exchange_64(field, initial_value); 29 | } 30 | 31 | int hdr_writer_reader_phaser_init(struct hdr_writer_reader_phaser* p) 32 | { 33 | int rc; 34 | if (NULL == p) 35 | { 36 | return EINVAL; 37 | } 38 | 39 | p->start_epoch = 0; 40 | p->even_end_epoch = 0; 41 | p->odd_end_epoch = INT64_MIN; 42 | p->reader_mutex = hdr_mutex_alloc(); 43 | 44 | if (!p->reader_mutex) 45 | { 46 | return ENOMEM; 47 | } 48 | 49 | rc = hdr_mutex_init(p->reader_mutex); 50 | if (0 != rc) 51 | { 52 | return rc; 53 | } 54 | 55 | /* TODO: Should I fence here. */ 56 | 57 | return 0; 58 | } 59 | 60 | void hdr_writer_reader_phaser_destroy(struct hdr_writer_reader_phaser* p) 61 | { 62 | hdr_mutex_destroy(p->reader_mutex); 63 | hdr_mutex_free(p->reader_mutex); 64 | } 65 | 66 | int64_t hdr_phaser_writer_enter(struct hdr_writer_reader_phaser* p) 67 | { 68 | return hdr_atomic_add_fetch_64(&p->start_epoch, 1); 69 | } 70 | 71 | void hdr_phaser_writer_exit( 72 | struct hdr_writer_reader_phaser* p, int64_t critical_value_at_enter) 73 | { 74 | int64_t* end_epoch = 75 | (critical_value_at_enter < 0) ? &p->odd_end_epoch : &p->even_end_epoch; 76 | hdr_atomic_add_fetch_64(end_epoch, 1); 77 | } 78 | 79 | void hdr_phaser_reader_lock(struct hdr_writer_reader_phaser* p) 80 | { 81 | hdr_mutex_lock(p->reader_mutex); 82 | } 83 | 84 | void hdr_phaser_reader_unlock(struct hdr_writer_reader_phaser* p) 85 | { 86 | hdr_mutex_unlock(p->reader_mutex); 87 | } 88 | 89 | void hdr_phaser_flip_phase( 90 | struct hdr_writer_reader_phaser* p, int64_t sleep_time_ns) 91 | { 92 | bool caught_up; 93 | int64_t start_value_at_flip; 94 | /* TODO: is_held_by_current_thread */ 95 | unsigned int sleep_time_us = sleep_time_ns < 1000000000 ? (unsigned int) (sleep_time_ns / 1000) : 1000000; 96 | 97 | int64_t start_epoch = _hdr_phaser_get_epoch(&p->start_epoch); 98 | 99 | bool next_phase_is_even = (start_epoch < 0); 100 | 101 | /* Clear currently used phase end epoch.*/ 102 | int64_t initial_start_value; 103 | if (next_phase_is_even) 104 | { 105 | initial_start_value = 0; 106 | _hdr_phaser_set_epoch(&p->even_end_epoch, initial_start_value); 107 | } 108 | else 109 | { 110 | initial_start_value = INT64_MIN; 111 | _hdr_phaser_set_epoch(&p->odd_end_epoch, initial_start_value); 112 | } 113 | 114 | /* Reset start value, indicating new phase.*/ 115 | start_value_at_flip = _hdr_phaser_reset_epoch(&p->start_epoch, initial_start_value); 116 | 117 | do 118 | { 119 | int64_t* end_epoch = 120 | next_phase_is_even ? &p->odd_end_epoch : &p->even_end_epoch; 121 | 122 | caught_up = _hdr_phaser_get_epoch(end_epoch) == start_value_at_flip; 123 | 124 | if (!caught_up) 125 | { 126 | if (sleep_time_us <= 0) 127 | { 128 | hdr_yield(); 129 | } 130 | else 131 | { 132 | hdr_usleep(sleep_time_us); 133 | } 134 | } 135 | } 136 | while (!caught_up); 137 | } 138 | -------------------------------------------------------------------------------- /third_party/hdrhistogram_c/src/hdr_writer_reader_phaser.h: -------------------------------------------------------------------------------- 1 | /** 2 | * hdr_writer_reader_phaser.h 3 | * Written by Michael Barker and released to the public domain, 4 | * as explained at http://creativecommons.org/publicdomain/zero/1.0/ 5 | */ 6 | 7 | #ifndef HDR_WRITER_READER_PHASER_H 8 | #define HDR_WRITER_READER_PHASER_H 1 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "hdr_thread.h" 16 | 17 | HDR_ALIGN_PREFIX(8) 18 | struct hdr_writer_reader_phaser 19 | { 20 | int64_t start_epoch; 21 | int64_t even_end_epoch; 22 | int64_t odd_end_epoch; 23 | hdr_mutex* reader_mutex; 24 | } 25 | HDR_ALIGN_SUFFIX(8); 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | int hdr_writer_reader_phaser_init(struct hdr_writer_reader_phaser* p); 32 | 33 | void hdr_writer_reader_phaser_destroy(struct hdr_writer_reader_phaser* p); 34 | 35 | int64_t hdr_phaser_writer_enter(struct hdr_writer_reader_phaser* p); 36 | 37 | void hdr_phaser_writer_exit( 38 | struct hdr_writer_reader_phaser* p, int64_t critical_value_at_enter); 39 | 40 | void hdr_phaser_reader_lock(struct hdr_writer_reader_phaser* p); 41 | 42 | void hdr_phaser_reader_unlock(struct hdr_writer_reader_phaser* p); 43 | 44 | void hdr_phaser_flip_phase( 45 | struct hdr_writer_reader_phaser* p, int64_t sleep_time_ns); 46 | 47 | #ifdef __cplusplus 48 | } 49 | #endif 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /third_party/hnswlib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_HNSWLIB_VMSDK 2 | ${CMAKE_CURRENT_LIST_DIR}/bruteforce.h 3 | ${CMAKE_CURRENT_LIST_DIR}/hnswalg.h 4 | ${CMAKE_CURRENT_LIST_DIR}/hnswlib.h 5 | ${CMAKE_CURRENT_LIST_DIR}/space_ip.h 6 | ${CMAKE_CURRENT_LIST_DIR}/space_l2.h 7 | ${CMAKE_CURRENT_LIST_DIR}/stop_condition.h 8 | ${CMAKE_CURRENT_LIST_DIR}/visited_list_pool.h) 9 | 10 | valkey_search_create_proto_library("third_party/hnswlib/index.proto" 11 | "index_cc_proto") 12 | message(STATUS "Created proto library index_cc_proto") 13 | 14 | add_library(hnswlib_vmsdk INTERFACE ${SRCS_HNSWLIB_VMSDK}) 15 | target_include_directories(hnswlib_vmsdk INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 16 | target_link_libraries(hnswlib_vmsdk INTERFACE iostream) 17 | target_link_libraries(hnswlib_vmsdk INTERFACE simsimd) 18 | target_link_libraries(hnswlib_vmsdk INTERFACE memory_allocation_overrides) 19 | target_link_libraries(hnswlib_vmsdk INTERFACE status_macros) 20 | target_compile_definitions(hnswlib_vmsdk 21 | INTERFACE VMSDK_ENABLE_MEMORY_ALLOCATION_OVERRIDES) 22 | target_link_libraries(hnswlib_vmsdk INTERFACE index_cc_proto) 23 | 24 | set(SRCS_SIMSIMD ${CMAKE_CURRENT_LIST_DIR}/simsimd.h) 25 | 26 | add_library(simsimd INTERFACE ${SRCS_SIMSIMD}) 27 | target_include_directories(simsimd INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 28 | 29 | set(SRCS_IOSTREAM ${CMAKE_CURRENT_LIST_DIR}/iostream.h) 30 | 31 | add_library(iostream INTERFACE ${SRCS_IOSTREAM}) 32 | target_include_directories(iostream INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 33 | -------------------------------------------------------------------------------- /third_party/hnswlib/OWNERS: -------------------------------------------------------------------------------- 1 | jkmurphy 2 | yairg 3 | -------------------------------------------------------------------------------- /third_party/hnswlib/index.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package hnswlib.data_model; 4 | 5 | 6 | message BruteForceIndexHeader { 7 | uint64 max_elements = 1; 8 | uint64 size_per_element = 2; 9 | uint64 curr_element_count = 3; 10 | } 11 | 12 | message HNSWIndexHeader { 13 | uint64 offset_level_0 = 1; 14 | uint64 max_elements = 2; 15 | uint64 curr_element_count = 3; 16 | uint64 serialize_size_data_per_element = 4; 17 | uint64 label_offset = 5; 18 | uint64 offset_data = 6; 19 | int32 max_level = 7; 20 | uint32 enterpoint_node = 8; 21 | uint64 max_M = 9; 22 | uint64 max_M_0 = 10; 23 | uint64 M = 11; 24 | double mult = 12; 25 | uint64 ef_construction = 13; 26 | } -------------------------------------------------------------------------------- /third_party/hnswlib/iostream.h: -------------------------------------------------------------------------------- 1 | #ifndef THIRD_PARTY_HNSWLIB_IOSTREAM_H_ 2 | #define THIRD_PARTY_HNSWLIB_IOSTREAM_H_ 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "absl/status/status.h" 15 | #include "absl/status/statusor.h" 16 | 17 | #ifdef VMSDK_ENABLE_MEMORY_ALLOCATION_OVERRIDES 18 | #include "vmsdk/src/memory_allocation_overrides.h" // IWYU pragma: keep 19 | #endif 20 | 21 | namespace hnswlib { 22 | 23 | class VectorTracker { 24 | public: 25 | virtual ~VectorTracker() = default; 26 | virtual char *TrackVector(uint64_t internal_id, char *vector, size_t len) = 0; 27 | }; 28 | 29 | class InputStream { 30 | public: 31 | virtual ~InputStream() = default; 32 | virtual absl::StatusOr> LoadChunk() = 0; 33 | }; 34 | 35 | class OutputStream { 36 | public: 37 | virtual ~OutputStream() = default; 38 | virtual absl::Status SaveChunk(const char *data, size_t len) = 0; 39 | }; 40 | 41 | } // namespace hnswlib 42 | 43 | #endif // THIRD_PARTY_HNSWLIB_IOSTREAM_H_ 44 | -------------------------------------------------------------------------------- /third_party/hnswlib/simsimd.h: -------------------------------------------------------------------------------- 1 | #ifndef THIRD_PARTY_HNSWLIB_SIMSIMD_H_ 2 | #define THIRD_PARTY_HNSWLIB_SIMSIMD_H_ 3 | 4 | #include 5 | 6 | #include "third_party/simsimd/c/lib.c" 7 | 8 | #include "third_party/simsimd/include/simsimd/simsimd.h" 9 | #include "third_party/simsimd/include/simsimd/types.h" 10 | 11 | 12 | inline float InnerProductDistanceSimsimd(const void *pVect1, const void *pVect2, 13 | const void *qty_ptr) { 14 | simsimd_size_t dim = *static_cast(qty_ptr); 15 | const simsimd_f32_t *vec1 = static_cast(pVect1); 16 | const simsimd_f32_t *vec2 = static_cast(pVect2); 17 | simsimd_distance_t distance; 18 | simsimd_dot_f32(vec1, vec2, dim, &distance); 19 | return 1.0f - distance; 20 | } 21 | 22 | inline float L2SqrSimsimd(const void *pVect1, const void *pVect2, 23 | const void *qty_ptr) { 24 | simsimd_size_t dim = *static_cast(qty_ptr); 25 | const simsimd_f32_t *vec1 = static_cast(pVect1); 26 | const simsimd_f32_t *vec2 = static_cast(pVect2); 27 | simsimd_distance_t distance; 28 | simsimd_l2sq_f32(vec1, vec2, dim, &distance); 29 | return distance; 30 | } 31 | 32 | #endif // THIRD_PARTY_HNSWLIB_SIMSIMD_H_ 33 | -------------------------------------------------------------------------------- /third_party/hnswlib/visited_list_pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace hnswlib { 8 | typedef unsigned short int vl_type; 9 | 10 | class VisitedList { 11 | public: 12 | vl_type curV; 13 | vl_type *mass; 14 | unsigned int numelements; 15 | 16 | VisitedList(int numelements1) { 17 | curV = -1; 18 | numelements = numelements1; 19 | mass = new vl_type[numelements]; 20 | } 21 | 22 | void reset() { 23 | curV++; 24 | if (curV == 0) { 25 | memset(mass, 0, sizeof(vl_type) * numelements); 26 | curV++; 27 | } 28 | } 29 | 30 | ~VisitedList() { delete[] mass; } 31 | }; 32 | /////////////////////////////////////////////////////////// 33 | // 34 | // Class for multi-threaded pool-management of VisitedLists 35 | // 36 | ///////////////////////////////////////////////////////// 37 | 38 | class VisitedListPool { 39 | std::deque pool; 40 | std::mutex poolguard; 41 | int numelements; 42 | 43 | public: 44 | VisitedListPool(int initmaxpools, int numelements1) { 45 | numelements = numelements1; 46 | for (int i = 0; i < initmaxpools; i++) 47 | pool.push_front(new VisitedList(numelements)); 48 | } 49 | 50 | VisitedList *getFreeVisitedList() { 51 | VisitedList *rez; 52 | { 53 | std::unique_lock lock(poolguard); 54 | if (pool.size() > 0) { 55 | rez = pool.front(); 56 | pool.pop_front(); 57 | } else { 58 | rez = new VisitedList(numelements); 59 | } 60 | } 61 | rez->reset(); 62 | return rez; 63 | } 64 | 65 | void releaseVisitedList(VisitedList *vl) { 66 | std::unique_lock lock(poolguard); 67 | pool.push_front(vl); 68 | } 69 | 70 | ~VisitedListPool() { 71 | while (pool.size()) { 72 | VisitedList *rez = pool.front(); 73 | pool.pop_front(); 74 | delete rez; 75 | } 76 | } 77 | }; 78 | } // namespace hnswlib 79 | -------------------------------------------------------------------------------- /third_party/simsimd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(c) 2 | -------------------------------------------------------------------------------- /third_party/simsimd/METADATA: -------------------------------------------------------------------------------- 1 | # go/google3metadata 2 | # proto-file: devtools/metadata/metadata.proto 3 | # proto-message: MetaData 4 | 5 | name: "simsimd" 6 | description: 7 | "Simsimd provides SIMD-accelerated vector similarity, distance functions, and" 8 | " dot products." 9 | 10 | third_party { 11 | type: PACKAGE 12 | identifier: { 13 | type: "Git" 14 | value: "https://github.com/ashvardanian/SimSIMD" 15 | primary_source: true 16 | version: "5.0.1" 17 | } 18 | last_upgrade_date { year: 2024 month: 8 day: 28 } 19 | } 20 | presubmit: { 21 | check_tests: { 22 | project: "third_party.simsimd" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /third_party/simsimd/OWNERS: -------------------------------------------------------------------------------- 1 | sumish 2 | yairg 3 | jkmurphy -------------------------------------------------------------------------------- /third_party/simsimd/VERSION: -------------------------------------------------------------------------------- 1 | 5.0.1 -------------------------------------------------------------------------------- /third_party/simsimd/bluze.textproto: -------------------------------------------------------------------------------- 1 | # proto-file: devtools/blueprint/blueprint_file.proto 2 | # proto-message: BlueprintFile 3 | # DO NOT EDIT! Regenerate the contents by running go/bluze after changing any BUILD file or the Blueprint. 4 | # Override the default values in third_party.simsimd.blueprint instead. 5 | 6 | buildable_unit: { 7 | name: "third_party.simsimd" 8 | build_pattern: "//third_party/simsimd/..." 9 | test_pattern: "//third_party/simsimd/..." 10 | test_tag_filter: "-nofastbuild" 11 | build_tag_filter: "-nofastbuild" 12 | enable_coverage: true 13 | enable_presubmit: true 14 | enable_continuous_build: true 15 | continuous_build_email: { 16 | build_cop_email_id: "cloud-redis-team+buildgardener@google.com" 17 | } 18 | enable_release: false 19 | } 20 | buildable_unit: { 21 | name: "third_party.simsimd.opt" 22 | test_pattern: "//third_party/simsimd/..." 23 | test_tag_filter: "-noopt" 24 | build_flag: "--compilation_mode=opt" 25 | enable_coverage: false 26 | enable_presubmit: false 27 | enable_continuous_build: false 28 | continuous_build_email: { 29 | build_cop_email_id: "cloud-redis-team+buildgardener@google.com" 30 | } 31 | enable_release: false 32 | [tap.tap_settings]: { 33 | on_demand: true 34 | on_demand_frequency: EVERY_4_HOURS 35 | } 36 | } 37 | continuous_tests: { 38 | name: "third_party.simsimd" 39 | buildable_unit_name: "third_party.simsimd" 40 | } 41 | continuous_tests: { 42 | name: "third_party.simsimd.opt" 43 | buildable_unit_name: "third_party.simsimd.opt" 44 | } 45 | -------------------------------------------------------------------------------- /third_party/simsimd/c/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_LIB ${CMAKE_CURRENT_LIST_DIR}/lib.c) 2 | 3 | valkey_search_add_static_library(simsimd_c "${SRCS_LIB}") 4 | target_include_directories(simsimd_c PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 5 | target_link_libraries(simsimd_c PUBLIC simsimd) 6 | -------------------------------------------------------------------------------- /third_party/simsimd/include/simsimd/geospatial.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file geospatial.h 3 | * @brief SIMD-accelerated Geo-Spatial distance functions. 4 | * @author Ash Vardanian 5 | * @date July 1, 2023 6 | * 7 | * Contains: 8 | * - Haversine (Great Circle) distance 9 | * - TODO: Vincenty's distance function for Oblate Spheroid Geodesics 10 | * 11 | * For datatypes: 12 | * - 32-bit IEEE-754 floating point 13 | * - 64-bit IEEE-754 floating point 14 | * 15 | * For hardware architectures: 16 | * - Arm (NEON, SVE) 17 | * - x86 (AVX512) 18 | * 19 | * x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/ 20 | * Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/ 21 | * Oblate Spheroid Geodesic: https://mathworld.wolfram.com/OblateSpheroidGeodesic.html 22 | * Staging experiments: https://github.com/ashvardanian/HaversineSimSIMD 23 | */ 24 | #ifndef SIMSIMD_GEOSPATIAL_H 25 | #define SIMSIMD_GEOSPATIAL_H 26 | 27 | #include "types.h" 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /third_party/simsimd/third_party.simsimd.blueprint: -------------------------------------------------------------------------------- 1 | include "devtools/blueprint/bluze/public/bluze.ncl"; 2 | include bytes "third_party/simsimd/bluze.textproto" as textproto; 3 | 4 | // See go/bluze/guide before editing. To check the generated final blueprint run 5 | // rncl third_party/simsimd/third_party.simsimd.blueprint printproto blueprint_file 6 | 7 | blueprint_file = ::bluze::BlueprintFile( 8 | textproto, 9 | 10 | project_name = "third_party.simsimd", 11 | teams_product_id = 12217812178, 12 | tech_lead = ["sumish", "yairg", "jkmurphy"], 13 | dev_mailing_list = "cloud-redis-team@google.com", 14 | mdb_groups = ["third-party-tap-forge"], 15 | buganizer_component_ids = [1288886], 16 | metadata_path = "//depot/google3/third_party/simsimd/METADATA", 17 | 18 | // Customize your blueprint here: go/blueprint/howto-write. 19 | ); 20 | -------------------------------------------------------------------------------- /vmsdk/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(src) 2 | add_subdirectory(testing) 3 | -------------------------------------------------------------------------------- /vmsdk/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024-present, VMSDK contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /vmsdk/README.md: -------------------------------------------------------------------------------- 1 | # VMSDK++ 2 | 3 | Valkey Module SDK++ is a C++ library that provides common functionalities for 4 | developing Valkey modules. -------------------------------------------------------------------------------- /vmsdk/src/blocked_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VMSDK_SRC_BLOCKED_CLIENT_H_ 31 | #define VMSDK_SRC_BLOCKED_CLIENT_H_ 32 | 33 | #include 34 | 35 | #include "absl/container/flat_hash_map.h" 36 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 37 | 38 | namespace vmsdk { 39 | 40 | class BlockedClient { 41 | public: 42 | BlockedClient(RedisModuleCtx *ctx, bool handle_duplication); 43 | BlockedClient(RedisModuleCtx *ctx, 44 | RedisModuleCmdFunc reply_callback = nullptr, 45 | RedisModuleCmdFunc timeout_callback = nullptr, 46 | void (*free_privdata)(RedisModuleCtx *, void *) = nullptr, 47 | long long timeout_ms = 0); 48 | BlockedClient(BlockedClient &&other) noexcept 49 | : blocked_client_(std::exchange(other.blocked_client_, nullptr)), 50 | private_data_(std::exchange(other.private_data_, nullptr)), 51 | tracked_client_id_(std::exchange(other.tracked_client_id_, 0)), 52 | time_measurement_ongoing_( 53 | std::exchange(other.time_measurement_ongoing_, false)) {} 54 | 55 | BlockedClient &operator=(BlockedClient &&other) noexcept; 56 | 57 | BlockedClient(const BlockedClient &other) = delete; 58 | BlockedClient &operator=(const BlockedClient &other) = delete; 59 | 60 | operator RedisModuleBlockedClient *() const { return blocked_client_; } 61 | void SetReplyPrivateData(void *private_data); 62 | void UnblockClient(); 63 | void MeasureTimeStart(); 64 | void MeasureTimeEnd(); 65 | ~BlockedClient() { UnblockClient(); } 66 | 67 | private: 68 | RedisModuleBlockedClient *blocked_client_{nullptr}; 69 | void *private_data_{nullptr}; 70 | bool time_measurement_ongoing_{false}; 71 | unsigned long long tracked_client_id_{0}; 72 | }; 73 | 74 | struct BlockedClientEntry { 75 | size_t cnt{0}; 76 | RedisModuleBlockedClient *blocked_client{nullptr}; 77 | }; 78 | absl::flat_hash_map & 79 | TrackedBlockedClients(); 80 | } // namespace vmsdk 81 | #endif // VMSDK_SRC_BLOCKED_CLIENT_H_ 82 | -------------------------------------------------------------------------------- /vmsdk/src/concurrency.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VMSDK_SRC_CONCURRENCY_H_ 31 | #define VMSDK_SRC_CONCURRENCY_H_ 32 | 33 | #include 34 | #include 35 | #include 36 | 37 | namespace vmsdk { 38 | 39 | // Returns the number of physical CPU cores. 40 | size_t GetPhysicalCPUCoresCount(); 41 | 42 | namespace helper { 43 | 44 | // Parses /proc/cpuinfo content and extracts the number of physical cores. 45 | size_t ParseCPUInfo(std::istream& cpuinfo); 46 | 47 | // Parses the output of the command `lscpu` and return the number of physical 48 | // cores. 49 | size_t ParseLscpuOutput(const std::string& lscpu_output); 50 | 51 | // Extracts an integer value from a formatted key-value string. 52 | int ExtractInteger(const std::string& line); 53 | 54 | } // namespace helper 55 | } // namespace vmsdk 56 | 57 | #endif // VMSDK_SRC_CONCURRENCY_H_ 58 | -------------------------------------------------------------------------------- /vmsdk/src/memory_allocation.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include "vmsdk/src/memory_allocation.h" 31 | 32 | #include 33 | 34 | #include 35 | #include 36 | 37 | namespace vmsdk { 38 | 39 | // Use the standard system allocator by default. Note that this is required 40 | // since any allocation done before Redis module initialization (namely global 41 | // static constructors that do heap allocation, which are run on dl_open) cannot 42 | // invoke Redis modules api since the associated C function pointers are only 43 | // initialized as part of the module initialization process. Refer 44 | // https://redis.com/blog/using-the-redis-allocator-in-rust for more details. 45 | // 46 | // We use a combination of a thread local static variable and a global atomic 47 | // variable to perform the switch to the new allocator. The global is only 48 | // accessed during the initial loading phase, and once we switch allocators the 49 | // thread local variable is exclusively used. This should guarantee that the 50 | // switch is done atomically while not having performance impact during steady 51 | // state. 52 | thread_local static bool thread_using_valkey_module_alloc = false; 53 | static std::atomic use_valkey_module_alloc_switch = false; 54 | 55 | bool IsUsingValkeyAlloc() { 56 | if (!thread_using_valkey_module_alloc && 57 | use_valkey_module_alloc_switch.load(std::memory_order_relaxed)) { 58 | thread_using_valkey_module_alloc = true; 59 | return true; 60 | } 61 | return thread_using_valkey_module_alloc; 62 | } 63 | void UseValkeyAlloc() { 64 | use_valkey_module_alloc_switch.store(true, std::memory_order_relaxed); 65 | } 66 | std::atomic used_memory_bytes{0}; 67 | 68 | void ResetValkeyAlloc() { 69 | use_valkey_module_alloc_switch.store(false, std::memory_order_relaxed); 70 | thread_using_valkey_module_alloc = false; 71 | used_memory_bytes.store(0, std::memory_order_relaxed); 72 | } 73 | 74 | uint64_t GetUsedMemoryCnt() { return used_memory_bytes; } 75 | 76 | void ReportAllocMemorySize(uint64_t size) { 77 | vmsdk::used_memory_bytes.fetch_add(size, std::memory_order_relaxed); 78 | } 79 | void ReportFreeMemorySize(uint64_t size) { 80 | if (size > used_memory_bytes) { 81 | vmsdk::used_memory_bytes.store(0, std::memory_order_relaxed); 82 | } else { 83 | vmsdk::used_memory_bytes.fetch_sub(size, std::memory_order_relaxed); 84 | } 85 | } 86 | 87 | } // namespace vmsdk 88 | -------------------------------------------------------------------------------- /vmsdk/src/memory_allocation.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VMSDK_SRC_MEMORY_ALLOCATION_H_ 31 | #define VMSDK_SRC_MEMORY_ALLOCATION_H_ 32 | 33 | #include 34 | 35 | namespace vmsdk { 36 | 37 | // Updates the custom allocator to perform any future allocations using the 38 | // Valkey allocator. 39 | void UseValkeyAlloc(); 40 | bool IsUsingValkeyAlloc(); 41 | 42 | // Switch back to the default allocator. No guarantees around atomicity. Only 43 | // safe in single-threaded or testing environments. 44 | void ResetValkeyAlloc(); 45 | 46 | // Report used memory counter. 47 | uint64_t GetUsedMemoryCnt(); 48 | 49 | void ReportAllocMemorySize(uint64_t size); 50 | void ReportFreeMemorySize(uint64_t size); 51 | 52 | } // namespace vmsdk 53 | 54 | #endif // VMSDK_SRC_MEMORY_ALLOCATION_H_ 55 | -------------------------------------------------------------------------------- /vmsdk/src/module_type.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VMSDK_SRC_MODULETYPE_H_ 31 | #define VMSDK_SRC_MODULETYPE_H_ 32 | 33 | #include 34 | 35 | #include "absl/status/status.h" 36 | #include "absl/strings/string_view.h" 37 | #include "vmsdk/src/managed_pointers.h" 38 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 39 | 40 | namespace vmsdk { 41 | 42 | class ModuleType { 43 | public: 44 | ModuleType(RedisModuleCtx *ctx, absl::string_view key, 45 | RedisModuleType *module_type); 46 | virtual ~ModuleType() = default; 47 | absl::Status Register(RedisModuleCtx *ctx); 48 | absl::Status Deregister(RedisModuleCtx *ctx); 49 | 50 | static absl::Status Register(RedisModuleCtx *ctx, absl::string_view key, 51 | void *ptr, RedisModuleType *module_type); 52 | static absl::Status Deregister(RedisModuleCtx *ctx, absl::string_view key); 53 | 54 | protected: 55 | RedisModuleType *module_type_{nullptr}; 56 | vmsdk::UniqueRedisDetachedThreadSafeContext detached_ctx_; 57 | std::string key_; 58 | }; 59 | 60 | } // namespace vmsdk 61 | #endif // VMSDK_SRC_MODULETYPE_H_ 62 | -------------------------------------------------------------------------------- /vmsdk/src/status/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_STATUS_MACROS 2 | ${CMAKE_CURRENT_LIST_DIR}/status_builder.cc 3 | ${CMAKE_CURRENT_LIST_DIR}/source_location.h 4 | ${CMAKE_CURRENT_LIST_DIR}/status_builder.h 5 | ${CMAKE_CURRENT_LIST_DIR}/status_macros.h) 6 | 7 | valkey_search_add_static_library(status_macros "${SRCS_STATUS_MACROS}") 8 | target_include_directories(status_macros PUBLIC ${CMAKE_CURRENT_LIST_DIR}) 9 | target_link_libraries(status_macros PUBLIC valkey_module) 10 | -------------------------------------------------------------------------------- /vmsdk/src/status/source_location.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef VMSDK_SRC_STATUS_SOURCE_LOCATION_H_ 31 | #define VMSDK_SRC_STATUS_SOURCE_LOCATION_H_ 32 | 33 | #include 34 | 35 | namespace vmsdk { 36 | 37 | // Class representing a specific location in the source code of a program. 38 | // SourceLocation is copyable. 39 | class SourceLocation { 40 | public: 41 | // Avoid this constructor; it populates the object with placeholder values. 42 | constexpr SourceLocation() : line_(0), file_name_(nullptr) {} 43 | 44 | // Wrapper to invoke the private constructor below. This should only be 45 | // used by the AIS_LOC macro, hence the name. 46 | static constexpr SourceLocation DoNotInvokeDirectly(std::uint_least32_t line, 47 | const char* file_name) { 48 | return SourceLocation(line, file_name); 49 | } 50 | 51 | static constexpr SourceLocation current( 52 | std::uint_least32_t line = __builtin_LINE(), 53 | const char* file_name = __builtin_FILE()) { 54 | return SourceLocation(line, file_name); 55 | } 56 | 57 | // The line number of the captured source location. 58 | constexpr std::uint_least32_t line() const { return line_; } 59 | 60 | // The file name of the captured source location. 61 | constexpr const char* file_name() const { return file_name_; } 62 | 63 | // column() and function_name() are omitted because we don't have a 64 | // way to support them. 65 | 66 | private: 67 | // Do not invoke this constructor directly. Instead, use the 68 | // AIS_LOC macro below. 69 | // 70 | // file_name must outlive all copies of the SourceLocation 71 | // object, so in practice it should be a std::string literal. 72 | constexpr SourceLocation(std::uint_least32_t line, const char* file_name) 73 | : line_(line), file_name_(file_name) {} 74 | 75 | std::uint_least32_t line_; 76 | const char* file_name_; 77 | }; 78 | 79 | } // namespace vmsdk 80 | 81 | // If a function takes a SourceLocation parameter, pass this as the argument. 82 | #define VMSDK_STREAMS_LOC \ 83 | vmsdk::SourceLocation::DoNotInvokeDirectly(__LINE__, __FILE__) 84 | 85 | #define VMSDK_LOC_CURRENT_DEFAULT_ARG = vmsdk::SourceLocation::current() 86 | 87 | #endif // VMSDK_SRC_STATUS_SOURCE_LOCATION_H_ 88 | -------------------------------------------------------------------------------- /vmsdk/src/testing_infra/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(VMSDK_TESTING_INFRA_SRCS 2 | ${CMAKE_CURRENT_LIST_DIR}/utils.cc ${CMAKE_CURRENT_LIST_DIR}/utils.h 3 | ${CMAKE_CURRENT_LIST_DIR}/module.h) 4 | 5 | valkey_search_add_static_library(vmsdk_testing_infra 6 | "${VMSDK_TESTING_INFRA_SRCS}") 7 | -------------------------------------------------------------------------------- /vmsdk/src/testing_infra/utils.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, valkey-search contributors 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #include 31 | #include 32 | 33 | #include "absl/strings/str_split.h" 34 | #include "absl/strings/string_view.h" 35 | #include "vmsdk/src/valkey_module_api/valkey_module.h" 36 | 37 | namespace vmsdk { 38 | 39 | std::vector ToRedisStringVector( 40 | absl::string_view params_str, absl::string_view exclude) { 41 | std::vector params = 42 | absl::StrSplit(params_str, ' ', absl::SkipEmpty()); 43 | std::vector ret; 44 | for (size_t i = 0; i < params.size(); i += 2) { 45 | if (exclude == params[i]) { 46 | continue; 47 | } 48 | ret.push_back( 49 | RedisModule_CreateString(nullptr, params[i].data(), params[i].size())); 50 | if (i + 1 == params.size()) { 51 | break; 52 | } 53 | ret.push_back(RedisModule_CreateString(nullptr, params[i + 1].data(), 54 | params[i + 1].size())); 55 | } 56 | return ret; 57 | } 58 | 59 | } // namespace vmsdk 60 | -------------------------------------------------------------------------------- /vmsdk/src/thread_safe_vector.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "absl/algorithm/container.h" 10 | #include "absl/synchronization/mutex.h" 11 | 12 | /// A thread safe vector protected by a mutex 13 | namespace vmsdk { 14 | template 15 | class ThreadSafeVector { 16 | public: 17 | // This type is neither copyable nor assignable. 18 | ThreadSafeVector(const ThreadSafeVector&) = delete; 19 | ThreadSafeVector& operator=(const ThreadSafeVector&) = delete; 20 | ThreadSafeVector() = default; 21 | 22 | size_t Size() const { 23 | absl::ReaderMutexLock lock{&mutex_}; 24 | return vec_.size(); 25 | } 26 | bool IsEmpty() const { return Size() == 0; } 27 | 28 | /// Pop the first item that matches the predicate (starting from the top of 29 | /// the list) 30 | std::optional PopIf(std::function predicate) { 31 | absl::MutexLock lock{&mutex_}; 32 | if (vec_.empty()) { 33 | return std::nullopt; 34 | } 35 | auto where = absl::c_find_if(vec_, predicate); 36 | if (where == vec_.end()) { 37 | return std::nullopt; 38 | } 39 | auto item = std::move(*where); 40 | vec_.erase(where); 41 | return item; 42 | } 43 | 44 | /// Remove up to `count` items from the end of the list and return them. 45 | std::vector PopBackMulti(size_t count) { 46 | absl::MutexLock lock{&mutex_}; 47 | count = std::min(count, vec_.size()); 48 | if (count == 0) { 49 | return {}; 50 | } 51 | 52 | // Move the last "count" items to our result vector and resize the original 53 | // vector 54 | std::vector items; 55 | items.reserve(count); 56 | std::move(vec_.end() - count, vec_.end(), std::back_inserter(items)); 57 | vec_.resize(vec_.size() - count); 58 | return items; 59 | } 60 | 61 | /// Append `item` at the end of the vector 62 | void Add(T item) { 63 | absl::MutexLock lock{&mutex_}; 64 | vec_.push_back(std::move(item)); 65 | } 66 | 67 | /// Clear the list. Before items are removed from the list, apply `callback` 68 | /// for each item. 69 | /// 70 | /// `callback` can be `nullptr`. 71 | void ClearWithCallback(std::function callback) { 72 | absl::MutexLock lock{&mutex_}; 73 | if (callback != nullptr) { 74 | for (auto& item : vec_) { 75 | callback(item); 76 | } 77 | } 78 | vec_.clear(); 79 | } 80 | 81 | /// This is the same as calling `ClearWithCallback()` with `nullptr` as the 82 | /// callback 83 | void Clear() { ClearWithCallback(nullptr); } 84 | 85 | private: 86 | mutable absl::Mutex mutex_; 87 | std::vector vec_ ABSL_GUARDED_BY(mutex_); 88 | }; 89 | } // namespace vmsdk 90 | -------------------------------------------------------------------------------- /vmsdk/src/valkey_module_api/.clang-tidy: -------------------------------------------------------------------------------- 1 | Checks: '-*' # Disable all checks by default. 2 | HeaderFilterRegex: '.*' -------------------------------------------------------------------------------- /vmsdk/src/valkey_module_api/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(SRCS_VALKEY_MODULE ${CMAKE_CURRENT_LIST_DIR}/valkey_module.h) 2 | 3 | add_library(valkey_module INTERFACE ${SRCS_VALKEY_MODULE}) 4 | target_include_directories(valkey_module INTERFACE ${CMAKE_CURRENT_LIST_DIR}) 5 | 6 | -------------------------------------------------------------------------------- /vmsdk/versionscript.lds: -------------------------------------------------------------------------------- 1 | { 2 | global: 3 | # Only expose the RedisModule APIs 4 | RedisModule_OnLoad; 5 | RedisModule_OnUnload; 6 | local: *; # hide everything else 7 | }; --------------------------------------------------------------------------------