├── .clang-format ├── .clang-tidy ├── .dockerignore ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── 1-bug-report.yaml │ ├── 2-feature-request.yaml │ └── 3-docs-bug.yaml ├── dependabot.yml ├── pr-title-checker-config.json ├── release-drafter.yml └── workflows │ ├── import_braft.yml │ ├── issue-translator.yml │ ├── kiwidb.yml │ ├── pr-title-checker.yaml │ ├── release-drafter.yml │ └── release.yml ├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── LICENSE.md ├── README.md ├── README_CN.md ├── build_support ├── cpplint.py ├── run_clang_format.py ├── run_clang_tidy.py └── run_clang_tidy_extra.py ├── cmake ├── braft.cmake ├── brpc.cmake ├── findTools.cmake ├── fmt.cmake ├── gflags.cmake ├── gtest.cmake ├── leveldb.cmake ├── lz4.cmake ├── openssl.cmake ├── precommit.cmake ├── protobuf.cmake ├── rocksdb.cmake ├── snappy.cmake ├── spdlog.cmake ├── zlib.cmake └── zstd.cmake ├── etc ├── conf │ └── kiwi.conf └── script │ ├── build.sh │ ├── check_format.sh │ ├── kiwitests.sh │ ├── precommit.sh │ └── save_load.sh ├── src ├── CMakeLists.txt ├── base_cmd.cc ├── base_cmd.h ├── client.cc ├── client.h ├── client_map.cc ├── client_map.h ├── cmd_admin.cc ├── cmd_admin.h ├── cmd_hash.cc ├── cmd_hash.h ├── cmd_keys.cc ├── cmd_keys.h ├── cmd_kv.cc ├── cmd_kv.h ├── cmd_list.cc ├── cmd_list.h ├── cmd_raft.cc ├── cmd_raft.h ├── cmd_set.cc ├── cmd_set.h ├── cmd_table_manager.cc ├── cmd_table_manager.h ├── cmd_thread_pool.cc ├── cmd_thread_pool.h ├── cmd_thread_pool_worker.cc ├── cmd_thread_pool_worker.h ├── cmd_zset.cc ├── cmd_zset.h ├── common.cc ├── common.h ├── config.cc ├── config.h ├── config_parser.cc ├── config_parser.h ├── crc64.c ├── db.cc ├── db.h ├── helper.cc ├── helper.h ├── kiwi.cc ├── kiwi.h ├── kiwi_logo.h ├── net │ ├── CMakeLists.txt │ ├── base_event.h │ ├── base_socket.cc │ ├── base_socket.h │ ├── callback_function.h │ ├── client_socket.cc │ ├── client_socket.h │ ├── config.h │ ├── connection.h │ ├── epoll_event.cc │ ├── epoll_event.h │ ├── event_server.cc │ ├── event_server.h │ ├── io_thread.cc │ ├── io_thread.h │ ├── kqueue_event.cc │ ├── kqueue_event.h │ ├── listen_socket.cc │ ├── listen_socket.h │ ├── net_event.h │ ├── net_options.h │ ├── socket_addr.h │ ├── stream_socket.cc │ ├── stream_socket.h │ ├── thread_manager.h │ ├── timer.cc │ ├── timer.h │ └── timer_task.h ├── options.h ├── pubsub.cc ├── pubsub.h ├── raft │ ├── CMakeLists.txt │ ├── binlog.proto │ ├── raft.cc │ ├── raft.h │ ├── raft.proto │ ├── raft_service.h │ ├── snapshot.cc │ └── snapshot.h ├── replication.cc ├── replication.h ├── resp │ ├── CMakeLists.txt │ ├── resp2_encode.cc │ ├── resp2_encode.h │ ├── resp2_parse.cc │ ├── resp2_parse.h │ ├── resp_encode.cc │ ├── resp_encode.h │ ├── resp_parse.h │ └── tests │ │ ├── CMakeLists.txt │ │ ├── resp2_encode_test.cc │ │ └── resp2_parse_test.cc ├── slow_log.cc ├── slow_log.h ├── std │ ├── CMakeLists.txt │ ├── env.cc │ ├── env.h │ ├── kiwi_slot.cc │ ├── kiwi_slot.h │ ├── lock_free_ring_buffer.h │ ├── lock_mgr.cc │ ├── lock_mgr.h │ ├── log.h │ ├── memory_file.cc │ ├── memory_file.h │ ├── mutex.h │ ├── mutex_impl.cc │ ├── mutex_impl.h │ ├── noncopyable.h │ ├── scope_record_lock.cc │ ├── scope_record_lock.h │ ├── std_coding.cc │ ├── std_coding.h │ ├── std_defer.h │ ├── std_hash.cc │ ├── std_hash.h │ ├── std_mutex.h │ ├── std_slice.h │ ├── std_status.cc │ ├── std_status.h │ ├── std_string.cc │ ├── std_string.h │ ├── std_util.cc │ ├── std_util.h │ ├── tests │ │ ├── CMakeLists.txt │ │ ├── lock_free_ring_buffer_test.cc │ │ ├── std_string_test.cc │ │ └── std_util_test.cc │ ├── thread_pool.cc │ └── thread_pool.h ├── storage │ ├── CMakeLists.txt │ ├── detect_environment │ ├── include │ │ └── storage │ │ │ ├── build_version.h │ │ │ ├── slot_indexer.h │ │ │ ├── storage.h │ │ │ ├── storage_define.h │ │ │ ├── util.h │ │ │ └── version.h │ ├── src │ │ ├── base_data_key_format.h │ │ ├── base_data_value_format.h │ │ ├── base_filter.h │ │ ├── base_key_format.h │ │ ├── base_meta_value_format.h │ │ ├── base_value_format.h │ │ ├── batch.h │ │ ├── build_version.cc.in │ │ ├── coding.h │ │ ├── custom_comparator.h │ │ ├── debug.h │ │ ├── lists_data_key_format.h │ │ ├── lists_filter.h │ │ ├── lists_meta_value_format.h │ │ ├── lock_mgr.h │ │ ├── log_index.cc │ │ ├── log_index.h │ │ ├── lru_cache.h │ │ ├── murmurhash.cc │ │ ├── murmurhash.h │ │ ├── mutex.h │ │ ├── mutex_impl.h │ │ ├── options_helper.cc │ │ ├── options_helper.h │ │ ├── redis.cc │ │ ├── redis.h │ │ ├── redis_hashes.cc │ │ ├── redis_hyperloglog.cc │ │ ├── redis_hyperloglog.h │ │ ├── redis_lists.cc │ │ ├── redis_sets.cc │ │ ├── redis_strings.cc │ │ ├── redis_zsets.cc │ │ ├── scope_record_lock.h │ │ ├── scope_snapshot.h │ │ ├── storage.cc │ │ ├── storage_murmur3.h │ │ ├── strings_filter.h │ │ ├── strings_value_format.h │ │ ├── type_iterator.h │ │ ├── util.cc │ │ ├── zsets_data_key_format.h │ │ └── zsets_filter.h │ └── tests │ │ ├── CMakeLists.txt │ │ ├── custom_comparator_test.cc │ │ ├── flush_oldest_cf_test.cc │ │ ├── hashes_filter_test.cc │ │ ├── hashes_test.cc │ │ ├── keys_test.cc │ │ ├── kv_format_test.cc │ │ ├── lists_filter_test.cc │ │ ├── lists_test.cc │ │ ├── lock_mgr_test.cc │ │ ├── log_index_collector_test.cc │ │ ├── log_index_test.cc │ │ ├── lru_cache_test.cc │ │ ├── options_test.cc │ │ ├── sets_test.cc │ │ ├── strings_filter_test.cc │ │ ├── strings_test.cc │ │ └── zsets_test.cc ├── store.cc ├── store.h ├── unbounded_buffer.cc └── unbounded_buffer.h └── tests ├── README.md ├── admin_test.go ├── assets ├── default.conf ├── encodings.rdb └── hash-zipmap.rdb ├── consistency_test.go ├── go.mod ├── go.sum ├── hash_test.go ├── helpers ├── bg_complex_data.tcl ├── gen_write_load.tcl └── redis_queue.py ├── key_test.go ├── kiwi_suite_test.go ├── list_test.go ├── print_log.sh ├── sentinel ├── run.tcl ├── tests │ ├── 00-base.tcl │ ├── 01-conf-update.tcl │ ├── 02-slaves-reconf.tcl │ ├── 03-runtime-reconf.tcl │ ├── 04-slave-selection.tcl │ ├── 05-manual.tcl │ └── includes │ │ └── init-tests.tcl └── tmp │ └── .gitignore ├── set_test.go ├── string_test.go ├── support ├── redis.tcl ├── server.tcl ├── test.tcl ├── tmpfile.tcl └── util.tcl ├── test_helper.tcl ├── unit ├── acl.tcl ├── aofrw.tcl ├── auth.tcl ├── basic.tcl ├── bitops.tcl ├── command.tcl ├── dump.tcl ├── expire.tcl ├── geo.tcl ├── hyperloglog.tcl ├── introspection.tcl ├── keys.tcl ├── latency-monitor.tcl ├── limits.tcl ├── maxmemory.tcl ├── memefficiency.tcl ├── multi.tcl ├── obuf-limits.tcl ├── other.tcl ├── printver.tcl ├── protocol.tcl ├── pubsub.tcl ├── quit.tcl ├── scan.tcl ├── scripting.tcl ├── slowlog.tcl ├── sort.tcl ├── tcl │ ├── aof-race.tcl │ ├── aof.tcl │ ├── convert-zipmap-hash-on-load.tcl │ ├── rdb.tcl │ ├── redis-cli.tcl │ ├── replication-2.tcl │ ├── replication-3.tcl │ ├── replication-4.tcl │ ├── replication-psync.tcl │ └── replication.tcl ├── type.tcl └── type │ ├── hash.tcl │ ├── list-2.tcl │ ├── list-3.tcl │ ├── list-common.tcl │ ├── list.tcl │ ├── set.tcl │ ├── string.tcl │ └── zset.tcl ├── util └── pikiwidb.go └── zset_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | *pb.cc 7 | *pb.h 8 | 9 | # Precompiled Headers 10 | *.gch 11 | *.pch 12 | 13 | # Compiled Dynamic libraries 14 | *.so 15 | *.dylib 16 | *.dll 17 | 18 | # Fortran module files 19 | *.mod 20 | 21 | # Compiled Static libraries 22 | *.lai 23 | *.la 24 | *.a 25 | *.lib 26 | 27 | # Executables 28 | *.exe 29 | *.out 30 | *.app 31 | 32 | # Log path 33 | make_config.mk 34 | log/ 35 | lib/ 36 | tools/ 37 | output/ 38 | 39 | # DB 40 | db/ 41 | dump/ 42 | 43 | # third party 44 | gdb.txt 45 | tags 46 | 47 | make_config.mk 48 | src/*.d 49 | src/build_version.cc 50 | 51 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | tests/* linguist-vendored 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1-bug-report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Report a bug in the kiwi 3 | labels: [ "☢️ Bug" ] 4 | 5 | body: 6 | - type: dropdown 7 | id: is-regression 8 | attributes: 9 | label: Is this a regression? 10 | options: 11 | - 'Yes' 12 | - 'No' 13 | validations: 14 | required: true 15 | 16 | - type: textarea 17 | id: description 18 | attributes: 19 | label: Description 20 | validations: 21 | required: true 22 | 23 | - type: input 24 | id: reproduction 25 | attributes: 26 | label: Please provide a link to a minimal reproduction of the bug 27 | 28 | - type: textarea 29 | attributes: 30 | label: Screenshots or videos 31 | description: If you can, upload any screenshots of the bug. 32 | value: | 33 | ![images](https://camo.githubusercontent.com/3f51b5a32e6e5d5adabdebc5ef968150bdabc8d17a8dc1a535b8fb255d2165d0/68747470733a2f2f67772e616c697061796f626a656374732e636f6d2f7a6f732f616e7466696e63646e2f79396b776737445643642f726570726f647563652e676966) 34 | 35 | - type: textarea 36 | id: environment 37 | attributes: 38 | label: Please provide the version you discovered this bug in (check about page for version information) 39 | render: true 40 | placeholder: | 41 | Version: v0.0.1 42 | Commit Hash: 7c9b67171b4d2fbdd37218e59dbabe64451d9d68 43 | Build Date: Oct 11, 2023, 10:49 AM GMT+8 44 | OS: linux x64 45 | 46 | - type: textarea 47 | id: other 48 | attributes: 49 | label: Anything else? 50 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-feature-request.yaml: -------------------------------------------------------------------------------- 1 | name: 'Feature Request' 2 | description: Suggest a feature for kiwi 3 | labels: [ "✏️ Feature" ] 4 | 5 | body: 6 | - type: dropdown 7 | id: affected-packages 8 | attributes: 9 | label: Which kiwi functionalities are relevant/related to the feature request? 10 | options: 11 | - other 12 | multiple: true 13 | 14 | - type: textarea 15 | id: description 16 | attributes: 17 | label: Description 18 | validations: 19 | required: true 20 | 21 | - type: textarea 22 | id: proposed-solution 23 | attributes: 24 | label: Proposed solution 25 | validations: 26 | required: true 27 | 28 | - type: textarea 29 | id: alternatives-considered 30 | attributes: 31 | label: Alternatives considered 32 | validations: 33 | required: true -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/3-docs-bug.yaml: -------------------------------------------------------------------------------- 1 | name: 'Docs or kiwi Bug Report' 2 | description: Report an issue in kiwi's documentation 3 | labels: [ "📒 Documentation" ] 4 | 5 | body: 6 | - type: textarea 7 | id: description 8 | attributes: 9 | label: Describe the problem that you experienced 10 | validations: 11 | required: true 12 | 13 | - type: input 14 | id: affected-url 15 | attributes: 16 | label: Enter the URL of the topic with the problem 17 | 18 | - type: textarea 19 | id: documentation-goal 20 | attributes: 21 | label: Describe what you were looking for in the documentation 22 | 23 | - type: textarea 24 | id: reproduction-steps 25 | attributes: 26 | label: Describe the actions that led you to experience the problem 27 | 28 | - type: textarea 29 | id: expected-vs-actual-behavior 30 | attributes: 31 | label: Describe what you want to experience that would fix the problem 32 | 33 | - type: textarea 34 | id: screenshot 35 | attributes: 36 | label: Add a screenshot if that helps illustrate the problem 37 | 38 | - type: textarea 39 | id: exception-or-error 40 | attributes: 41 | label: If this problem caused an exception or error, please paste it here 42 | render: true 43 | placeholder: | 44 | ``` 45 | Paste the exception or error here inside a markdown code block, 46 | which is annotated by three grave \(`\) characters before and after the text block. 47 | ``` 48 | 49 | - type: textarea 50 | id: browser-info 51 | attributes: 52 | label: If the problem is browser-specific, please specify the device, OS, browser, and version 53 | render: true 54 | 55 | - type: textarea 56 | id: additional-info 57 | attributes: 58 | label: Provide any additional information here in as much as detail as you can 59 | render: true -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | target-branch: "3.0" 13 | 14 | - package-ecosystem: "github-actions" 15 | # Workflow files stored in the 16 | # default location of `.github/workflows` 17 | directory: "/" 18 | schedule: 19 | interval: "monthly" 20 | target-branch: "3.0" 21 | -------------------------------------------------------------------------------- /.github/pr-title-checker-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "LABEL": { 3 | "name": "Invalid PR Title", 4 | "color": "B60205" 5 | }, 6 | "CHECKS": { 7 | "regexp": "^(feat|fix|test|refactor|chore|upgrade|bump|style|docs|perf|build|ci|revert)(\\(.*\\))?:[^\u4e00-\u9fa5]+$", 8 | "ignoreLabels": [ 9 | "ignore-title" 10 | ] 11 | } 12 | } -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: '$RESOLVED_VERSION' 2 | tag-template: '$RESOLVED_VERSION' 3 | categories: 4 | - title: '❗ Breaking Changes:' 5 | labels: 6 | - '❗ Breaking Change' 7 | - title: '🚀 New Features:' 8 | labels: 9 | - '✏️ Feature' 10 | - title: '🐛 Fixes:' 11 | labels: 12 | - '☢️ Bug' 13 | - title: '📚 Documentation:' 14 | labels: 15 | - '📒 Documentation' 16 | - title: '🧹 Updates:' 17 | labels: 18 | - '🧹 Updates' 19 | - '🤖 Dependencies' 20 | change-template: '- $TITLE (#$NUMBER)' 21 | change-title-escapes: '\<*_&' 22 | exclude-contributors: 23 | - dependabot 24 | - dependabot[bot] 25 | version-resolver: 26 | major: 27 | labels: 28 | - '❗ Breaking Change' 29 | minor: 30 | labels: 31 | - '✏️ Feature' 32 | patch: 33 | labels: 34 | - '📒 Documentation' 35 | - '☢️ Bug' 36 | - '🤖 Dependencies' 37 | - '🧹 Updates' 38 | default: patch 39 | template: | 40 | $CHANGES 41 | 42 | Version tags: 43 | - `https://github.com/$OWNER/$REPOSITORY/releases/tag/v$RESOLVED_VERSION` 44 | 45 | **📒 Documentation**: kiwi 46 | 47 | **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION 48 | 49 | Thanks to $CONTRIBUTORS for making this release possible. 50 | 51 | autolabeler: 52 | - label: '📒 Documentation' 53 | files: 54 | - '*.md' 55 | title: 56 | - '/(docs|doc:|\[doc\]|typos|comment|documentation)/i' 57 | - label: '☢️ Bug' 58 | title: 59 | - '/(fix|race|bug|missing|correct)/i' 60 | - label: '🧹 Updates' 61 | title: 62 | - '/(improve|update|update|refactor|deprecated|remove|unused|test)/i' 63 | - label: '🤖 Dependencies' 64 | title: 65 | - '/(bump|dependencies)/i' 66 | - label: '✏️ Feature' 67 | title: 68 | - '/(feature|feat|create|implement|add)/i' 69 | -------------------------------------------------------------------------------- /.github/workflows/import_braft.yml: -------------------------------------------------------------------------------- 1 | name: Import BRaft Actions (Temporary) 2 | 3 | on: 4 | push: 5 | branches: [ "import-braft" ] 6 | pull_request: 7 | branches: [ "import-braft" ] 8 | 9 | jobs: 10 | check_format: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Build 17 | run: cmake -S . -B build 18 | 19 | - name: Check Format 20 | working-directory: ${{ github.workspace }}/build 21 | run: make check-format 22 | 23 | build_on_macos: 24 | runs-on: macos-latest 25 | needs: check_format 26 | 27 | steps: 28 | - uses: actions/checkout@v4 29 | 30 | - name: Build --verbose 31 | env: 32 | CPLUS_INCLUDE_PATH: /opt/homebrew/include 33 | run: | 34 | brew install autoconf 35 | brew install go 36 | bash ./etc/script/build.sh 37 | 38 | - name: Run Go E2E Tests 39 | working-directory: ${{ github.workspace }}/build-release 40 | run: | 41 | cd ../tests 42 | go mod tidy 43 | go test ./kiwi_suite_test.go ./consistency_test.go -v 44 | 45 | build_on_ubuntu: 46 | runs-on: ubuntu-latest 47 | needs: check_format 48 | 49 | steps: 50 | - uses: actions/checkout@v4 51 | 52 | - name: Build 53 | run: | 54 | bash ./etc/script/build.sh --verbos 55 | 56 | - name: Run Go E2E Tests 57 | working-directory: ${{ github.workspace }}/build-release 58 | run: | 59 | cd ../tests 60 | go mod tidy 61 | go test ./kiwi_suite_test.go ./consistency_test.go -v 62 | -------------------------------------------------------------------------------- /.github/workflows/issue-translator.yml: -------------------------------------------------------------------------------- 1 | name: Issue Translator 2 | on: 3 | issue_comment: 4 | types: [created] 5 | issues: 6 | types: [opened] 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: usthe/issues-translate-action@v2.7 13 | with: 14 | IS_MODIFY_TITLE: false 15 | CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically. -------------------------------------------------------------------------------- /.github/workflows/pr-title-checker.yaml: -------------------------------------------------------------------------------- 1 | name: "PR Title Checker" 2 | on: 3 | pull_request_target: 4 | types: 5 | - opened 6 | - edited 7 | - synchronize 8 | - labeled 9 | - unlabeled 10 | 11 | jobs: 12 | check: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: thehanimo/pr-title-checker@v1.4.2 16 | with: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | pass_on_octokit_error: false 19 | configuration_path: .github/pr-title-checker-config.json #(optional. defaults to .github/pr-title-checker-config.json) -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - unstable 7 | pull_request_target: 8 | types: [ opened, reopened, synchronize ] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | update_release_draft: 15 | permissions: 16 | contents: write 17 | pull-requests: write 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: release-drafter/release-drafter@v5 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | 8 | jobs: 9 | build: 10 | name: Build binary 11 | strategy: 12 | matrix: 13 | include: 14 | - arch: x86_64-unknown-linux-gnu 15 | os: ubuntu-latest 16 | file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-amd64 17 | file_ext: .tar.gz 18 | - arch: aarch64-unknown-linux-gnu 19 | os: ubuntu-latest 20 | file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-arm64 21 | file_ext: .tar.gz 22 | - arch: x86_64-apple-darwin 23 | os: macos-latest 24 | file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-amd64 25 | file_ext: .tar.gz 26 | - arch: aarch64-apple-darwin 27 | os: macos-latest 28 | file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-arm64 29 | file_ext: .tar.gz 30 | 31 | runs-on: ${{ matrix.os }} 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | 36 | - name: Build 37 | run: | 38 | bash ./etc/script/build.sh 39 | 40 | - name: Calculate checksum and rename binary 41 | working-directory: ${{ github.workspace }}/bin 42 | shell: bash 43 | run: | 44 | chmod +x ${{ github.event.repository.name }} 45 | tar -zcvf ${{ matrix.file_name }}${{ matrix.file_ext }} ${{ github.event.repository.name }} 46 | echo $(shasum -a 256 ${{ matrix.file_name }}${{ matrix.file_ext }} | cut -f1 -d' ') > ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum 47 | 48 | - name: Upload artifacts 49 | uses: actions/upload-artifact@v3 50 | with: 51 | name: ${{ matrix.file_name }}${{ matrix.file_ext }} 52 | path: bin/${{ matrix.file_name }}${{ matrix.file_ext }} 53 | 54 | - name: Upload checksum of artifacts 55 | uses: actions/upload-artifact@v3 56 | with: 57 | name: ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum 58 | path: bin/${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum 59 | 60 | release: 61 | name: Release artifacts 62 | needs: [ build ] 63 | runs-on: ubuntu-latest 64 | steps: 65 | - name: Download artifacts 66 | uses: actions/download-artifact@v3 67 | 68 | - name: Publish release 69 | uses: softprops/action-gh-release@v1 70 | with: 71 | name: "Release ${{ github.ref_name }}" 72 | generate_release_notes: true 73 | files: | 74 | **/${{ github.event.repository.name }}-* 75 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # vim temp files 2 | *.swp 3 | # Compiled Object files 4 | *.slo 5 | *.lo 6 | *.o 7 | *.obj 8 | *pb.cc 9 | *pb.h 10 | deps-debug/ 11 | deps-release/ 12 | cmake-build-debug/ 13 | cmake-build-release/ 14 | cmake-build-release-release/ 15 | build/ 16 | build-debug/ 17 | build-release/ 18 | download/ 19 | CMakeFiles/ 20 | CMakeCache.txt 21 | 22 | # clang-format 23 | temp.txt 24 | 25 | # Precompiled Headers 26 | *.gch 27 | *.pch 28 | 29 | # Compiled Dynamic libraries 30 | *.so 31 | *.dylib 32 | *.dll 33 | 34 | 35 | # Compiled Static libraries 36 | *.lai *.la 37 | *.a 38 | *.lib 39 | 40 | # Executables 41 | *.exe 42 | *.out 43 | *.app 44 | 45 | # Log path 46 | make_config.mk 47 | logs/ 48 | lib/ 49 | output/ 50 | *.log 51 | 52 | # DB 53 | db/ 54 | dump/ 55 | src/dbsync/ 56 | 57 | # third party 58 | gdb.txt 59 | tags 60 | 61 | # IDE 62 | .vscode 63 | 64 | # generate 65 | make_config.mk 66 | src/*.d 67 | src/build_version.cc 68 | 69 | #cache 70 | .cache 71 | 72 | .idea/ 73 | 74 | 75 | #build 76 | build-release/ 77 | build-debug/ 78 | buildtrees 79 | deps 80 | deps-debug/ 81 | deps-release/ 82 | 83 | #develop container 84 | .devcontainer 85 | 86 | # include codis fe javascript lib files 87 | !codis/cmd/fe/assets/** 88 | 89 | tests/tmp 90 | bin 91 | 92 | compile_commands.json 93 | 94 | # build support 95 | build_support/__pycache__ 96 | build_support/clang_format_exclusions.txt 97 | 98 | # pkg 99 | pkg 100 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: cpp 4 | 5 | os: 6 | - linux 7 | 8 | env: 9 | global: 10 | - PROTOBUF_VERSION=2.5.0 11 | 12 | install: 13 | - wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.bz2 14 | - tar xvf protobuf-$PROTOBUF_VERSION.tar.bz2 15 | - ( cd protobuf-$PROTOBUF_VERSION && ./configure --prefix=/usr && make && sudo make install ) 16 | 17 | addons: 18 | apt: 19 | packages: ['libsnappy-dev', 'libprotobuf-dev', 'libgoogle-glog-dev'] 20 | 21 | compiler: 22 | - gcc 23 | 24 | language: cpp 25 | 26 | script: make 27 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2023, arana-db Foundation 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 13 | OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | -------------------------------------------------------------------------------- /build_support/run_clang_tidy_extra.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | A helper class, to suppress execution of clang-tidy. 5 | 6 | In clang-tidy-6.0, if the clang-tidy configuration file suppresses ALL checks, 7 | (e.g. via a .clang-tidy file), clang-tidy will print usage information and 8 | exit with a return code of 0. Harmless but verbose. In later versions of 9 | clang-tidy the return code becomes 1, making this a bigger problem. 10 | 11 | This helper addresses the problem by suppressing execution according to 12 | the configuration in this file. 13 | """ 14 | 15 | import re 16 | 17 | class CheckConfig(object): 18 | """ Check paths against the built-in config """ 19 | 20 | def __init__(self): 21 | self._init_config() 22 | # debug prints 23 | self.debug = False 24 | return 25 | 26 | def _init_config(self): 27 | """ Any path matching one of the ignore_pats regular expressions, 28 | denotes that we do NOT want to run clang-tidy on that item. 29 | """ 30 | self.ignore_pats = [".*/third_party/.*", ] 31 | return 32 | 33 | def should_skip(self, path): 34 | """ Should execution of clang-tidy be skipped? 35 | path - to check, against the configuration. 36 | Typically the full path. 37 | returns - False if we want to run clang-tidy 38 | True of we want to skip execution on this item 39 | """ 40 | for pat in self.ignore_pats: 41 | if re.match(pat, path): 42 | if self.debug: 43 | print("match pat: {}, {} => don't run".format(pat, path)) 44 | return True 45 | return False 46 | 47 | -------------------------------------------------------------------------------- /cmake/braft.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | INCLUDE(ExternalProject) 7 | 8 | SET(BRAFT_SOURCES_DIR "${LIB_SOURCE_DIR}/braft" CACHE PATH "Path to braft sources") 9 | SET(BRAFT_INSTALL_DIR ${LIB_INSTALL_PREFIX}) 10 | SET(BRAFT_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "brpc include directory." FORCE) 11 | SET(BRAFT_LIBRARIES "${LIB_INSTALL_DIR}/libbraft.a" CACHE FILEPATH "brpc library." FORCE) 12 | 13 | IF (CMAKE_SYSTEM_NAME STREQUAL "Darwin") 14 | SET(BRAFT_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-U,__Z13GetStackTracePPvii") 15 | ELSE () 16 | SET(BRAFT_CXX_FLAGS "${CMAKE_CXX_FLAGS}") 17 | ENDIF () 18 | 19 | ExternalProject_Add( 20 | braft 21 | ${EXTERNAL_PROJECT_LOG_ARGS} 22 | DEPENDS brpc 23 | GIT_REPOSITORY "https://github.com/arana-db/braft.git" 24 | GIT_TAG v1.1.2-beta20250101 25 | GIT_SHALLOW true 26 | SOURCE_DIR ${BRAFT_SOURCES_DIR} 27 | GIT_SHALLOW true 28 | UPDATE_COMMAND "" 29 | CMAKE_ARGS 30 | ${EXTERNAL_PROJECT_C} 31 | ${EXTERNAL_PROJECT_CXX} 32 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 33 | # To be compatible with cmake 4.0 34 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 35 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 36 | -DCMAKE_CXX_FLAGS=${BRAFT_CXX_FLAGS} 37 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 38 | -DCMAKE_LIBRARY_PATH=${LIB_INSTALL_PREFIX} 39 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 40 | -DBRPC_LIB=${BRPC_LIBRARIES} 41 | -DBRPC_INCLUDE_PATH=${BRPC_INCLUDE_DIR} 42 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 43 | -DBRPC_WITH_GLOG=OFF 44 | -DCMAKE_FIND_LIBRARY_SUFFIXES=${LIB_INSTALL_PREFIX} 45 | -DLEVELDB_WITH_SNAPPY=ON 46 | -DLEVELDB_LIB=${LEVELDB_LIBRARIES} 47 | -DLEVELDB_INCLUDE_PATH=${LEVELDB_INCLUDE_DIR} 48 | 49 | -DGFLAGS_INCLUDE_PATH=${GFLAGS_INCLUDE_DIR} 50 | -DGFLAGS_LIB=${GFLAGS_LIBRARIES} 51 | 52 | -DPROTOC_LIB=${PROTOC_LIBRARY} 53 | -DPROTOBUF_LIBRARY=${PROTOBUF_LIBRARY} 54 | -DPROTOBUF_INCLUDE_DIRS=${PROTOBUF_INCLUDE_DIR} 55 | -DPROTOBUF_PROTOC_EXECUTABLE=${PROTOBUF_PROTOC} 56 | -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} 57 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 58 | ${EXTERNAL_GENERATOR} 59 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 60 | BUILD_BYPRODUCTS ${BRAFT_LIBRARIES} 61 | ) 62 | -------------------------------------------------------------------------------- /cmake/findTools.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | FIND_PROGRAM(AUTOCONF autoconf PATHS /usr/bin /usr/local/bin) 7 | 8 | IF (${AUTOCONF} MATCHES AUTOCONF-NOTFOUND) 9 | MESSAGE(FATAL_ERROR "not find autoconf on localhost") 10 | ELSE() 11 | MESSAGE(STATUS "found autoconf at ${AUTOCONF}") 12 | ENDIF () 13 | 14 | FIND_PROGRAM(CLANG_FORMAT_BIN 15 | NAMES clang-format 16 | HINTS ${CLANG_SEARCH_PATH}) 17 | IF ("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") 18 | MESSAGE(WARNING "couldn't find clang-format.") 19 | ELSE () 20 | MESSAGE(STATUS "found clang-format at ${CLANG_FORMAT_BIN}") 21 | ENDIF () 22 | 23 | FIND_PROGRAM(CLANG_TIDY_BIN 24 | NAMES clang-tidy clang-tidy-12 clang-tidy-14 25 | HINTS ${CLANG_SEARCH_PATH}) 26 | IF ("${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND") 27 | MESSAGE(WARNING "couldn't find clang-tidy.") 28 | ELSE () 29 | MESSAGE(STATUS "found clang-tidy at ${CLANG_TIDY_BIN}") 30 | ENDIF () 31 | 32 | FIND_PROGRAM(CPPLINT_BIN 33 | NAMES cpplint cpplint.py 34 | HINTS "${BUILD_SUPPORT_DIR}") 35 | IF ("${CPPLINT_BIN}" STREQUAL "CPPLINT_BIN-NOTFOUND") 36 | MESSAGE(WARNING "couldn't find cpplint.py") 37 | ELSE () 38 | MESSAGE(STATUS "found cpplint at ${CPPLINT_BIN}") 39 | ENDIF () 40 | 41 | FIND_PROGRAM(CLANG_APPLY_REPLACEMENTS_BIN 42 | NAMES clang-apply-replacements clang-apply-replacements-12 clang-apply-replacements-14 43 | HINTS ${CLANG_SEARCH_PATH}) 44 | IF ("${CLANG_APPLY_REPLACEMENTS_BIN}" STREQUAL "CLANG_APPLY_REPLACEMENTS_BIN-NOTFOUND") 45 | MESSAGE(WARNING "couldn't find clang-apply-replacements.") 46 | ELSE () 47 | MESSAGE(STATUS "found clang-apply-replacements at ${CLANG_APPLY_REPLACEMENTS_BIN}") 48 | ENDIF () 49 | 50 | # Configure CCache if available 51 | find_program(CCACHE_FOUND ccache) 52 | if(CCACHE_FOUND) 53 | set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) 54 | set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) 55 | endif(CCACHE_FOUND) 56 | 57 | OPTION(WITH_COMMAND_DOCS "build with command docs support" OFF) 58 | IF (WITH_COMMAND_DOCS) 59 | ADD_DEFINITIONS(-DWITH_COMMAND_DOCS) 60 | ENDIF () 61 | 62 | IF (${CMAKE_BUILD_TYPE} MATCHES "RELEASE") 63 | MESSAGE(STATUS "make RELEASE version") 64 | ADD_DEFINITIONS(-DBUILD_RELEASE) 65 | SET(BuildType "Release") 66 | ELSE () 67 | MESSAGE(STATUS "make DEBUG version") 68 | ADD_DEFINITIONS(-DBUILD_DEBUG) 69 | SET(BuildType "Debug") 70 | ENDIF () 71 | -------------------------------------------------------------------------------- /cmake/fmt.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | 7 | IF (${LIB_BUILD_TYPE} STREQUAL DEBUG) 8 | SET(LIB_FMT libfmtd.a) 9 | ELSE () 10 | SET(LIB_FMT libfmt.a) 11 | ENDIF () 12 | 13 | SET(FMT_SOURCES_DIR "${LIB_SOURCE_DIR}/fmt" CACHE PATH "Path to fmt sources") 14 | SET(FMT_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "fmt include directory." FORCE) 15 | SET(FMT_LIBRARIES "${LIB_INSTALL_DIR}/${LIB_FMT}" CACHE FILEPATH "fmt library directory." FORCE) 16 | 17 | ExternalProject_Add( 18 | fmt 19 | URL https://github.com/fmtlib/fmt/archive/10.1.1.zip 20 | URL_HASH SHA256=3c2e73019178ad72b0614a3124f25de454b9ca3a1afe81d5447b8d3cbdb6d322 21 | DOWNLOAD_NO_PROGRESS 1 22 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 23 | DOWNLOAD_NAME "fmt-10.1.1.zip" 24 | SOURCE_DIR ${FMT_SOURCES_DIR} 25 | UPDATE_COMMAND "" 26 | CMAKE_ARGS 27 | ${EXTERNAL_PROJECT_C} 28 | ${EXTERNAL_PROJECT_CXX} 29 | ${EXTERNAL_PROJECT_CXX_FLAGS} 30 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 31 | # To be compatible with cmake 4.0 32 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 33 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 34 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 35 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 36 | -DFMT_DOC=FALSE 37 | -DFMT_TEST=FALSE 38 | -DSHARED_LIBS=FALSE 39 | ${EXTERNAL_GENERATOR} 40 | BUILD_BYPRODUCTS ${FMT_LIBRARIES} 41 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 42 | ) 43 | -------------------------------------------------------------------------------- /cmake/gflags.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | IF (${LIB_BUILD_TYPE} STREQUAL DEBUG) 7 | SET(LIB_GFLAGS libgflags_debug.a) 8 | ELSE () 9 | SET(LIB_GFLAGS libgflags.a) 10 | ENDIF () 11 | 12 | SET(GFLAGS_SOURCES_DIR "${LIB_SOURCE_DIR}/gflags" CACHE PATH "Path to gflags sources") 13 | SET(GFLAGS_INCLUDE_DIR ${LIB_INCLUDE_DIR} CACHE PATH "gflags include directory." FORCE) 14 | SET(GFLAGS_LIBRARIES ${LIB_INSTALL_DIR}/${LIB_GFLAGS} CACHE FILEPATH "gflags library." FORCE) 15 | SET(GFLAGS_LIBRARY ${LIB_INSTALL_DIR}/${LIB_GFLAGS} CACHE FILEPATH "gflags library." FORCE) 16 | 17 | ExternalProject_Add( 18 | gflags 19 | URL https://github.com/gflags/gflags/archive/v2.2.2.zip 20 | URL_HASH SHA256=19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5 21 | DOWNLOAD_NO_PROGRESS 1 22 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 23 | DOWNLOAD_NAME "gflags-2.2.2.zip" 24 | SOURCE_DIR ${GFLAGS_SOURCES_DIR} 25 | UPDATE_COMMAND "" 26 | CMAKE_ARGS 27 | ${EXTERNAL_GENERATOR} 28 | ${EXTERNAL_PROJECT_C} 29 | ${EXTERNAL_PROJECT_CXX} 30 | ${EXTERNAL_PROJECT_CXX_FLAGS} 31 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 32 | # To be compatible with cmake 4.0 33 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 34 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 35 | -DGFLAGS_BUILD_STATIC_LIBS=ON 36 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 37 | -DBUILD_SHARED_LIBS=OFF 38 | -DGFLAGS_BUILD_SHARED_LIBS=OFF 39 | -DGFLAGS_BUILD_gflags_LIB=ON 40 | -DGFLAGS_BUILD_gflags_nothreads_LIB=ON 41 | -DGFLAGS_NAMESPACE=gflags 42 | -DGFLAGS_BUILD_TESTING=OFF 43 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 44 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 45 | BUILD_BYPRODUCTS ${GFLAGS_LIBRARY} 46 | ) 47 | -------------------------------------------------------------------------------- /cmake/gtest.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(GTEST_SOURCES_DIR "${LIB_SOURCE_DIR}/gtest" CACHE PATH "Path to gtest sources") 7 | SET(GTEST_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "gtest include directory." FORCE) 8 | SET(GTEST_LIBRARIES "${LIB_INSTALL_DIR}/libgtest.a" CACHE FILEPATH "gtest lib directory." FORCE) 9 | SET(GTEST_MAIN_LIBRARIES "${LIB_INSTALL_DIR}/libgtest_main.a" CACHE FILEPATH "gtest main include directory." FORCE) 10 | SET(GMOCK_LIBRARIES "${LIB_INSTALL_DIR}/libgmock.a" CACHE FILEPATH "gmock directory." FORCE) 11 | 12 | ExternalProject_Add( 13 | gtest 14 | GIT_REPOSITORY https://github.com/google/googletest.git 15 | GIT_TAG v1.14.0 16 | GIT_SHALLOW true 17 | SOURCE_DIR ${GTEST_SOURCES_DIR} 18 | GIT_SHALLOW true 19 | UPDATE_COMMAND "" 20 | CMAKE_ARGS 21 | ${EXTERNAL_GENERATOR} 22 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 23 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 24 | -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 25 | ${EXTERNAL_PROJECT_C} 26 | ${EXTERNAL_PROJECT_CXX} 27 | ${EXTERNAL_PROJECT_CXX_FLAGS} 28 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 29 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 30 | BUILD_BYPRODUCTS ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} 31 | ) 32 | -------------------------------------------------------------------------------- /cmake/leveldb.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(LEVELDB_SOURCES_DIR "${LIB_SOURCE_DIR}/leveldb" CACHE PATH "Path to leveldb sources") 7 | SET(LEVELDB_INCLUDE_DIR "${LIB_INCLUDE_DIR}/leveldb" CACHE PATH "leveldb include directory." FORCE) 8 | SET(LEVELDB_LIBRARIES "${LIB_INSTALL_DIR}/libleveldb.a" CACHE FILEPATH "leveldb include directory." FORCE) 9 | SET(LEVELDB_INSTALL_LIBDIR "${LIB_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") 10 | 11 | ExternalProject_Add( 12 | leveldb 13 | ${EXTERNAL_PROJECT_LOG_ARGS} 14 | DEPENDS snappy 15 | GIT_REPOSITORY "https://github.com/google/leveldb.git" 16 | GIT_TAG "1.23" 17 | SOURCE_DIR ${LEVELDB_SOURCES_DIR} 18 | GIT_SHALLOW true 19 | UPDATE_COMMAND "" 20 | CMAKE_ARGS 21 | ${EXTERNAL_PROJECT_C} 22 | ${EXTERNAL_PROJECT_CXX} 23 | ${EXTERNAL_PROJECT_CXX_FLAGS} 24 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 25 | -DCMAKE_INSTALL_INCLUDEDIR=${LEVELDB_INCLUDE_DIR} 26 | -DCMAKE_FIND_LIBRARY_SUFFIXES=${LIB_INSTALL_PREFIX} 27 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 28 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 29 | -DSnappy_INCLUDE_DIR=${Snappy_INCLUDE_DIRS} 30 | -DSnappy_LIBRARIES=${Snappy_LIBRARIES} 31 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 32 | -DLEVELDB_BUILD_TESTS=OFF 33 | -DLEVELDB_BUILD_BENCHMARKS=OFF 34 | -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} 35 | ${EXTERNAL_GENERATOR} 36 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 37 | BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES} 38 | ) 39 | -------------------------------------------------------------------------------- /cmake/lz4.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(LZ4_SOURCES_DIR "${LIB_SOURCE_DIR}/lz4" CACHE PATH "Path to lz4 sources") 7 | SET(LZ4_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "lz4 include directory." FORCE) 8 | SET(LZ4_LIBRARIES "${LIB_INSTALL_DIR}/liblz4.a" CACHE FILEPATH "lz4 include directory." FORCE) 9 | 10 | ExternalProject_Add( 11 | lz4 12 | URL https://github.com/lz4/lz4/archive/refs/tags/v1.9.4.tar.gz 13 | URL_HASH SHA256=0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b 14 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 15 | DOWNLOAD_NAME "lz4-1.9.4.tar.gz" 16 | SOURCE_DIR ${LZ4_SOURCES_DIR} 17 | DOWNLOAD_NO_PROGRESS 1 18 | ${EXTERNAL_PROJECT_LOG_ARGS} 19 | SOURCE_SUBDIR build/cmake 20 | UPDATE_COMMAND "" 21 | CMAKE_ARGS 22 | ${EXTERNAL_PROJECT_C} 23 | ${EXTERNAL_PROJECT_CXX} 24 | ${EXTERNAL_PROJECT_CXX_FLAGS} 25 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 26 | # To be compatible with cmake 4.0 27 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 28 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 29 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 30 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 31 | -DBUILD_TESTING=OFF 32 | -DBUILD_STATIC_LIBS=ON 33 | -DBUILD_SHARED_LIBS=OFF 34 | ${EXTERNAL_GENERATOR} 35 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 36 | BUILD_BYPRODUCTS ${LZ4_LIBRARIES} 37 | ) 38 | -------------------------------------------------------------------------------- /cmake/openssl.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(OPENSSL_SOURCE_DIR "${LIB_SOURCE_DIR}/openssl" CACHE PATH "Path to OpenSSL sources") 7 | SET(OPENSSL_INSTALL_DIR "${LIB_INSTALL_PREFIX}") 8 | SET(OPENSSL_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "Openssl include directory." FORCE) 9 | SET(OPENSSL_LIB "lib") 10 | SET(OPENSSL_SSL_LIBRARY "${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libssl.a") 11 | SET(OPENSSL_CRYPTO_LIBRARY "${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libcrypto.a") 12 | 13 | FILE(MAKE_DIRECTORY ${OPENSSL_INCLUDE_DIR}) 14 | 15 | SET(OPENSSL_CFLAGS "${CMAKE_C_FLAGS} -w")#disable warnings 16 | IF (CMAKE_SYSTEM_NAME MATCHES "Darwin") 17 | EXECUTE_PROCESS( 18 | COMMAND xcrun --show-sdk-path 19 | OUTPUT_VARIABLE SDK_PATH 20 | OUTPUT_STRIP_TRAILING_WHITESPACE 21 | ) 22 | SET(OPENSSL_CFLAGS "${OPENSSL_CFLAGS} -I${SDK_PATH}/usr/include") 23 | ENDIF () 24 | 25 | ExternalProject_Add( 26 | OpenSSL 27 | URL https://github.com/openssl/openssl/archive/refs/tags/openssl-3.2.1.tar.gz 28 | URL_HASH SHA256=75cc6803ffac92625c06ea3c677fb32ef20d15a1b41ecc8dddbc6b9d6a2da84c 29 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 30 | DOWNLOAD_NAME "openssl-3.2.1.tar.gz" 31 | SOURCE_DIR ${OPENSSL_SOURCE_DIR} 32 | DOWNLOAD_NO_PROGRESS 1 33 | USES_TERMINAL_DOWNLOAD TRUE 34 | CONFIGURE_COMMAND 35 | env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} 36 | /config 37 | --prefix=${OPENSSL_INSTALL_DIR} 38 | --openssldir=${OPENSSL_INSTALL_DIR} 39 | --libdir=${OPENSSL_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR} 40 | no-docs 41 | BUILD_COMMAND make CFLAGS=${OPENSSL_CFLAGS} -j${CPU_CORE} 42 | TEST_COMMAND "" 43 | INSTALL_COMMAND make install_sw 44 | INSTALL_DIR ${OPENSSL_INSTALL_DIR} 45 | UPDATE_COMMAND "" 46 | BUILD_BYPRODUCTS ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY} 47 | ) 48 | 49 | ADD_LIBRARY(ssl STATIC IMPORTED GLOBAL) 50 | SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY}) 51 | SET_PROPERTY(TARGET ssl PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR}) 52 | ADD_DEPENDENCIES(ssl OpenSSL) 53 | 54 | ADD_LIBRARY(crypto STATIC IMPORTED GLOBAL) 55 | SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_INSTALL_DIR}/${OPENSSL_LIB}/libcrypto.a) 56 | SET_PROPERTY(TARGET crypto PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${OPENSSL_INCLUDE_DIR}) 57 | ADD_DEPENDENCIES(crypto OpenSSL) 58 | -------------------------------------------------------------------------------- /cmake/precommit.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2025-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | 7 | # define hooks source and target path 8 | SET(HOOKS_SOURCE_DIR "${CMAKE_SOURCE_DIR}/etc/script") 9 | set(GIT_HOOKS_DIR "${CMAKE_SOURCE_DIR}/.git/hooks") 10 | 11 | # # ensure pre-commit hook is installed 12 | # ADD_CUSTOM_TARGET(install_git_hooks 13 | # COMMAND ${CMAKE_COMMAND} -E make_directory ${GIT_HOOKS_DIR} 14 | # COMMAND ${CMAKE_COMMAND} -E copy_if_different 15 | # ${HOOKS_SOURCE_DIR}/precommit.sh 16 | # ${GIT_HOOKS_DIR}/pre-commit 17 | # COMMAND ${CMAKE_COMMAND} -E chmod 755 ${GIT_HOOKS_DIR}/pre-commit 18 | # COMMENT "Installing git hooks..." 19 | # ) 20 | -------------------------------------------------------------------------------- /cmake/protobuf.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | # Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp 7 | 8 | IF (${LIB_BUILD_TYPE} STREQUAL DEBUG) 9 | SET(LIB_PROTOBUF "libprotobufd.a") 10 | SET(LIB_PROTOC "libprotocd.a") 11 | ELSE () 12 | SET(LIB_PROTOBUF "libprotobuf.a") 13 | SET(LIB_PROTOC "libprotoc.a") 14 | ENDIF () 15 | 16 | SET(PROTOBUF_SOURCES_DIR "${LIB_SOURCE_DIR}/protobuf" CACHE PATH "Path to protobuf sources") 17 | SET(PROTOBUF_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "protobuf include directory." FORCE) 18 | SET(PROTOBUF_LIBRARY "${LIB_INSTALL_DIR}/${LIB_PROTOBUF}" CACHE FILEPATH "protobuf install directory." FORCE) 19 | SET(PROTOC_LIBRARY "${LIB_INSTALL_DIR}/${LIB_PROTOC}" CACHE FILEPATH "protoc install directory." FORCE) 20 | SET(PROTOBUF_PROTOC "${LIB_INSTALL_PREFIX}/bin/protoc") 21 | 22 | ExternalProject_Add( 23 | protobuf 24 | LOG_CONFIGURE 1 25 | LOG_BUILD 1 26 | LOG_INSTALL 1 27 | SOURCE_SUBDIR cmake 28 | DEPENDS zlib 29 | URL "https://github.com/protocolbuffers/protobuf/archive/v3.18.0.tar.gz" 30 | URL_HASH SHA256=14e8042b5da37652c92ef6a2759e7d2979d295f60afd7767825e3de68c856c54 31 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 32 | DOWNLOAD_NAME "protobuf-3.18.0.tar.gz" 33 | SOURCE_DIR ${PROTOBUF_SOURCES_DIR} 34 | DOWNLOAD_NO_PROGRESS 1 35 | UPDATE_COMMAND "" 36 | CMAKE_ARGS 37 | ${EXTERNAL_PROJECT_C} 38 | ${EXTERNAL_PROJECT_CXX} 39 | ${EXTERNAL_PROJECT_CXX_FLAGS} 40 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 41 | # To be compatible with cmake 4.0 42 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 43 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 44 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 45 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 46 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 47 | -DBUILD_SHARED_LIBS=OFF 48 | -Dprotobuf_BUILD_TESTS=OFF 49 | -Dprotobuf_BUILD_LIBPROTOC=ON 50 | ${EXTERNAL_GENERATOR} 51 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 52 | BUILD_BYPRODUCTS ${PROTOBUF_LIBRARY} 53 | ) 54 | -------------------------------------------------------------------------------- /cmake/rocksdb.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(ROCKSDB_SOURCES_DIR "${LIB_SOURCE_DIR}/rocksdb" CACHE PATH "Path to RocksDB sources") 7 | SET(ROCKSDB_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "rocksdb include directory." FORCE) 8 | SET(ROCKSDB_LIBRARIES "${LIB_INSTALL_DIR}/librocksdb.a" CACHE FILEPATH "rocksdb include directory." FORCE) 9 | 10 | ExternalProject_Add( 11 | rocksdb 12 | ${EXTERNAL_PROJECT_LOG_ARGS} 13 | DEPENDS gflags snappy zlib lz4 zstd 14 | GIT_REPOSITORY https://github.com/facebook/rocksdb.git 15 | GIT_TAG v9.11.1 16 | SOURCE_DIR ${ROCKSDB_SOURCES_DIR} 17 | GIT_SHALLOW true 18 | UPDATE_COMMAND "" 19 | CMAKE_ARGS 20 | ${EXTERNAL_PROJECT_C} 21 | ${EXTERNAL_PROJECT_CXX} 22 | ${EXTERNAL_PROJECT_CXX_FLAGS} 23 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 24 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 25 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 26 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 27 | -DWITH_BENCHMARK=OFF 28 | -DWITH_BENCHMARK_TOOLS=OFF 29 | -DWITH_TOOLS=OFF 30 | -DWITH_CORE_TOOLS=OFF 31 | -DWITH_TESTS=OFF 32 | -DWITH_TRACE_TOOLS=OFF 33 | -DWITH_EXAMPLES=OFF 34 | -DROCKSDB_BUILD_SHARED=OFF 35 | -DWITH_JEMALLOC=${JEMALLOC_ON} 36 | -DFAIL_ON_WARNINGS=OFF 37 | -DWITH_LIBURING=OFF 38 | -DWITH_TESTS=OFF 39 | -DWITH_LZ4=ON 40 | -DWITH_SNAPPY=ON 41 | -DWITH_ZLIB=ON 42 | -DWITH_ZSTD=ON 43 | -DWITH_GFLAGS=ON 44 | -DUSE_RTTI=ON 45 | ${EXTERNAL_GENERATOR} 46 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 47 | BUILD_BYPRODUCTS ${ROCKSDB_LIBRARIES} 48 | ) 49 | -------------------------------------------------------------------------------- /cmake/snappy.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(Snappy_SOURCES_DIR "${LIB_SOURCE_DIR}/snappy" CACHE PATH "Path to snappy sources") 7 | SET(Snappy_INCLUDE_DIRS "${LIB_INCLUDE_DIR}" CACHE PATH "Snappy include directory." FORCE) 8 | SET(Snappy_LIBRARIES "${LIB_INSTALL_DIR}/libsnappy.a" CACHE FILEPATH "Snappy install directory." FORCE) 9 | 10 | ExternalProject_Add( 11 | snappy 12 | ${EXTERNAL_PROJECT_LOG_ARGS} 13 | GIT_REPOSITORY "https://github.com/google/snappy.git" 14 | GIT_TAG "1.2.1" 15 | SOURCE_DIR ${Snappy_SOURCES_DIR} 16 | GIT_SHALLOW true 17 | UPDATE_COMMAND "" 18 | CMAKE_ARGS 19 | ${EXTERNAL_PROJECT_C} 20 | ${EXTERNAL_PROJECT_CXX} 21 | ${EXTERNAL_PROJECT_CXX_FLAGS} 22 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 23 | # To be compatible with cmake 4.0 24 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 25 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 26 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 27 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 28 | -DSNAPPY_BUILD_TESTS=OFF 29 | -DSNAPPY_BUILD_BENCHMARKS=OFF 30 | -DBUILD_STATIC_LIBS=ON 31 | -DBUILD_SHARED_LIBS=OFF 32 | ${EXTERNAL_GENERATOR} 33 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 34 | BUILD_BYPRODUCTS ${Snappy_LIBRARIES} 35 | ) 36 | -------------------------------------------------------------------------------- /cmake/spdlog.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | IF (BUILD_TYPE STREQUAL DEBUG) 7 | SET(SPDLOG_LIB "libspdlogd.a") 8 | ELSE () 9 | SET(SPDLOG_LIB "libspdlog.a") 10 | ENDIF () 11 | 12 | SET(SPDLOG_SOURCES_DIR "${LIB_SOURCE_DIR}/spdlog" CACHE PATH "Path to spdlog sources") 13 | SET(SPDLOG_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "spdlog include directory." FORCE) 14 | SET(SPDLOG_LIBRARIES "${LIB_INSTALL_DIR}/${SPDLOG_LIB}" CACHE FILEPATH "spdlog library directory." FORCE) 15 | 16 | ADD_DEFINITIONS(-DSPDLOG_FMT_EXTERNAL) 17 | 18 | ExternalProject_Add( 19 | spdlog 20 | ${EXTERNAL_PROJECT_LOG_ARGS} 21 | DEPENDS fmt 22 | URL https://github.com/gabime/spdlog/archive/v1.12.0.zip 23 | URL_HASH SHA256=6174bf8885287422a6c6a0312eb8a30e8d22bcfcee7c48a6d02d1835d7769232 24 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 25 | DOWNLOAD_NAME "spdlog-1.12.0.zip" 26 | SOURCE_DIR ${SPDLOG_SOURCES_DIR} 27 | DOWNLOAD_NO_PROGRESS 1 28 | UPDATE_COMMAND "" 29 | CMAKE_ARGS 30 | ${EXTERNAL_PROJECT_C} 31 | ${EXTERNAL_PROJECT_CXX} 32 | ${EXTERNAL_PROJECT_CXX_FLAGS} 33 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 34 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 35 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 36 | -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 37 | -DSPDLOG_BUILD_EXAMPLE=OFF 38 | -DSPDLOG_FMT_EXTERNAL=OFF 39 | ${EXTERNAL_GENERATOR} 40 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 41 | BUILD_BYPRODUCTS ${SPDLOG_LIBRARIES} 42 | ) 43 | -------------------------------------------------------------------------------- /cmake/zlib.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(ZLIB_SOURCES_DIR "${LIB_SOURCE_DIR}/zlib" CACHE PATH "Path to zlib sources") 7 | SET(ZLIB_INCLUDE_DIR "${LIB_INCLUDE_DIR}" CACHE PATH "zlib include directory." FORCE) 8 | SET(ZLIB_LIBRARIES "${LIB_INSTALL_DIR}/libz.a" CACHE FILEPATH "zlib library." FORCE) 9 | 10 | ExternalProject_Add( 11 | zlib 12 | ${EXTERNAL_PROJECT_LOG_ARGS} 13 | GIT_REPOSITORY "https://github.com/madler/zlib.git" 14 | GIT_TAG "v1.2.8" 15 | GIT_SHALLOW true 16 | SOURCE_DIR ${ZLIB_SOURCES_DIR} 17 | GIT_SHALLOW true 18 | UPDATE_COMMAND "" 19 | CMAKE_ARGS 20 | ${EXTERNAL_PROJECT_C} 21 | ${EXTERNAL_PROJECT_CXX} 22 | ${EXTERNAL_PROJECT_CXX_FLAGS} 23 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 24 | # To be compatible with cmake 4.0 25 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 26 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 27 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 28 | -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 29 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON 30 | ${EXTERNAL_GENERATOR} 31 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 32 | BUILD_BYPRODUCTS ${ZLIB_LIBRARIES} 33 | ) 34 | -------------------------------------------------------------------------------- /cmake/zstd.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | SET(zstd_SOURCES_DIR "${LIB_SOURCE_DIR}/zstd" CACHE PATH "Path to zstd sources") 7 | SET(zstd_INCLUDE_DIRS "${LIB_INCLUDE_DIR}" CACHE PATH "zstd include directory." FORCE) 8 | SET(zstd_LIBRARIES "${LIB_INSTALL_DIR}/libzstd.a" CACHE FILEPATH "zstd include directory." FORCE) 9 | 10 | ExternalProject_Add( 11 | zstd 12 | ${EXTERNAL_PROJECT_LOG_ARGS} 13 | URL https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-1.5.4.tar.gz 14 | URL_HASH SHA256=0f470992aedad543126d06efab344dc5f3e171893810455787d38347343a4424 15 | DOWNLOAD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/download" 16 | DOWNLOAD_NAME "zstd-1.5.4.tar.gz" 17 | SOURCE_DIR ${zstd_SOURCES_DIR} 18 | DOWNLOAD_NO_PROGRESS 1 19 | SOURCE_SUBDIR build/cmake 20 | UPDATE_COMMAND "" 21 | CMAKE_ARGS 22 | ${EXTERNAL_PROJECT_C} 23 | ${EXTERNAL_PROJECT_CXX} 24 | ${EXTERNAL_PROJECT_CXX_FLAGS} 25 | ${EXTERNAL_PROJECT_CXX_LINK_FLAGS} 26 | # To be compatible with cmake 4.0 27 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 28 | -DCMAKE_INSTALL_PREFIX=${LIB_INSTALL_PREFIX} 29 | -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} 30 | -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} 31 | -DBUILD_TESTING=OFF 32 | -DZSTD_BUILD_STATIC=ON 33 | -DZSTD_BUILD_SHARED=OFF 34 | ${EXTERNAL_GENERATOR} 35 | BUILD_COMMAND ${EXTERNAL_BUILD} -j${CPU_CORE} 36 | BUILD_BYPRODUCTS ${zstd_LIBRARIES} 37 | ) 38 | -------------------------------------------------------------------------------- /etc/script/check_format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Check if clang-format is installed 4 | if ! command -v clang-format &> /dev/null; then 5 | echo "Error: clang-format is not installed" 6 | echo "Please install clang-format first" 7 | exit 1 8 | fi 9 | 10 | # Set color output 11 | RED='\033[0;31m' 12 | GREEN='\033[0;32m' 13 | NC='\033[0m' # No Color 14 | 15 | # Counters 16 | error_count=0 17 | checked_count=0 18 | 19 | # Check file 20 | check_file() { 21 | local file=$1 22 | # Create a temporary file with formatted content 23 | clang-format $file > temp.txt 24 | 25 | # Compare the original file with the formatted file 26 | if ! diff -u "$file" temp.txt > /dev/null; then 27 | echo -e "${RED}Needs formatting: $file${NC}" 28 | # Show specific differences 29 | diff -u "$file" temp.txt | grep -E "^[\+\-]" | head -n 10 30 | echo "..." 31 | ((error_count++)) 32 | else 33 | echo -e "${GREEN}Correctly formatted: $file${NC}" 34 | fi 35 | ((checked_count++)) 36 | 37 | # Clean up temporary file 38 | rm temp.txt 39 | } 40 | 41 | # Main function 42 | main() { 43 | echo "Starting code format check..." 44 | 45 | # Find all C/C++ source files 46 | find ./src -type f \( -name "*.cpp" -o -name "*.hpp" -o -name "*.c" -o -name "*.h" -o -name "*.cc" \) -print0 | while IFS= read -r -d '' file; do 47 | check_file "$file" 48 | done 49 | 50 | # Output summary 51 | echo "----------------------------------------" 52 | echo "Check completed!" 53 | echo "Total files checked: $checked_count" 54 | echo "Files needing formatting: $error_count" 55 | 56 | # Return non-zero if there are errors 57 | [ "$error_count" -gt 0 ] && exit 1 || exit 0 58 | } 59 | 60 | # Run the main function 61 | main 62 | -------------------------------------------------------------------------------- /etc/script/kiwitests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # clear the log file 4 | function cleanup() { 5 | rm -rf ./logs* 6 | rm -rf ./db* 7 | rm -rf dbsync/ 8 | rm src/redis-server 9 | } 10 | 11 | # check if tcl is installed 12 | function check_tcl { 13 | if [ -z "$(which tclsh)" ]; then 14 | echo "tclsh is not installed" 15 | exit 1 16 | fi 17 | } 18 | 19 | # handle different build directories. 20 | function setup_build_dir { 21 | BUILD_DIR="./bin" 22 | echo "BUILD_DIR: $BUILD_DIR" 23 | } 24 | 25 | # setup kiwi bin and conf 26 | function setup_kiwi_bin { 27 | kiwi_BIN="./$BUILD_DIR/kiwi" 28 | if [ ! -f "$kiwi_BIN" ]; then 29 | echo "kiwi bin not found" 30 | exit 1 31 | fi 32 | cp $kiwi_BIN src/redis-server 33 | cp ./etc/conf/kiwi.conf tests/assets/default.conf 34 | } 35 | 36 | 37 | cleanup 38 | 39 | check_tcl 40 | 41 | setup_build_dir 42 | 43 | setup_kiwi_bin 44 | 45 | echo "run kiwi tests $1" 46 | 47 | if [ "$1" == "all" ]; then 48 | tclsh tests/test_helper.tcl --clients 1 49 | else 50 | tclsh tests/test_helper.tcl --clients 1 --single unit/$1 51 | fi 52 | 53 | if [ $? -ne 0 ]; then 54 | echo "kiwi tests failed" 55 | cleanup 56 | exit 1 57 | fi 58 | 59 | # You can use './kiwi.sh all clean 'to ensure that the 60 | # data can be deleted immediately after the test 61 | if [ "$2" == "clean" ]; then 62 | cleanup 63 | fi -------------------------------------------------------------------------------- /etc/script/precommit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Path to the check_format.sh script 4 | SCRIPT_PATH="etc/script/check_format.sh" 5 | 6 | # Check if the script exists 7 | if [ ! -f "$SCRIPT_PATH" ]; then 8 | echo "Error: $SCRIPT_PATH not found!" 9 | exit 1 10 | fi 11 | 12 | # Make sure the script is executable 13 | chmod +x "$SCRIPT_PATH" 14 | 15 | # Run the script on all staged files 16 | #STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep -E '\.(cpp|cc|h|hpp|c)$') 17 | 18 | # if [ -n "$STAGED_FILES" ]; then 19 | # for file in $STAGED_FILES; do 20 | # "$SCRIPT_PATH" "$file" 21 | # if [ $? -ne 0 ]; then 22 | # echo "Commit aborted due to formatting issues." 23 | # exit 1 24 | # fi 25 | # done 26 | # fi 27 | 28 | sh $SCRIPT_PATH 29 | 30 | exit 0 # Path to the check_format.sh script 31 | 32 | -------------------------------------------------------------------------------- /etc/script/save_load.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | killall -9 kiwi 4 | mkdir leader follower1 5 | 6 | 7 | PWD=`pwd` 8 | PROJECT_HOME="${PWD}/../" 9 | BIN="${PROJECT_HOME}/bin/kiwi" 10 | CONF="${PROJECT_HOME}/etc/conf/kiwi.conf" 11 | cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & 12 | cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & 13 | sleep 5 14 | 15 | redis-cli -p 7777 raft.cluster init 16 | 17 | redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset 18 | redis-cli -p 7777 raft.node dosnapshot 19 | redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset 20 | redis-cli -p 7777 raft.node dosnapshot 21 | redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset 22 | 23 | redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 24 | -------------------------------------------------------------------------------- /src/client_map.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "client.h" 8 | 9 | namespace kiwi { 10 | class ClientMap { 11 | private: 12 | ClientMap() = default; 13 | // 禁用复制构造函数和赋值运算符 14 | 15 | private: 16 | std::map> clients_; 17 | std::shared_mutex client_map_mutex_; 18 | 19 | public: 20 | static ClientMap& getInstance() { 21 | static ClientMap instance; 22 | return instance; 23 | } 24 | 25 | ClientMap(const ClientMap&) = delete; 26 | ClientMap& operator=(const ClientMap&) = delete; 27 | 28 | // client info function 29 | kiwi::ClientInfo GetClientsInfoById(int id); 30 | uint32_t GetAllClientInfos(std::vector& results); 31 | 32 | bool AddClient(int id, std::weak_ptr); 33 | 34 | bool RemoveClientById(int id); 35 | 36 | bool KillAllClients(); 37 | bool KillClientById(int client_id); 38 | bool KillClientByAddrPort(const std::string& addr_port); 39 | }; 40 | 41 | } // namespace kiwi 42 | -------------------------------------------------------------------------------- /src/cmd_raft.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | The declarations of a set of instructions and functions 8 | related to Raft are written here. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | 15 | #include "base_cmd.h" 16 | 17 | namespace kiwi { 18 | 19 | /* RAFT.NODE ADD [id] [address:port] 20 | * Add a new node to the cluster. The [id] can be an explicit non-zero value, 21 | * or zero to let the cluster choose one. 22 | * Reply: 23 | * -NOCLUSTER || 24 | * -LOADING || 25 | * -CLUSTERDOWN || 26 | * -MOVED : || 27 | * *2 28 | * : 29 | * : 30 | * 31 | * RAFT.NODE REMOVE [id] 32 | * Remove an existing node from the cluster. 33 | * Reply: 34 | * -NOCLUSTER || 35 | * -LOADING || 36 | * -CLUSTERDOWN || 37 | * -MOVED : || 38 | * +OK 39 | */ 40 | class RaftNodeCmd : public BaseCmd { 41 | public: 42 | RaftNodeCmd(const std::string &name, int16_t arity); 43 | 44 | protected: 45 | bool DoInitial(PClient *client) override; 46 | 47 | private: 48 | void DoCmd(PClient *client) override; 49 | void DoCmdAdd(PClient *client); 50 | void DoCmdRemove(PClient *client); 51 | void DoCmdSnapshot(PClient *client); 52 | 53 | static constexpr std::string_view kAddCmd = "ADD"; 54 | static constexpr std::string_view kRemoveCmd = "REMOVE"; 55 | static constexpr std::string_view kDoSnapshot = "DOSNAPSHOT"; 56 | }; 57 | 58 | /* RAFT.CLUSTER INIT 59 | * Initializes a new Raft cluster. 60 | * is an optional 32 character string, if set, cluster will use it for the id 61 | * Reply: 62 | * +OK [group_id] 63 | * 64 | * RAFT.CLUSTER JOIN [addr:port] 65 | * Join an existing cluster. 66 | * The operation is asynchronous and may take place/retry in the background. 67 | * Reply: 68 | * +OK 69 | */ 70 | class RaftClusterCmd : public BaseCmd { 71 | public: 72 | RaftClusterCmd(const std::string &name, int16_t arity); 73 | 74 | protected: 75 | bool DoInitial(PClient *client) override; 76 | 77 | private: 78 | void DoCmd(PClient *client) override; 79 | void DoCmdInit(PClient *client); 80 | void DoCmdJoin(PClient *client); 81 | 82 | static constexpr std::string_view kInitCmd = "INIT"; 83 | static constexpr std::string_view kJoinCmd = "JOIN"; 84 | }; 85 | 86 | } // namespace kiwi 87 | -------------------------------------------------------------------------------- /src/cmd_table_manager.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Defined a command table, because kiwi needs to manage 8 | commands in an integrated way. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "base_cmd.h" 21 | 22 | namespace kiwi { 23 | 24 | using CmdTable = std::unordered_map>; 25 | 26 | class CmdTableManager { 27 | public: 28 | CmdTableManager(); 29 | ~CmdTableManager() = default; 30 | 31 | public: 32 | void InitCmdTable(); 33 | std::pair GetCommand(const std::string& cmdName, PClient* client); 34 | // uint32_t DistributeKey(const std::string& key, uint32_t slot_num); 35 | bool CmdExist(const std::string& cmd) const; 36 | uint32_t GetCmdId(); 37 | 38 | private: 39 | std::unique_ptr cmds_; 40 | 41 | uint32_t cmdId_ = 0; 42 | 43 | mutable std::shared_mutex mutex_; 44 | }; 45 | 46 | } // namespace kiwi 47 | -------------------------------------------------------------------------------- /src/cmd_thread_pool_worker.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Defined a set of functions for retrieving commands from 8 | the thread pool and executing them. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | 16 | #include "cmd_table_manager.h" 17 | #include "cmd_thread_pool.h" 18 | 19 | namespace kiwi { 20 | 21 | class CmdWorkThreadPoolWorker { 22 | public: 23 | explicit CmdWorkThreadPoolWorker(CmdThreadPool *pool, int onceTask, std::string name) 24 | : pool_(pool), once_task_(onceTask), name_(std::move(name)) { 25 | cmd_table_manager_.InitCmdTable(); 26 | } 27 | 28 | void Work(); 29 | 30 | void Stop(); 31 | 32 | // load the task from the thread pool 33 | virtual void LoadWork() = 0; 34 | 35 | virtual ~CmdWorkThreadPoolWorker() = default; 36 | 37 | protected: 38 | std::vector> self_task_; // the task that the worker get from the thread pool 39 | CmdThreadPool *pool_ = nullptr; 40 | const int once_task_ = 0; // the max task num that the worker can get from the thread pool 41 | const std::string name_; 42 | bool running_ = true; 43 | 44 | kiwi::CmdTableManager cmd_table_manager_; 45 | }; 46 | 47 | // fast worker 48 | class CmdFastWorker : public CmdWorkThreadPoolWorker { 49 | public: 50 | explicit CmdFastWorker(CmdThreadPool *pool, int onceTask, std::string name) 51 | : CmdWorkThreadPoolWorker(pool, onceTask, std::move(name)) {} 52 | 53 | void LoadWork() override; 54 | }; 55 | 56 | // slow worker 57 | class CmdSlowWorker : public CmdWorkThreadPoolWorker { 58 | public: 59 | explicit CmdSlowWorker(CmdThreadPool *pool, int onceTask, std::string name) 60 | : CmdWorkThreadPoolWorker(pool, onceTask, std::move(name)) {} 61 | 62 | // when the slow worker queue is empty, it will try to get the fast worker 63 | void LoadWork() override; 64 | 65 | private: 66 | bool loop_more_ = false; // When the slow queue is empty, try to get the fast queue 67 | int wait_time_ = 200; // When the slow queue is empty, wait 200 ms to check again 68 | }; 69 | 70 | } // namespace kiwi 71 | -------------------------------------------------------------------------------- /src/config_parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | * config_parser.h 3 | * Declared a set of functions for parsing configuration data 4 | * as required. 5 | * 6 | * Copyright (c) 2023-present, arana-db Community All rights reserved. 7 | * 8 | * src/config_parser.h 9 | * 10 | */ 11 | 12 | #pragma once 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #ifdef CONFIG_DEBUG 20 | # include 21 | #endif 22 | 23 | namespace kiwi { 24 | 25 | class ConfigParser { 26 | public: 27 | using Data = std::map>; 28 | 29 | bool Load(const char* FileName); 30 | 31 | template 32 | T GetData(const char* key, const T& default_ = T()) const; 33 | 34 | const std::vector& GetDataVector(const char* key) const; 35 | 36 | const Data& GetMap() { return data_; } 37 | 38 | #ifdef CONFIG_DEBUG 39 | void Print() { 40 | std::cout << "//////////////////" << std::endl; 41 | std::map::const_iterator it = data_.begin(); 42 | while (it != data_.end()) { 43 | std::cout << it->first << ":" << it->second << "\n"; 44 | ++it; 45 | } 46 | } 47 | #endif 48 | 49 | private: 50 | Data data_; 51 | 52 | template 53 | T toType(const std::string& data) const; 54 | }; 55 | 56 | template 57 | inline T ConfigParser::toType(const std::string& data) const { 58 | T t; 59 | std::istringstream os(data); 60 | os >> t; 61 | return t; 62 | } 63 | 64 | template <> 65 | inline const char* ConfigParser::toType(const std::string& data) const { 66 | return data.c_str(); 67 | } 68 | 69 | template <> 70 | inline std::string ConfigParser::toType(const std::string& data) const { 71 | return data; 72 | } 73 | 74 | template 75 | inline T ConfigParser::GetData(const char* key, const T& default_) const { 76 | auto it = data_.find(key); 77 | if (it == data_.end()) { 78 | return default_; 79 | } 80 | 81 | return toType(it->second[0]); // only return first value 82 | } 83 | 84 | } // namespace kiwi 85 | -------------------------------------------------------------------------------- /src/db.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Declared a set of functions responsible for interfacing 8 | with RocksDB. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | 16 | #include "std/log.h" 17 | #include "std/noncopyable.h" 18 | #include "storage/storage.h" 19 | 20 | namespace kiwi { 21 | 22 | class DB { 23 | public: 24 | DB(int db_index, const std::string& db_path); 25 | ~DB(); 26 | 27 | rocksdb::Status Open(); 28 | 29 | std::unique_ptr& GetStorage() { return storage_; } 30 | 31 | void Lock() { storage_mutex_.lock(); } 32 | 33 | void UnLock() { storage_mutex_.unlock(); } 34 | 35 | void LockShared() { storage_mutex_.lock_shared(); } 36 | 37 | void UnLockShared() { storage_mutex_.unlock_shared(); } 38 | 39 | void CreateCheckpoint(const std::string& path, bool sync); 40 | 41 | void LoadDBFromCheckpoint(const std::string& path, bool sync = true); 42 | 43 | int GetDbIndex() { return db_index_; } 44 | 45 | private: 46 | const int db_index_ = 0; 47 | const std::string db_path_; 48 | /** 49 | * If you want to change the pointer that points to storage, 50 | * you must first acquire a mutex lock. 51 | * If you only want to access the pointer, 52 | * you just need to obtain a shared lock. 53 | */ 54 | std::shared_mutex storage_mutex_; 55 | std::unique_ptr storage_; 56 | bool opened_ = false; 57 | }; 58 | 59 | } // namespace kiwi 60 | -------------------------------------------------------------------------------- /src/kiwi_logo.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Responsible for defining the text form of the kiwi logo, 8 | the output of this LOGO must be combined with the definition 9 | in practice. 10 | */ 11 | 12 | #pragma once 13 | 14 | const char* kiwiLogo = 15 | "\n _ _ __ _ \n" 16 | " ( )( ) / )( ) _ _ \n" 17 | " _ _ _ __ _ _ ___ _ _ ______ _| || |_ /' /' | |/') (_) _ _ _ (_) \n" 18 | " /'_` )( '__)/'_` )/' _ `\\ /'_` )(______)/'_` || '_`\\ /' /' | , < | |( ) ( ) ( )| | \n" 19 | " ( (_| || | ( (_| || ( ) |( (_| | ( (_| || |_) ) /' /' | |\\`\\ | || \\_/ \\_/ || | \n" 20 | " `\\__,_)(_) `\\__,_)(_) (_)`\\__,_) `\\__,_)(_,__/'(_/' (_) (_)(_)`\\___x___/'(_) \n" 21 | "Kiwi(%s) %d bits \n" 22 | "Port: %d\n" 23 | "Github: https://github.com/arana-db/kiwi\n\n\n"; 24 | -------------------------------------------------------------------------------- /src/net/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 3.25) 7 | 8 | PROJECT(net) 9 | 10 | SET(CMAKE_CXX_STANDARD 20) 11 | 12 | AUX_SOURCE_DIRECTORY(. NET_SRC) 13 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 14 | 15 | ADD_LIBRARY(net ${NET_SRC}) 16 | 17 | TARGET_INCLUDE_DIRECTORIES(net 18 | PRIVATE ${PSTD_INCLUDE_DIR} 19 | PRIVATE ${FMT_INCLUDE_DIR} 20 | ) 21 | 22 | TARGET_LINK_LIBRARIES(net kstd) 23 | -------------------------------------------------------------------------------- /src/net/base_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | 12 | #include "net_event.h" 13 | #include "socket_addr.h" 14 | 15 | namespace net { 16 | 17 | static constexpr int SOCKET_WIN_SIZE = 128 * 1024; 18 | 19 | class BaseSocket : public NetEvent { 20 | public: 21 | enum { 22 | SOCKET_NONE = 0, // error socket 23 | SOCKET_TCP, 24 | SOCKET_UDP, 25 | SOCKET_LISTEN_TCP, 26 | SOCKET_LISTEN_UDP, 27 | }; 28 | 29 | explicit BaseSocket(int fd) : NetEvent(fd) {} 30 | 31 | ~BaseSocket() override = default; 32 | 33 | void OnError() override {}; 34 | 35 | void Close() override; 36 | 37 | static int CreateTCPSocket(const SocketAddr &addr); 38 | 39 | static int CreateUDPSocket(); 40 | 41 | // Called when the socket is created 42 | void OnCreate(); 43 | 44 | void SetNonBlock(bool noBlock); 45 | 46 | void SetNodelay(); 47 | 48 | void SetTcpKeepAlive(); 49 | 50 | void SetSndBuf(socklen_t size = SOCKET_WIN_SIZE); 51 | 52 | void SetRcvBuf(socklen_t size = SOCKET_WIN_SIZE); 53 | 54 | void SetReuseAddr(); 55 | 56 | bool SetReusePort(); 57 | 58 | bool GetLocalAddr(SocketAddr &); 59 | 60 | bool GetPeerAddr(SocketAddr &); 61 | 62 | int SocketType() { return type_; } 63 | 64 | void SetSocketType(int type) { type_ = type; } 65 | 66 | void SetBSTcpKeepAlive(uint32_t keep_alive) { tcp_keep_alive_ = keep_alive; } 67 | 68 | protected: 69 | bool NoBlock() const { return noBlock_; } 70 | 71 | private: 72 | int type_ = SOCKET_NONE; // socket type (TCP/UDP) 73 | bool noBlock_ = true; 74 | uint32_t tcp_keep_alive_ = 300; // TCP keepalive 75 | }; 76 | 77 | } // namespace net 78 | -------------------------------------------------------------------------------- /src/net/callback_function.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include "socket_addr.h" 13 | 14 | namespace net { 15 | 16 | template 17 | struct IsPointer : std::false_type {}; 18 | 19 | template 20 | struct IsPointer : std::true_type {}; 21 | 22 | template 23 | struct IsPointer> : std::true_type {}; 24 | 25 | template 26 | struct IsPointer> : std::true_type {}; 27 | 28 | template 29 | constexpr bool IsPointer_v = IsPointer::value; 30 | 31 | template 32 | concept HasSetFdFunction = requires(T t, uint64_t id, int8_t index) { 33 | // If T is of pointer type, then dereference and call the member function 34 | { (*t).SetConnId(id) } -> std::same_as; // SetFd return type is void 35 | { (*t).GetConnId() } -> std::same_as; // GetFd return type is int 36 | { (*t).SetThreadIndex(index) } -> std::same_as; // SetThreadIndex return type is void 37 | { (*t).GetThreadIndex() } -> std::same_as; // GetThreadIndex return type is int8_t 38 | } || std::is_class_v; // If T is an ordinary class, the member function is called directly 39 | 40 | template 41 | requires HasSetFdFunction 42 | using OnInit = std::function; 43 | 44 | template 45 | requires HasSetFdFunction 46 | using OnCreate = std::function; 47 | 48 | template 49 | requires HasSetFdFunction 50 | using OnMessage = std::function; 51 | 52 | template 53 | requires HasSetFdFunction 54 | using OnClose = std::function; 55 | 56 | } // namespace net 57 | -------------------------------------------------------------------------------- /src/net/client_socket.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #include 9 | 10 | #include 11 | #include "client_socket.h" 12 | 13 | namespace net { 14 | 15 | bool ClientSocket::Connect() { 16 | fd_ = CreateTCPSocket(addr_); 17 | if (fd_ == -1) { 18 | onConnectFail_("CreateTCPSocket open socket failed"); 19 | return false; 20 | } 21 | SetNonBlock(true); 22 | SetNodelay(); 23 | SetRcvBuf(); 24 | SetSndBuf(); 25 | 26 | auto ret = connect(Fd(), addr_.Get(), addr_.Len()); 27 | if (0 != ret) { 28 | if (EINPROGRESS == errno) { 29 | return true; 30 | } 31 | 32 | std::ostringstream oss; 33 | oss << "IP:" << addr_.GetIP() << " port:" << addr_.GetPort() << " connect failed with error: " << strerror(errno); 34 | Close(); 35 | onConnectFail_(oss.str()); 36 | return false; 37 | } 38 | 39 | return true; 40 | } 41 | 42 | } // namespace net 43 | -------------------------------------------------------------------------------- /src/net/client_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "stream_socket.h" 11 | 12 | namespace net { 13 | 14 | class ClientSocket : public StreamSocket { 15 | public: 16 | explicit ClientSocket(const SocketAddr& addr) : StreamSocket(0, SOCKET_TCP), addr_(addr) {}; 17 | 18 | ~ClientSocket() override = default; 19 | 20 | bool Connect(); 21 | 22 | void SetFailCallback(const std::function& cb) { onConnectFail_ = cb; } 23 | 24 | private: 25 | SocketAddr addr_; 26 | std::function onConnectFail_; 27 | }; 28 | 29 | } // namespace net 30 | -------------------------------------------------------------------------------- /src/net/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | namespace net { 11 | 12 | #ifdef __linux__ 13 | # define HAVE_EPOLL 1 14 | #endif 15 | 16 | #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) 17 | # define HAVE_KQUEUE 1 18 | #endif 19 | 20 | #ifdef __linux__ 21 | # define HAVE_ACCEPT4 1 22 | #endif 23 | 24 | #if defined(__x86_64__) || defined(_M_X64) || defined(__ppc64__) || defined(__aarch64__) || defined(__64BIT__) || \ 25 | defined(_LP64) || defined(__LP64__) || defined(__arm64__) 26 | # define HAVE_64BIT 1 27 | #else 28 | # define HAVE_32BIT 1 29 | #endif 30 | 31 | } // namespace net 32 | -------------------------------------------------------------------------------- /src/net/connection.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | 12 | #include "socket_addr.h" 13 | 14 | namespace net { 15 | 16 | class NetEvent; 17 | 18 | // Auxiliary structure 19 | struct Connection { 20 | explicit Connection(std::unique_ptr net_event) : net_event_(std::move(net_event)) {} 21 | Connection(const Connection&) = delete; 22 | Connection& operator=(const Connection&) = delete; 23 | ~Connection() = default; 24 | 25 | std::unique_ptr net_event_; 26 | 27 | SocketAddr addr_; 28 | 29 | uint64_t conn_id_ = 0; 30 | 31 | int fd_ = 0; 32 | }; 33 | 34 | } // namespace net -------------------------------------------------------------------------------- /src/net/epoll_event.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "config.h" 11 | #include "listen_socket.h" 12 | 13 | #ifdef HAVE_EPOLL 14 | 15 | # include 16 | # include 17 | # include 18 | # include 19 | 20 | # include "base_event.h" 21 | 22 | namespace net { 23 | 24 | class EpollEvent : public BaseEvent { 25 | public: 26 | explicit EpollEvent(const std::vector> &listen_sockets, int8_t mode) 27 | : BaseEvent(listen_sockets, mode, BaseEvent::EVENT_TYPE_EPOLL) {}; 28 | 29 | ~EpollEvent() override { Close(); } 30 | 31 | // Initialize the epoll event 32 | bool Init() override; 33 | 34 | // add fd to poll 35 | void AddEvent(int fd, int mask) const; 36 | 37 | // Add event to epoll, mask is the event type 38 | void AddEvent(Connection *conn, int mask) override; 39 | 40 | // Delete event from epoll 41 | void DelEvent(int fd) override; 42 | 43 | // Poll event 44 | void EventPoll() override; 45 | 46 | // Add write event to epoll 47 | void AddWriteEvent(Connection *conn) override; 48 | 49 | // Delete write event from epoll 50 | void DelWriteEvent(Connection *conn) override; 51 | 52 | // Handle read event 53 | void EventRead(); 54 | 55 | // Handle write event 56 | void EventWrite(); 57 | 58 | // Do read event 59 | void DoRead(const epoll_event &event, Connection *conn, const std::shared_ptr &listen); 60 | 61 | // Do write event 62 | void DoWrite(const epoll_event &event, Connection *conn); 63 | 64 | // Handle error event 65 | void DoError(const epoll_event &event, std::string &&err); 66 | 67 | private: 68 | const int eventsSize = 1024; 69 | }; 70 | 71 | } // namespace net 72 | #endif 73 | -------------------------------------------------------------------------------- /src/net/event_server.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #include "event_server.h" 9 | 10 | namespace net { 11 | 12 | std::atomic g_connId = 0; 13 | 14 | uint64_t getConnId() { return g_connId.fetch_add(1); } 15 | 16 | } // namespace net 17 | -------------------------------------------------------------------------------- /src/net/io_thread.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #include "io_thread.h" 9 | 10 | namespace net { 11 | 12 | void IOThread::Stop() { 13 | bool run = true; 14 | if (running_.compare_exchange_strong(run, false)) { 15 | baseEvent_->Close(); 16 | Wait(); 17 | } 18 | } 19 | 20 | void IOThread::Wait() { 21 | if (thread_.joinable()) { 22 | thread_.join(); 23 | } 24 | } 25 | 26 | bool IOThread::Run() { 27 | if (!baseEvent_->Init()) { 28 | return false; 29 | } 30 | 31 | thread_ = std::thread([this] { baseEvent_->EventPoll(); }); 32 | return true; 33 | } 34 | 35 | } // namespace net 36 | -------------------------------------------------------------------------------- /src/net/io_thread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | 13 | #include "base_event.h" 14 | 15 | namespace net { 16 | 17 | class IOThread { 18 | public: 19 | explicit IOThread(const std::shared_ptr &event) : baseEvent_(event) {}; 20 | 21 | ~IOThread() = default; 22 | 23 | // Initialize the event and run the event loop 24 | bool Run(); 25 | 26 | void CloseConnection(int fd) { baseEvent_->DelEvent(fd); } 27 | 28 | // Stop the event loop and wait for the thread to exit 29 | void Stop(); 30 | 31 | // Wait for the thread to exit 32 | void Wait(); 33 | 34 | // Add read event to epoll when send message to client 35 | void SetWriteEvent(Connection *conn) { baseEvent_->AddWriteEvent(conn); } 36 | 37 | // Add new event to epoll when new connection 38 | void AddNewEvent(Connection *conn, int mask) { baseEvent_->AddEvent(conn, mask); } 39 | 40 | protected: 41 | std::atomic running_ = true; 42 | 43 | std::thread thread_; 44 | 45 | std::shared_ptr baseEvent_; // Event object 46 | }; 47 | 48 | } // namespace net 49 | -------------------------------------------------------------------------------- /src/net/kqueue_event.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "config.h" 11 | 12 | #ifdef HAVE_KQUEUE 13 | 14 | # include 15 | # include 16 | # include 17 | # include 18 | # include 19 | 20 | # include "base_event.h" 21 | 22 | namespace net { 23 | 24 | class KqueueEvent : public BaseEvent { 25 | public: 26 | explicit KqueueEvent(const std::vector> &listen_sockets, int8_t mode) 27 | : BaseEvent(std::move(listen_sockets), mode, BaseEvent::EVENT_TYPE_KQUEUE) {}; 28 | 29 | ~KqueueEvent() override { Close(); } 30 | 31 | bool Init() override; 32 | 33 | void AddEvent(int fd, int mask) const; 34 | 35 | void AddEvent(Connection *conn, int mask) override; 36 | 37 | void DelEvent(int fd) override; 38 | 39 | void AddWriteEvent(Connection *conn) override; 40 | 41 | void DelWriteEvent(Connection *conn) override; 42 | 43 | void EventPoll() override; 44 | 45 | void EventRead(); 46 | 47 | void EventWrite(); 48 | 49 | void DoRead(const struct kevent &event, Connection *conn, const std::shared_ptr &listen); 50 | 51 | void DoWrite(const struct kevent &event, Connection *conn); 52 | 53 | void DoError(const struct kevent &event, std::string &&err); 54 | 55 | private: 56 | const int eventsSize = 1024; 57 | }; 58 | 59 | } // namespace net 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/net/listen_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include "base_socket.h" 15 | 16 | namespace net { 17 | 18 | class ListenSocket : public BaseSocket { 19 | public: 20 | static ListenSocket *CreateTCPListen() { return new ListenSocket(SOCKET_LISTEN_TCP); } 21 | 22 | static ListenSocket *CreateUDPListen() { return new ListenSocket(SOCKET_LISTEN_UDP); } 23 | 24 | static const int LISTENQ; 25 | static bool REUSE_PORT; // Determine whether REUSE_PORT can be used 26 | 27 | void SetListenAddr(const SocketAddr &addr) { addr_ = addr; } 28 | 29 | SocketAddr GetListenAddr() const { return addr_; } 30 | 31 | // Accept new connection and create new connection object 32 | // when the connection is established, the OnCreate function is called 33 | int OnReadable(Connection *conn, std::string *readBuff) override; 34 | 35 | // The function is cant be used 36 | int OnWritable(Connection *conn, BaseEvent *event) override; 37 | 38 | // The function is cant be used 39 | void SendPacket(std::string &&msg, std::function addWriteFlag) override; 40 | 41 | // Initialize the socket and bind the address 42 | int Init() override; 43 | 44 | private: 45 | explicit ListenSocket(int type) : BaseSocket(0) { SetSocketType(type); } 46 | 47 | // Open the socket 48 | bool Open(); 49 | 50 | // Bind the address 51 | bool Bind(); 52 | 53 | // Start listening 54 | bool Listen(); 55 | 56 | private: 57 | // Accept new connection 58 | int Accept(sockaddr_in *clientAddr); 59 | 60 | SocketAddr addr_; // Listen address 61 | }; 62 | 63 | } // namespace net 64 | -------------------------------------------------------------------------------- /src/net/net_event.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | 12 | #include "callback_function.h" 13 | #include "connection.h" 14 | 15 | namespace net { 16 | 17 | // For human readability 18 | enum : std::int8_t { 19 | NE_ERROR = -1, 20 | NE_CLOSE = -2, 21 | NE_WAIT = -3, 22 | NE_OK = 0, 23 | }; 24 | 25 | enum class NetListen : std::uint8_t { 26 | OK = 0, 27 | OPEN_ERROR, 28 | BIND_ERROR, 29 | LISTEN_ERROR, 30 | }; 31 | 32 | class BaseEvent; 33 | 34 | // abstraction of all networks 35 | class NetEvent { 36 | public: 37 | explicit NetEvent(int fd) : fd_(fd) {} 38 | 39 | virtual ~NetEvent() = default; 40 | 41 | // Initialize the event 42 | virtual int Init() = 0; 43 | 44 | // Handle read event when the connection is readable and the data can be read 45 | virtual int OnReadable(Connection *conn, std::string *readBuff) = 0; 46 | 47 | // Handle write event when the connection is writable and the data can be sent 48 | virtual int OnWritable(Connection *conn, BaseEvent *event) = 0; 49 | 50 | virtual void OnError() = 0; 51 | 52 | // Send data 53 | virtual void SendPacket(std::string &&msg, std::function addWriteFlag) = 0; 54 | 55 | virtual void Close() = 0; 56 | 57 | int Fd() const { return fd_.load(); } 58 | 59 | protected: 60 | std::atomic fd_ = 0; 61 | }; 62 | 63 | } // namespace net 64 | -------------------------------------------------------------------------------- /src/net/net_options.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | namespace net { 11 | 12 | class NetOptions { 13 | public: 14 | NetOptions() = default; 15 | ~NetOptions() = default; 16 | 17 | void SetThreadNum(int8_t number) { thread_num_ = number; } 18 | 19 | int8_t GetThreadNum() const { return thread_num_; } 20 | 21 | void SetRwSeparation(bool spearation = true) { rw_separation_ = spearation; } 22 | 23 | bool GetRwSeparation() const { return rw_separation_; } 24 | 25 | void SetMaxClients(uint32_t maxClients) { max_clients_ = maxClients; } 26 | 27 | uint32_t GetMaxClients() const { return max_clients_; } 28 | void SetOpTcpKeepAlive(uint32_t tcpKeepAlive) { tcp_keepalive_timeout_ = tcpKeepAlive; } 29 | 30 | uint32_t GetOpTcpKeepAlive() const { return tcp_keepalive_timeout_; } 31 | 32 | private: 33 | bool rw_separation_ = true; // Whether to separate read and write 34 | 35 | int8_t thread_num_ = 1; // The number of threads 36 | uint32_t max_clients_ = 1; // The maximum number of connections(default 40000) 37 | uint32_t tcp_keepalive_timeout_ = 300; // The timeout of the keepalive connection in seconds 38 | }; 39 | 40 | } // namespace net 41 | -------------------------------------------------------------------------------- /src/net/stream_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "base_socket.h" 18 | #include "lock_free_ring_buffer.h" 19 | 20 | namespace net { 21 | 22 | class StreamSocket : public BaseSocket { 23 | public: 24 | StreamSocket(int fd, int type) : BaseSocket(fd) { SetSocketType(type); } 25 | 26 | int Init() override { return 1; }; 27 | 28 | int OnReadable(Connection *conn, std::string *readBuff) override; 29 | 30 | int OnWritable(Connection *conn, BaseEvent *event) override; 31 | 32 | void SendPacket(std::string &&msg, std::function addWriteFlag) override; 33 | 34 | int Read(std::string *readBuff); 35 | 36 | private: 37 | const int readBuffSize_ = 4 * 1024; // read from socket buff size 4K 38 | 39 | std::string sendData_; // send data buff 40 | 41 | LockFreeRingBuffer writeQueue_{8}; // write data queue 42 | 43 | size_t sendPos_ = 0; // send data buff pos 44 | 45 | std::atomic writeReady_ = false; // write ready flag 46 | std::mutex write_mutex_; // write mutex 47 | }; 48 | 49 | } // namespace net 50 | -------------------------------------------------------------------------------- /src/net/timer.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #include "timer.h" 9 | 10 | namespace net { 11 | 12 | int64_t Timer::AddTask(const std::shared_ptr& task) { 13 | if (!task) { 14 | return -1; 15 | } 16 | std::unique_lock l(lock_); 17 | int64_t _taskId = TaskId(); 18 | task->SetId(_taskId); 19 | queue_.push(task); 20 | return _taskId; 21 | } 22 | 23 | void Timer::RePushTask() { 24 | for (const auto& task : list_) { 25 | task->Next(); 26 | { queue_.push(task); } 27 | } 28 | list_.clear(); 29 | } 30 | 31 | void Timer::PopTask(std::shared_ptr& task, bool deleted) { 32 | if (!task) { 33 | return; 34 | } 35 | if (deleted) { 36 | DelMark(task->Id()); 37 | } else { 38 | list_.emplace_back(task); 39 | } 40 | 41 | queue_.pop(); 42 | } 43 | 44 | void Timer::OnTimer() { 45 | std::unique_lock l(lock_); 46 | auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) 47 | .count(); 48 | 49 | while (!queue_.empty()) { 50 | auto task = queue_.top(); 51 | if (Deleted(task->Id())) { 52 | PopTask(task, true); 53 | continue; 54 | } 55 | if (now >= task->Start()) { 56 | task->TimeOut(); 57 | PopTask(task, false); 58 | } else { 59 | break; 60 | } 61 | } 62 | 63 | RePushTask(); // reload the task 64 | } 65 | 66 | } // namespace net 67 | -------------------------------------------------------------------------------- /src/net/timer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "timer_task.h" 20 | 21 | namespace net { 22 | 23 | class Timer { 24 | public: 25 | explicit Timer(int64_t interval) : interval_(interval) {} 26 | 27 | ~Timer() = default; 28 | 29 | private: 30 | std::shared_mutex lock_; 31 | std::shared_mutex lockSet_; 32 | int64_t interval_; 33 | int64_t taskId_ = 0; 34 | std::set markDel; 35 | std::priority_queue> queue_; 36 | std::vector> list_; 37 | 38 | public: 39 | int64_t Interval() const { return interval_; } 40 | 41 | int64_t AddTask(const std::shared_ptr& task); 42 | 43 | void DelTask(int64_t taskId) { 44 | std::unique_lock l(lockSet_); 45 | markDel.insert(taskId); 46 | } 47 | 48 | void OnTimer(); 49 | 50 | private: 51 | int64_t TaskId() { return ++taskId_; } 52 | 53 | bool Deleted(int64_t taskId) { 54 | std::shared_lock l(lockSet_); 55 | return markDel.count(taskId); 56 | } 57 | 58 | void DelMark(int64_t taskId) { 59 | std::unique_lock l(lockSet_); 60 | markDel.erase(taskId); 61 | } 62 | 63 | void RePushTask(); 64 | 65 | void PopTask(std::shared_ptr& task, bool deleted); 66 | }; 67 | 68 | } // namespace net 69 | -------------------------------------------------------------------------------- /src/net/timer_task.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace net { 15 | 16 | class ITimerTask { 17 | public: 18 | explicit ITimerTask(int64_t interval) : ITimerTask(0, interval) {} 19 | 20 | ITimerTask(int64_t start, int64_t interval) : interval_(interval) { 21 | if (start <= 0) { 22 | start = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) 23 | .count(); 24 | } 25 | start_ = start; 26 | } 27 | 28 | virtual ~ITimerTask() = default; 29 | 30 | virtual void TimeOut() = 0; 31 | 32 | int64_t Start() const { return start_; } 33 | 34 | int64_t Interval() const { return interval_; } 35 | 36 | void Next() { // the next time it can be executed 37 | start_ += Interval(); 38 | } 39 | 40 | int64_t Id() const { return id_; } 41 | 42 | void SetId(int64_t id) { id_ = id; } 43 | 44 | bool operator<(ITimerTask& task1) const { return task1.Start() < Start(); } 45 | 46 | protected: 47 | int64_t start_ = 0; // Timestamp of the start of the task (in milliseconds) 48 | int64_t interval_ = 0; // Task interval (ms) 49 | int64_t id_ = 0; 50 | }; 51 | 52 | class CommonTimerTask : public ITimerTask { 53 | public: 54 | explicit CommonTimerTask(int64_t interval) : ITimerTask(interval) {} 55 | 56 | CommonTimerTask(int64_t start, int64_t interval) : ITimerTask(start, interval) {} 57 | 58 | void TimeOut() override { callback_(); } 59 | 60 | template 61 | void SetCallback(F&& f, Args&&... args) { 62 | auto temp = std::bind(std::forward(f), std::forward(args)...); 63 | callback_ = [temp]() { (void)temp(); }; 64 | } 65 | 66 | private: 67 | std::function callback_; 68 | }; 69 | 70 | } // namespace net 71 | -------------------------------------------------------------------------------- /src/options.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "common.h" 11 | #include "net/net_options.h" 12 | 13 | namespace kiwi { 14 | 15 | class Options : public net::NetOptions { 16 | public: 17 | Options() = default; 18 | ~Options() = default; 19 | 20 | void SetConfigName(const PString& cfg_file) { cfg_file_ = cfg_file; } 21 | 22 | const PString& GetConfigName() const { return cfg_file_; } 23 | 24 | void SetLogLevel(const PString& log_level) { log_level_ = log_level; } 25 | 26 | const PString& GetLogLevel() const { return log_level_; } 27 | 28 | void SetRedisCompatibleMode(bool mode) { redis_compatible_mode = mode; } 29 | 30 | bool GetRedisCompatibleMode() const { return redis_compatible_mode; } 31 | 32 | void SetIps(const std::vector& ips) { ips_ = ips; } 33 | 34 | const std::vector& GetIps() const { return ips_; } 35 | 36 | void SetUseRaft(const PString& use) { use_raft = use; } 37 | 38 | const PString& GetUseRaft() const { return use_raft; } 39 | 40 | void SetRaftIp(const PString& ip) { raft_ip = ip; } 41 | 42 | const PString& GetRaftIp() const { return raft_ip; } 43 | 44 | private: 45 | PString cfg_file_; 46 | PString log_level_; 47 | 48 | std::atomic redis_compatible_mode = false; 49 | std::vector ips_; 50 | PString use_raft; 51 | PString raft_ip; 52 | }; 53 | 54 | } // namespace kiwi 55 | -------------------------------------------------------------------------------- /src/pubsub.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Defined a set of functions related to the client's 8 | subscription mechanism. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "common.h" 19 | 20 | namespace kiwi { 21 | 22 | class PClient; 23 | class PPubsub { 24 | public: 25 | static PPubsub& Instance(); 26 | 27 | PPubsub(const PPubsub&) = delete; 28 | void operator=(const PPubsub&) = delete; 29 | 30 | std::size_t Subscribe(PClient* client, const PString& channel); 31 | std::size_t UnSubscribe(PClient* client, const PString& channel); 32 | std::size_t UnSubscribeAll(PClient* client); 33 | std::size_t PublishMsg(const PString& channel, const PString& msg); 34 | 35 | std::size_t PSubscribe(PClient* client, const PString& pchannel); 36 | std::size_t PUnSubscribeAll(PClient* client); 37 | std::size_t PUnSubscribe(PClient* client, const PString& pchannel); 38 | 39 | // introspect 40 | void PubsubChannels(std::vector& res, const char* pattern = nullptr) const; 41 | std::size_t PubsubNumsub(const PString& channel) const; 42 | std::size_t PubsubNumpat() const; 43 | 44 | void InitPubsubTimer(); 45 | void RecycleClients(PString& startChannel, PString& startPattern); 46 | 47 | private: 48 | PPubsub() = default; 49 | 50 | using Clients = std::set, std::owner_less > >; 51 | using ChannelClients = std::map; 52 | 53 | ChannelClients channels_; 54 | ChannelClients patternChannels_; 55 | 56 | PString startChannel_; 57 | PString startPattern_; 58 | static void recycleClients(ChannelClients& channels, PString& start); 59 | }; 60 | 61 | } // namespace kiwi 62 | -------------------------------------------------------------------------------- /src/raft/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | ADD_CUSTOM_COMMAND( 7 | OUTPUT "${PROTO_OUTPUT_DIR}/binlog.pb.cc" 8 | DEPENDS protobuf 9 | COMMAND ${PROTOBUF_PROTOC} 10 | ARGS -I ${CMAKE_CURRENT_SOURCE_DIR} 11 | --cpp_out ${PROTO_OUTPUT_DIR} 12 | ${CMAKE_CURRENT_SOURCE_DIR}/binlog.proto 13 | ) 14 | ADD_LIBRARY(binlog_pb STATIC "${PROTO_OUTPUT_DIR}/binlog.pb.cc") 15 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 16 | TARGET_INCLUDE_DIRECTORIES(binlog_pb PRIVATE ${PROTOBUF_INCLUDE_DIR}) 17 | 18 | ADD_CUSTOM_COMMAND( 19 | OUTPUT "${PROTO_OUTPUT_DIR}/raft.pb.cc" 20 | DEPENDS protobuf 21 | COMMAND ${PROTOBUF_PROTOC} 22 | ARGS -I ${CMAKE_CURRENT_SOURCE_DIR} 23 | --cpp_out ${PROTO_OUTPUT_DIR} 24 | ${CMAKE_CURRENT_SOURCE_DIR}/raft.proto 25 | ) 26 | 27 | ADD_LIBRARY(raft_pb STATIC "${PROTO_OUTPUT_DIR}/raft.pb.cc") 28 | TARGET_INCLUDE_DIRECTORIES(raft_pb PRIVATE ${PROTOBUF_INCLUDE_DIR}) 29 | 30 | FILE(GLOB RAFT_INST_SRC 31 | "${CMAKE_CURRENT_SOURCE_DIR}/*.cc" 32 | ) 33 | 34 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 35 | 36 | ADD_LIBRARY(raft ${RAFT_INST_SRC}) 37 | 38 | TARGET_INCLUDE_DIRECTORIES(raft 39 | PRIVATE ${PROJECT_SOURCE_DIR}/src 40 | PRIVATE ${rocksdb_SOURCE_DIR}/include 41 | PRIVATE ${BRAFT_INCLUDE_DIR} 42 | PRIVATE ${BRPC_INCLUDE_DIR} 43 | PRIVATE ${PROTO_OUTPUT_DIR} 44 | PRIVATE ${PROJECT_SOURCE_DIR}/src/storage/include 45 | PRIVATE ${PSTD_INCLUDE_DIR} 46 | ) 47 | 48 | IF (CMAKE_SYSTEM_NAME STREQUAL "Linux") 49 | SET(RAFT_INST_LIB ${RAFT_INST_LIB} rt) 50 | ENDIF () 51 | 52 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 53 | ADD_DEPENDENCIES(raft protobuf raft_pb binlog_pb braft brpc) 54 | TARGET_LINK_LIBRARIES(raft 55 | PRIVATE net; 56 | PRIVATE dl; 57 | PRIVATE ${FMT_LIBRARIES}; 58 | PRIVATE storage; 59 | PRIVATE kstd 60 | PRIVATE ${BRPC_LIBRARIES} 61 | PRIVATE ${BRAFT_LIBRARIES} 62 | PRIVATE ${OPENSSL_SSL_LIBRARY} 63 | PRIVATE ${OPENSSL_CRYPTO_LIBRARY} 64 | PRIVATE ${ZLIB_LIBRARIES} 65 | PRIVATE ${PROTOBUF_LIBRARY} 66 | PRIVATE ${LEVELDB_LIBRARIES} 67 | PRIVATE ${GFLAGS_LIBRARY} 68 | PRIVATE ${ROCKSDB_LIBRARIES} 69 | PRIVATE z 70 | PRIVATE ${RAFT_INST_LIB} 71 | ) 72 | 73 | SET_TARGET_PROPERTIES(raft PROPERTIES LINKER_LANGUAGE CXX) 74 | -------------------------------------------------------------------------------- /src/raft/binlog.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package kiwi; 3 | option optimize_for = LITE_RUNTIME; 4 | 5 | enum OperateType { 6 | kNoOperate = 0; 7 | kPut = 1; 8 | kDelete = 2; 9 | } 10 | 11 | message BinlogEntry { 12 | uint32 cf_idx = 1; 13 | OperateType op_type = 2; 14 | bytes key = 3; 15 | optional bytes value = 4; 16 | } 17 | 18 | message Binlog { 19 | uint32 db_id = 1; 20 | uint32 slot_idx = 2; 21 | repeated BinlogEntry entries = 3; 22 | } 23 | -------------------------------------------------------------------------------- /src/raft/raft.proto: -------------------------------------------------------------------------------- 1 | syntax="proto3"; 2 | package kiwi; 3 | option cc_generic_services = true; 4 | 5 | message DummyRequest { 6 | }; 7 | 8 | message DummyResponse { 9 | }; 10 | 11 | service DummyService { 12 | rpc DummyMethod(DummyRequest) returns (DummyResponse); 13 | }; 14 | -------------------------------------------------------------------------------- /src/raft/raft_service.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "raft.pb.h" 11 | 12 | namespace kiwi { 13 | 14 | class Raft; 15 | 16 | class DummyServiceImpl : public DummyService { 17 | public: 18 | explicit DummyServiceImpl(Raft* raft) : raft_(raft) {} 19 | void DummyMethod(::google::protobuf::RpcController* controller, const ::kiwi::DummyRequest* request, 20 | ::kiwi::DummyResponse* response, ::google::protobuf::Closure* done) override {} 21 | 22 | private: 23 | Raft* raft_ = nullptr; 24 | }; 25 | 26 | } // namespace kiwi 27 | -------------------------------------------------------------------------------- /src/raft/snapshot.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | 12 | #include "braft/file_system_adaptor.h" 13 | #include "braft/macros.h" 14 | #include "braft/snapshot.h" 15 | 16 | #define RAFT_INST_SNAPSHOT_META_FILE "__raft_snapshot_meta" 17 | #define RAFT_INST_SNAPSHOT_PATH "snapshot/snapshot_" 18 | #define IS_RDONLY 0x01 19 | 20 | namespace kiwi { 21 | 22 | class PosixFileSystemAdaptor : public braft::PosixFileSystemAdaptor { 23 | public: 24 | PosixFileSystemAdaptor() = default; 25 | ~PosixFileSystemAdaptor() override = default; 26 | 27 | braft::FileAdaptor* open(const std::string& path, int oflag, const ::google::protobuf::Message* file_meta, 28 | butil::File::Error* e) override; 29 | void AddAllFiles(const std::filesystem::path& dir, braft::LocalSnapshotMetaTable* snapshot_meta_memtable, 30 | const std::string& path); 31 | 32 | private: 33 | braft::raft_mutex_t mutex_; 34 | }; 35 | 36 | } // namespace kiwi 37 | -------------------------------------------------------------------------------- /src/resp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 3.25) 7 | 8 | PROJECT(resp) 9 | 10 | AUX_SOURCE_DIRECTORY(. RESP_SRC) 11 | 12 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 13 | 14 | SET(RESP_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "resp include directory." FORCE) 15 | 16 | ADD_LIBRARY(resp ${RESP_SRC}) 17 | 18 | TARGET_LINK_LIBRARIES(resp 19 | PRIVATE kstd 20 | PRIVATE ${FMT_LIBRARIES} 21 | ) 22 | 23 | ADD_SUBDIRECTORY(tests) 24 | 25 | TARGET_INCLUDE_DIRECTORIES(resp 26 | PRIVATE ${PSTD_INCLUDE_DIR} 27 | PRIVATE ${FMT_INCLUDE_DIR} 28 | PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} 29 | ) 30 | 31 | ADD_DEPENDENCIES(resp kstd fmt) 32 | -------------------------------------------------------------------------------- /src/resp/resp2_encode.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include "resp_encode.h" 9 | 10 | class Resp2Encode : public RespEncode { 11 | public: 12 | void SetRes(CmdRes ret, const std::string& content = "") override; 13 | void AppendArrayLen(int64_t ori) override { SetBulkStringLen(reply_, ori, "*"); } 14 | void AppendInteger(int64_t ori) override { SetBulkStringLen(reply_, ori, ":"); } 15 | void AppendStringRaw(const std::string& value) override { reply_.append(value); } 16 | void AppendSimpleString(const std::string& value) override; 17 | void AppendString(const std::string& value) override { AppendBulkString(reply_, value); } 18 | void AppendString(const char* value, int64_t size) override; 19 | void AppendStringVector(const std::vector& strArray) override; 20 | void SetLineString(const std::string& value) override { reply_ = value + CRLF; } 21 | void ClearReply() override { reply_.clear(); } 22 | }; 23 | -------------------------------------------------------------------------------- /src/resp/resp2_parse.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include "resp_parse.h" 9 | 10 | class Resp2Parse : public RespParse { 11 | public: 12 | explicit Resp2Parse() = default; 13 | 14 | RespResult Parse(std::string&& data) override { 15 | data_.append(data); 16 | return ParsePipeline(); 17 | }; 18 | 19 | RespParams GetParams() override { 20 | Reset(); 21 | auto result = std::move(params_); 22 | ClearParams(); 23 | return result; 24 | } 25 | 26 | void GetParams(RespParams& params) override { 27 | Reset(); 28 | params.swap(params_); 29 | ClearParams(); 30 | } 31 | 32 | private: 33 | void Reset() { 34 | pos_ = 0; 35 | data_.clear(); 36 | }; 37 | 38 | void ClearParams() { 39 | params_.resize(0); 40 | singleParams_.resize(0); 41 | singleParamsSize_ = -1; 42 | } 43 | 44 | static RespType PetRespType(char prefix); 45 | RespResult ParsePipeline(); 46 | std::pair ReadLine(); 47 | RespResult ParseInline(); 48 | RespResult ParseSimpleString(); 49 | RespResult parseError(); 50 | RespResult ParseInteger(); 51 | RespResult ParseBulkString(); 52 | RespResult ParseArray(); 53 | RespResult ParseResp(); 54 | void AppendParams(const std::string& param) { singleParams_.emplace_back(param); }; 55 | void MergeParams(); 56 | 57 | private: 58 | std::string data_; 59 | size_t pos_ = 0; 60 | std::vector singleParams_; 61 | // -1 means not array, 0 means null array 62 | int singleParamsSize_ = -1; 63 | RespParams params_; 64 | }; 65 | -------------------------------------------------------------------------------- /src/resp/resp_encode.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #include "resp_encode.h" 7 | 8 | void RespEncode::AppendBulkString(std::string& str, const std::string& value) { 9 | SetBulkStringLen(str, static_cast(value.size()), "$"); 10 | str.append(value.data(), value.size()); 11 | str.append(CRLF); 12 | } 13 | 14 | void RespEncode::SetBulkStringLen(std::string& str, int64_t ori, const std::string& prefix) { 15 | str.append(prefix); 16 | str.append(kstd::Int2string(ori)); 17 | str.append(CRLF); 18 | } 19 | -------------------------------------------------------------------------------- /src/resp/resp_encode.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include "std_string.h" 12 | 13 | enum class CmdRes : std::int8_t { 14 | kNone = 0, 15 | kOK, 16 | kPong, 17 | kSyntaxErr, 18 | kInvalidInt, 19 | kInvalidBitInt, 20 | kInvalidBitOffsetInt, 21 | kInvalidFloat, 22 | kOverFlow, 23 | kNotFound, 24 | kOutOfRange, 25 | kInvalidPwd, 26 | kNoneBgsave, 27 | kPurgeExist, 28 | kInvalidParameter, 29 | kWrongNum, 30 | kInvalidIndex, 31 | kInvalidDbType, 32 | kInvalidDB, 33 | kInconsistentHashTag, 34 | kErrOther, 35 | kErrMoved, 36 | kErrClusterDown, 37 | kUnknownCmd, 38 | kUnknownSubCmd, 39 | KIncrByOverFlow, 40 | kInvalidCursor, 41 | kWrongLeader, 42 | kMultiKey, 43 | kNoAuth, 44 | }; 45 | 46 | constexpr char CRLF[] = "\r\n"; 47 | 48 | class RespEncode { 49 | public: 50 | virtual ~RespEncode() = default; 51 | 52 | static void AppendBulkString(std::string& str, const std::string& value); 53 | static void SetBulkStringLen(std::string& str, int64_t ori, const std::string& prefix); 54 | 55 | void Reply(std::string& str) { str.swap(reply_); } 56 | 57 | virtual void SetRes(CmdRes ret, const std::string& content = "") = 0; 58 | virtual void AppendArrayLen(int64_t ori) = 0; 59 | virtual void AppendInteger(int64_t ori) = 0; 60 | virtual void AppendStringRaw(const std::string& value) = 0; 61 | virtual void AppendSimpleString(const std::string& value) = 0; 62 | virtual void AppendString(const std::string& value) = 0; 63 | virtual void AppendString(const char* value, int64_t size) = 0; 64 | virtual void AppendStringVector(const std::vector& strArray) = 0; 65 | virtual void SetLineString(const std::string& value) = 0; 66 | virtual void ClearReply() = 0; 67 | 68 | protected: 69 | std::string reply_; 70 | CmdRes ret_ = CmdRes::kNone; 71 | }; 72 | -------------------------------------------------------------------------------- /src/resp/resp_parse.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | enum class RespType { SimpleString, Error, Integer, BulkString, Array, Inline }; 12 | 13 | enum class RespResult { OK, ERROR, WAIT }; 14 | 15 | using RespParams = std::vector>; 16 | 17 | class RespParse { 18 | public: 19 | virtual RespResult Parse(std::string&& data) = 0; 20 | virtual RespParams GetParams() = 0; 21 | virtual void GetParams(RespParams& params) = 0; 22 | virtual ~RespParse() = default; 23 | }; 24 | -------------------------------------------------------------------------------- /src/resp/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 3.25) 7 | 8 | INCLUDE(GoogleTest) 9 | SET(CMAKE_CXX_STANDARD 20) 10 | 11 | AUX_SOURCE_DIRECTORY(../ DIR_SRCS) 12 | 13 | FILE(GLOB_RECURSE GTEST_TEST_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") 14 | 15 | FOREACH (gtest_test_source ${GTEST_TEST_SOURCE}) 16 | GET_FILENAME_COMPONENT(gtest_test_filename ${gtest_test_source} NAME) 17 | STRING(REPLACE ".cc" "" gtest_test_name ${gtest_test_filename}) 18 | 19 | ADD_EXECUTABLE(${gtest_test_name} ${gtest_test_source} ${DIR_SRCS}) 20 | TARGET_INCLUDE_DIRECTORIES(${gtest_test_name} 21 | PRIVATE ${GTEST_INCLUDE_DIR} 22 | PRIVATE ${PSTD_INCLUDE_DIR} 23 | ) 24 | 25 | ADD_DEPENDENCIES(${gtest_test_name} kstd gtest) 26 | TARGET_LINK_LIBRARIES(${gtest_test_name} 27 | PRIVATE kstd 28 | PRIVATE ${GTEST_LIBRARIES} 29 | ) 30 | GTEST_DISCOVER_TESTS(${gtest_test_name}) 31 | ENDFOREACH () -------------------------------------------------------------------------------- /src/slow_log.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Implement the slow log feature. 8 | 9 | The slow log is responsible for recording commands that exceed the threshold time. 10 | */ 11 | 12 | #include 13 | #include 14 | 15 | #include "log.h" 16 | #include "slow_log.h" 17 | 18 | namespace kiwi { 19 | 20 | PSlowLog& PSlowLog::Instance() { 21 | static PSlowLog slog; 22 | 23 | return slog; 24 | } 25 | 26 | PSlowLog::PSlowLog() : threshold_(0), logger_(nullptr) {} 27 | 28 | PSlowLog::~PSlowLog() {} 29 | 30 | void PSlowLog::SetThreshold(unsigned int v) { threshold_ = v; } 31 | 32 | void PSlowLog::SetLogLimit(std::size_t maxCount) { logMaxCount_ = maxCount; } 33 | 34 | void PSlowLog::Begin() { 35 | if (!threshold_) { 36 | return; 37 | } 38 | 39 | timeval begin; 40 | gettimeofday(&begin, 0); 41 | beginUs_ = begin.tv_sec * 1000000 + begin.tv_usec; 42 | } 43 | 44 | void PSlowLog::EndAndStat(const std::vector& cmds) { 45 | if (!threshold_ || beginUs_ == 0) { 46 | return; 47 | } 48 | 49 | timeval end; 50 | gettimeofday(&end, 0); 51 | auto used = (end.tv_sec * 1000000) + end.tv_usec - beginUs_; 52 | 53 | if (used >= threshold_) { 54 | INFO("+ Used:(us) {}", used); 55 | 56 | for (const auto& param : cmds) { 57 | INFO("{}", param); 58 | } 59 | 60 | if (cmds[0] == "slowlog") { 61 | return; 62 | } 63 | 64 | SlowLogItem item; 65 | item.used = static_cast(used); 66 | item.cmds = cmds; 67 | 68 | logs_.emplace_front(std::move(item)); 69 | if (logs_.size() > logMaxCount_) { 70 | logs_.pop_back(); 71 | } 72 | } 73 | } 74 | 75 | } // namespace kiwi 76 | -------------------------------------------------------------------------------- /src/slow_log.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Declared a set of features related to the slow log. 8 | */ 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | 15 | #include "common.h" 16 | 17 | class Logger; 18 | 19 | namespace kiwi { 20 | 21 | struct SlowLogItem { 22 | unsigned used; 23 | std::vector cmds; 24 | 25 | SlowLogItem() : used(0) {} 26 | 27 | SlowLogItem(SlowLogItem&& item) noexcept : used(item.used), cmds(std::move(item.cmds)) {} 28 | }; 29 | 30 | class PSlowLog { 31 | public: 32 | static PSlowLog& Instance(); 33 | 34 | PSlowLog(const PSlowLog&) = delete; 35 | void operator=(const PSlowLog&) = delete; 36 | 37 | void Begin(); 38 | void EndAndStat(const std::vector& cmds); 39 | 40 | void SetThreshold(unsigned int); 41 | void SetLogLimit(std::size_t maxCount); 42 | 43 | void ClearLogs() { logs_.clear(); } 44 | std::size_t GetLogsCount() const { return logs_.size(); } 45 | const std::deque& GetLogs() const { return logs_; } 46 | 47 | private: 48 | PSlowLog(); 49 | ~PSlowLog(); 50 | 51 | unsigned int threshold_; 52 | long long beginUs_; 53 | Logger* logger_; 54 | 55 | std::size_t logMaxCount_; 56 | std::deque logs_; 57 | }; 58 | 59 | } // namespace kiwi 60 | -------------------------------------------------------------------------------- /src/std/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | AUX_SOURCE_DIRECTORY(. STD_SRC) 7 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 8 | SET(PSTD_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "kstd include directory." FORCE) 9 | ADD_LIBRARY(kstd ${STD_SRC}) 10 | 11 | ADD_SUBDIRECTORY(tests) 12 | 13 | TARGET_INCLUDE_DIRECTORIES(kstd 14 | PRIVATE ${ROCKSDB_SOURCES_DIR}/include 15 | PRIVATE ${LIB_INCLUDE_DIR} 16 | PRIVATE ${PSTD_INCLUDE_DIR} 17 | ) 18 | 19 | TARGET_LINK_LIBRARIES(kstd; 20 | PRIVATE ${SPDLOG_LIBRARIES} 21 | PRIVATE pthread 22 | PRIVATE ${FMT_LIBRARIES} 23 | ) 24 | 25 | SET_TARGET_PROPERTIES(kstd PROPERTIES LINKER_LANGUAGE CXX) 26 | ADD_DEPENDENCIES(kstd rocksdb spdlog) 27 | -------------------------------------------------------------------------------- /src/std/kiwi_slot.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present The storage Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | 8 | #include "kiwi_slot.h" 9 | 10 | // get slot tag 11 | static const char *GetSlotsTag(const std::string &str, int *plen) { 12 | const char *s = str.data(); 13 | int i, j, n = static_cast(str.length()); 14 | for (i = 0; i < n && s[i] != '{'; i++) { 15 | } 16 | if (i == n) { 17 | return nullptr; 18 | } 19 | i++; 20 | for (j = i; j < n && s[j] != '}'; j++) { 21 | } 22 | if (j == n) { 23 | return nullptr; 24 | } 25 | if (plen != nullptr) { 26 | *plen = j - i; 27 | } 28 | return s + i; 29 | } 30 | 31 | // get db instance number of the key 32 | uint32_t GetSlotID(const std::string &str) { return GetSlotsID(str, nullptr, nullptr); } 33 | 34 | // get db instance number of the key 35 | uint32_t GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag) { 36 | const char *s = str.data(); 37 | int taglen; 38 | int hastag = 0; 39 | const char *tag = GetSlotsTag(str, &taglen); 40 | if (tag == nullptr) { 41 | tag = s, taglen = static_cast(str.length()); 42 | } else { 43 | hastag = 1; 44 | } 45 | auto crc = crc32(0L, (const Bytef *)tag, taglen); 46 | if (pcrc != nullptr) { 47 | *pcrc = uint32_t(crc); 48 | } 49 | if (phastag != nullptr) { 50 | *phastag = hastag; 51 | } 52 | return static_cast(crc); 53 | } 54 | -------------------------------------------------------------------------------- /src/std/kiwi_slot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef kiwi_SLOT_H_ 7 | #define kiwi_SLOT_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | // get db instance number of the key 14 | uint32_t GetSlotID(const std::string& str); 15 | 16 | // get db instance number of the key 17 | uint32_t GetSlotsID(const std::string& str, uint32_t* pcrc, int* phastag); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/std/lock_free_ring_buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | // calculate the capacity of the ring buffer, 14 | // which is the smallest power of 2 that is greater than or equal to size 15 | // e.g. size = 10, return 16 16 | static size_t calculateRingBuffCapacity(size_t size) { 17 | // if num is already a power of 2, return it 18 | if ((size & (size - 1)) == 0) { 19 | return size; 20 | } 21 | 22 | size_t result = size; 23 | result |= (result >> 1); 24 | result |= (result >> 2); 25 | result |= (result >> 4); 26 | result |= (result >> 8); 27 | result |= (result >> 16); 28 | result += 1; 29 | 30 | return result; 31 | } 32 | 33 | template 34 | class LockFreeRingBuffer { 35 | public: 36 | explicit LockFreeRingBuffer(size_t capacity) 37 | : capacity_(calculateRingBuffCapacity(capacity)), head_(0), tail_(0), buffer_(capacity_) {} 38 | 39 | // push an item into the queue 40 | template 41 | bool Push(U&& item) { 42 | size_t current_tail = tail_.load(std::memory_order_relaxed); 43 | size_t next_tail = (current_tail + 1) & (capacity_ - 1); 44 | if (next_tail == head_.load(std::memory_order_acquire)) { 45 | // queue is full 46 | return false; 47 | } 48 | buffer_[current_tail] = std::forward(item); 49 | tail_.store(next_tail, std::memory_order_release); 50 | return true; 51 | } 52 | 53 | // pop an item from the queue, if the queue is empty, return false 54 | bool Pop(T& item) { 55 | size_t current_head = head_.load(std::memory_order_relaxed); 56 | if (current_head == tail_.load(std::memory_order_acquire)) { 57 | // queue is empty 58 | return false; 59 | } 60 | 61 | item = std::move(buffer_[current_head]); 62 | head_.store((current_head + 1) & (capacity_ - 1), std::memory_order_release); 63 | return true; 64 | } 65 | 66 | bool Empty() { return head_.load(std::memory_order_acquire) == tail_.load(std::memory_order_acquire); } 67 | bool Full() { 68 | return ((tail_.load(std::memory_order_acquire) + 1) & (capacity_ - 1)) == head_.load(std::memory_order_acquire); 69 | } 70 | 71 | private: 72 | const size_t capacity_; 73 | std::atomic head_; 74 | std::atomic tail_; 75 | std::vector buffer_; 76 | }; 77 | -------------------------------------------------------------------------------- /src/std/lock_mgr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | 13 | #include "mutex.h" 14 | #include "noncopyable.h" 15 | 16 | namespace kstd::lock { 17 | struct LockMap; 18 | struct LockMapStripe; 19 | 20 | class LockMgr : public kstd::noncopyable { 21 | public: 22 | LockMgr(size_t default_num_stripes, int64_t max_num_locks, const std::shared_ptr& factory); 23 | 24 | ~LockMgr(); 25 | 26 | // Attempt to lock key. If OK status is returned, the caller is responsible 27 | // for calling UnLock() on this key. 28 | Status TryLock(const std::string& key); 29 | 30 | // Unlock a key locked by TryLock(). 31 | void UnLock(const std::string& key); 32 | 33 | private: 34 | // Default number of lock map stripes 35 | const size_t default_num_stripes_ [[maybe_unused]]; 36 | 37 | // Limit on number of keys locked per column family 38 | const int64_t max_num_locks_; 39 | 40 | // Used to allocate mutexes/condvars to use when locking keys 41 | std::shared_ptr mutex_factory_; 42 | 43 | // Map to locked key info 44 | std::shared_ptr lock_map_; 45 | 46 | Status Acquire(const std::shared_ptr& stripe, const std::string& key); 47 | 48 | Status AcquireLocked(const std::shared_ptr& stripe, const std::string& key); 49 | 50 | void UnLockKey(const std::string& key, const std::shared_ptr& stripe); 51 | }; 52 | 53 | } // namespace kstd::lock 54 | -------------------------------------------------------------------------------- /src/std/mutex_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "mutex.h" 11 | 12 | #include 13 | 14 | namespace kstd::lock { 15 | // Default implementation of MutexFactory. 16 | class MutexFactoryImpl : public MutexFactory { 17 | public: 18 | std::shared_ptr AllocateMutex() override; 19 | std::shared_ptr AllocateCondVar() override; 20 | }; 21 | } // namespace kstd::lock 22 | -------------------------------------------------------------------------------- /src/std/noncopyable.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | namespace kstd { 11 | 12 | class noncopyable { 13 | protected: 14 | noncopyable() = default; 15 | virtual ~noncopyable() = default; 16 | 17 | private: 18 | noncopyable(const noncopyable&) = delete; 19 | void operator=(const noncopyable&) = delete; 20 | }; 21 | 22 | } // namespace kstd 23 | -------------------------------------------------------------------------------- /src/std/scope_record_lock.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #include 9 | 10 | #include "scope_record_lock.h" 11 | 12 | namespace kstd::lock { 13 | 14 | MultiScopeRecordLock::MultiScopeRecordLock(const std::shared_ptr& lock_mgr, 15 | const std::vector& keys) 16 | : lock_mgr_(lock_mgr), keys_(keys) { 17 | std::string pre_key; 18 | std::sort(keys_.begin(), keys_.end()); 19 | if (!keys_.empty() && keys_[0].empty()) { 20 | lock_mgr_->TryLock(pre_key); 21 | } 22 | 23 | for (const auto& key : keys_) { 24 | if (pre_key != key) { 25 | lock_mgr_->TryLock(key); 26 | pre_key = key; 27 | } 28 | } 29 | } 30 | MultiScopeRecordLock::~MultiScopeRecordLock() { 31 | std::string pre_key; 32 | if (!keys_.empty() && keys_[0].empty()) { 33 | lock_mgr_->UnLock(pre_key); 34 | } 35 | 36 | for (const auto& key : keys_) { 37 | if (pre_key != key) { 38 | lock_mgr_->UnLock(key); 39 | pre_key = key; 40 | } 41 | } 42 | } 43 | 44 | void MultiRecordLock::Lock(const std::vector& keys) { 45 | std::vector internal_keys = keys; 46 | std::sort(internal_keys.begin(), internal_keys.end()); 47 | // init to be "" 48 | std::string pre_key; 49 | // consider internal_keys "" "" "a" 50 | if (!internal_keys.empty()) { 51 | lock_mgr_->TryLock(internal_keys.front()); 52 | pre_key = internal_keys.front(); 53 | } 54 | 55 | for (const auto& key : internal_keys) { 56 | if (pre_key != key) { 57 | lock_mgr_->TryLock(key); 58 | pre_key = key; 59 | } 60 | } 61 | } 62 | 63 | void MultiRecordLock::Unlock(const std::vector& keys) { 64 | std::vector internal_keys = keys; 65 | std::sort(internal_keys.begin(), internal_keys.end()); 66 | std::string pre_key; 67 | if (!internal_keys.empty()) { 68 | lock_mgr_->UnLock(internal_keys.front()); 69 | pre_key = internal_keys.front(); 70 | } 71 | 72 | for (const auto& key : internal_keys) { 73 | if (pre_key != key) { 74 | lock_mgr_->UnLock(key); 75 | pre_key = key; 76 | } 77 | } 78 | } 79 | } // namespace kstd::lock 80 | -------------------------------------------------------------------------------- /src/std/scope_record_lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "lock_mgr.h" 16 | #include "noncopyable.h" 17 | #include "rocksdb/slice.h" 18 | 19 | namespace kstd::lock { 20 | 21 | using Slice = rocksdb::Slice; 22 | 23 | class ScopeRecordLock final : public kstd::noncopyable { 24 | public: 25 | ScopeRecordLock(const std::shared_ptr& lock_mgr, const Slice& key) : lock_mgr_(lock_mgr), key_(key) { 26 | lock_mgr_->TryLock(key_.ToString()); 27 | } 28 | ~ScopeRecordLock() { lock_mgr_->UnLock(key_.ToString()); } 29 | 30 | private: 31 | std::shared_ptr const lock_mgr_; 32 | Slice key_; 33 | }; 34 | 35 | class MultiScopeRecordLock final : public kstd::noncopyable { 36 | public: 37 | MultiScopeRecordLock(const std::shared_ptr& lock_mgr, const std::vector& keys); 38 | ~MultiScopeRecordLock(); 39 | 40 | private: 41 | std::shared_ptr const lock_mgr_; 42 | std::vector keys_; 43 | }; 44 | 45 | class MultiRecordLock : public noncopyable { 46 | public: 47 | explicit MultiRecordLock(const std::shared_ptr& lock_mgr) : lock_mgr_(lock_mgr) {} 48 | ~MultiRecordLock() = default; 49 | 50 | void Lock(const std::vector& keys); 51 | void Unlock(const std::vector& keys); 52 | 53 | private: 54 | std::shared_ptr const lock_mgr_; 55 | }; 56 | 57 | } // namespace kstd::lock 58 | -------------------------------------------------------------------------------- /src/std/std_defer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | 13 | // CTAD: See https://en.cppreference.com/w/cpp/language/class_template_argument_deduction 14 | #if __cpp_deduction_guides >= 201606 15 | 16 | template 17 | class ExecuteOnScopeExit { 18 | public: 19 | ExecuteOnScopeExit(F&& f) : func_(std::move(f)) {} 20 | ExecuteOnScopeExit(const F& f) : func_(f) {} 21 | ~ExecuteOnScopeExit() { func_(); } 22 | 23 | ExecuteOnScopeExit(const ExecuteOnScopeExit& e) = delete; 24 | ExecuteOnScopeExit& operator=(const ExecuteOnScopeExit& f) = delete; 25 | 26 | private: 27 | F func_; 28 | }; 29 | 30 | #else 31 | 32 | class ExecuteOnScopeExit { 33 | public: 34 | ExecuteOnScopeExit() = default; 35 | 36 | // movable 37 | ExecuteOnScopeExit(ExecuteOnScopeExit&&) = default; 38 | ExecuteOnScopeExit& operator=(ExecuteOnScopeExit&&) = default; 39 | 40 | // non copyable 41 | ExecuteOnScopeExit(const ExecuteOnScopeExit& e) = delete; 42 | void operator=(const ExecuteOnScopeExit& f) = delete; 43 | 44 | template 45 | ExecuteOnScopeExit(F&& f) : func_(std::forward(f)) {} 46 | 47 | ~ExecuteOnScopeExit() noexcept { 48 | if (func_) func_(); 49 | } 50 | 51 | private: 52 | std::function func_; 53 | }; 54 | 55 | #endif 56 | 57 | #define _CONCAT(a, b) a##b 58 | #define _MAKE_DEFER_(line) ExecuteOnScopeExit _CONCAT(defer, line) = [&]() 59 | 60 | // !!! DEFER 61 | #undef DEFER 62 | #define DEFER _MAKE_DEFER_(__LINE__) 63 | -------------------------------------------------------------------------------- /src/std/std_mutex.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "noncopyable.h" 20 | 21 | namespace kstd { 22 | 23 | using Mutex = std::mutex; 24 | using CondVar = std::condition_variable; 25 | using RWMutex = std::shared_mutex; 26 | 27 | using OnceType = std::once_flag; 28 | 29 | template 30 | void InitOnce(OnceType& once, F&& f, Args&&... args) { 31 | return std::call_once(once, std::forward(f), std::forward(args)...); 32 | } 33 | 34 | class RefMutex : public kstd::noncopyable { 35 | public: 36 | RefMutex() = default; 37 | ~RefMutex() = default; 38 | 39 | // Lock and Unlock will increase and decrease refs_, 40 | // should check refs before Unlock 41 | void Lock(); 42 | void Unlock(); 43 | 44 | void Ref(); 45 | void Unref(); 46 | bool IsLastRef() { return refs_ == 1; } 47 | 48 | private: 49 | std::mutex mu_; 50 | int refs_ = 0; 51 | }; 52 | 53 | class RecordMutex : public kstd::noncopyable { 54 | public: 55 | RecordMutex() = default; 56 | ; 57 | ~RecordMutex(); 58 | 59 | void MultiLock(const std::vector& keys); 60 | void Lock(const std::string& key); 61 | void MultiUnlock(const std::vector& keys); 62 | void Unlock(const std::string& key); 63 | 64 | private: 65 | Mutex mutex_; 66 | 67 | std::unordered_map records_; 68 | }; 69 | 70 | class RecordLock : public kstd::noncopyable { 71 | public: 72 | RecordLock(RecordMutex* mu, std::string key) : mu_(mu), key_(std::move(key)) { mu_->Lock(key_); } 73 | ~RecordLock() { mu_->Unlock(key_); } 74 | 75 | private: 76 | RecordMutex* const mu_; 77 | std::string key_; 78 | }; 79 | 80 | } // namespace kstd 81 | -------------------------------------------------------------------------------- /src/std/std_status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "std_status.h" 6 | #include 7 | #include 8 | 9 | namespace kstd { 10 | 11 | const char* Status::CopyState(const char* state) { 12 | uint32_t size; 13 | memcpy(&size, state, sizeof(size)); 14 | char* result = new char[size + 5]; 15 | memcpy(result, state, size + 5); 16 | return result; 17 | } 18 | 19 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 20 | assert(code != kOk); 21 | const uint32_t len1 = static_cast(msg.size()); 22 | const uint32_t len2 = static_cast(msg2.size()); 23 | const uint32_t size = len1 + (len2 != 0U ? (2 + len2) : 0); 24 | char* result = new char[size + 5]; 25 | memcpy(result, &size, sizeof(size)); 26 | result[4] = static_cast(code); 27 | memcpy(result + 5, msg.data(), len1); 28 | if (len2 != 0U) { 29 | result[5 + len1] = ':'; 30 | result[6 + len1] = ' '; 31 | memcpy(result + 7 + len1, msg2.data(), len2); 32 | } 33 | state_ = result; 34 | } 35 | 36 | std::string Status::ToString() const { 37 | if (!state_) { 38 | return "OK"; 39 | } else { 40 | char tmp[30]; 41 | const char* type; 42 | switch (code()) { 43 | case kOk: 44 | type = "OK"; 45 | break; 46 | case kNotFound: 47 | type = "NotFound: "; 48 | break; 49 | case kCorruption: 50 | type = "Corruption: "; 51 | break; 52 | case kNotSupported: 53 | type = "Not implemented: "; 54 | break; 55 | case kInvalidArgument: 56 | type = "Invalid argument: "; 57 | break; 58 | case kIOError: 59 | type = "IO error: "; 60 | break; 61 | case kEndFile: 62 | type = "End file: "; 63 | break; 64 | case kIncomplete: 65 | type = "InComplete: "; 66 | break; 67 | case kComplete: 68 | type = "Complete: "; 69 | break; 70 | case kTimeout: 71 | type = "Timeout: "; 72 | break; 73 | case kAuthFailed: 74 | type = "AuthFailed: "; 75 | break; 76 | case kBusy: 77 | type = "Busy:"; 78 | break; 79 | case kError: 80 | type = "Error:"; 81 | break; 82 | default: 83 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", static_cast(code())); 84 | type = tmp; 85 | break; 86 | } 87 | std::string result(type); 88 | uint32_t length; 89 | memcpy(&length, state_, sizeof(length)); 90 | result.append(state_ + 5, length); 91 | return result; 92 | } 93 | } 94 | 95 | } // namespace kstd 96 | -------------------------------------------------------------------------------- /src/std/std_util.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | 8 | #include "std_util.h" 9 | 10 | namespace kstd { 11 | 12 | std::mt19937 gen; 13 | 14 | void InitRandom() { 15 | std::random_device rd; 16 | gen.seed(rd()); 17 | } 18 | 19 | int RandomInt(int max) { return RandomInt(0, max); } 20 | 21 | int RandomInt(int min, int max) { 22 | std::uniform_int_distribution<> dist(min, max); 23 | return dist(gen); 24 | } 25 | 26 | double RandomDouble() { 27 | std::uniform_real_distribution dis(0.0, 1.0); 28 | return dis(gen); 29 | } 30 | 31 | } // namespace kstd 32 | -------------------------------------------------------------------------------- /src/std/std_util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace kstd { 13 | 14 | extern std::mt19937 gen; 15 | 16 | // init random seed 17 | void InitRandom(); 18 | 19 | // return [0, max) random number 20 | int RandomInt(int max); 21 | 22 | // return [min, max] random number 23 | int RandomInt(int min, int max); 24 | 25 | // Returns an out-of-order vector of length n, and the numbers in the vector are unique 26 | 27 | template 28 | std::vector RandomPerm(T n) { 29 | if (n <= 0) { 30 | return {}; 31 | } 32 | std::vector perm(n); 33 | for (T i = 0; i < n; ++i) { 34 | perm[i] = i; 35 | } 36 | std::shuffle(perm.begin(), perm.end(), gen); 37 | return perm; 38 | } 39 | 40 | // return [0, 1] random double 41 | double RandomDouble(); 42 | 43 | // returns t as a Unix time, the number of elapsed since January 1, 1970 UTC. 44 | inline int64_t UnixTimestamp() { 45 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); 46 | } 47 | 48 | // returns t as a Unix time, the number of milliseconds elapsed since January 1, 1970 UTC. 49 | inline int64_t UnixMilliTimestamp() { 50 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) 51 | .count(); 52 | } 53 | 54 | // returns t as a Unix time, the number of Microseconds elapsed since January 1, 1970 UTC. 55 | inline int64_t UnixMicroTimestamp() { 56 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) 57 | .count(); 58 | } 59 | 60 | // returns t as a Unix time, the number of nanoseconds elapsed since January 1, 1970 UTC. 61 | inline int64_t UnixNanoTimestamp() { 62 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) 63 | .count(); 64 | } 65 | 66 | } // namespace kstd 67 | -------------------------------------------------------------------------------- /src/std/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 3.25) 7 | 8 | INCLUDE(GoogleTest) 9 | SET(CMAKE_CXX_STANDARD 20) 10 | 11 | AUX_SOURCE_DIRECTORY(.. DIR_SRCS) 12 | 13 | FILE(GLOB_RECURSE GTEST_TEST_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") 14 | 15 | FOREACH (std_test_source ${GTEST_TEST_SOURCE}) 16 | GET_FILENAME_COMPONENT(std_test_filename ${std_test_source} NAME) 17 | STRING(REPLACE ".cc" "" std_test_name ${std_test_filename}) 18 | 19 | # set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) 20 | ADD_EXECUTABLE(${std_test_name} ${std_test_source}) 21 | TARGET_INCLUDE_DIRECTORIES(${std_test_name} 22 | PRIVATE ${PROJECT_SOURCE_DIR}/src 23 | PRIVATE ${GTEST_INCLUDE_DIR} 24 | ) 25 | 26 | ADD_DEPENDENCIES(${std_test_name} kstd gtest) 27 | TARGET_LINK_LIBRARIES(${std_test_name} 28 | PRIVATE kstd 29 | PRIVATE ${GTEST_LIBRARIES} 30 | ) 31 | GTEST_DISCOVER_TESTS(${std_test_name}) 32 | ENDFOREACH () 33 | -------------------------------------------------------------------------------- /src/std/tests/lock_free_ring_buffer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include 8 | 9 | #include "../lock_free_ring_buffer.h" 10 | 11 | class LockFreeRingBufferTest : public ::testing::Test {}; 12 | 13 | TEST_F(LockFreeRingBufferTest, Empty) { 14 | LockFreeRingBuffer ring_buffer(10); 15 | ASSERT_EQ(ring_buffer.Empty(), true); 16 | } 17 | 18 | TEST_F(LockFreeRingBufferTest, Full) { 19 | LockFreeRingBuffer ring_buffer(10); 20 | for (int i = 0; i < 20; ++i) { 21 | ring_buffer.Push(i); 22 | if (i <= 13) { 23 | ASSERT_EQ(ring_buffer.Full(), false); 24 | } else { 25 | ASSERT_EQ(ring_buffer.Full(), true); 26 | } 27 | } 28 | } 29 | 30 | TEST_F(LockFreeRingBufferTest, SingleThread) { 31 | LockFreeRingBuffer ring_buffer(10); 32 | int pushSize = 0; 33 | for (int i = 0; i < 20; ++i) { 34 | if (!ring_buffer.Push(i)) { 35 | break; 36 | } 37 | ++pushSize; 38 | } 39 | ASSERT_EQ(pushSize, 15); 40 | ASSERT_EQ(ring_buffer.Full(), true); 41 | for (int i = 0; i < pushSize; ++i) { 42 | int item; 43 | ASSERT_EQ(ring_buffer.Pop(item), true); 44 | ASSERT_EQ(item, i); 45 | } 46 | ASSERT_EQ(ring_buffer.Empty(), true); 47 | } 48 | 49 | TEST_F(LockFreeRingBufferTest, MultipleThread) { 50 | LockFreeRingBuffer ring_buffer(10); 51 | const int SIZE = 10000; 52 | std::thread producer([&ring_buffer]() { 53 | for (int i = 0; i < SIZE; ++i) { 54 | while (!ring_buffer.Push(i)) { 55 | } 56 | } 57 | }); 58 | 59 | std::thread consumer([&ring_buffer]() { 60 | int i = 0; 61 | while (i < SIZE) { 62 | int item; 63 | while (!ring_buffer.Pop(item)) { 64 | } 65 | ASSERT_EQ(item, i); 66 | ++i; 67 | } 68 | }); 69 | producer.join(); 70 | consumer.join(); 71 | } 72 | 73 | int main(int argc, char** argv) { 74 | ::testing::InitGoogleTest(&argc, argv); 75 | return RUN_ALL_TESTS(); 76 | } 77 | -------------------------------------------------------------------------------- /src/std/tests/std_util_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "std/std_util.h" 7 | #include 8 | 9 | class UtilTest : public ::testing::Test {}; 10 | 11 | TEST(UtilTest, RandomInt) { 12 | int min = 0; 13 | int max = 100; 14 | int random_int = kstd::RandomInt(min, max); 15 | ASSERT_GE(random_int, min); 16 | ASSERT_LE(random_int, max); 17 | } 18 | 19 | TEST(UtilTest, RandomInt2) { 20 | int max = 100; 21 | for (int i = 0; i < 100; ++i) { 22 | int random_int = kstd::RandomInt(max); 23 | ASSERT_LE(random_int, max); 24 | } 25 | } 26 | 27 | TEST(UtilTest, RandomDouble) { 28 | double random_double = kstd::RandomDouble(); 29 | ASSERT_GE(random_double, 0.0); 30 | ASSERT_LE(random_double, 1.0); 31 | } 32 | 33 | TEST(UtilTest, RandomPerm) { 34 | std::vector perm = kstd::RandomPerm(10); 35 | ASSERT_EQ(perm.size(), 10); 36 | for (int i = 0; i < 10; ++i) { 37 | ASSERT_EQ(std::count(perm.begin(), perm.end(), i), 1); 38 | } 39 | } 40 | 41 | TEST(UtilTest, RandomPermEmpty) { 42 | std::vector perm = kstd::RandomPerm(0); 43 | ASSERT_EQ(perm.size(), 0); 44 | } 45 | 46 | TEST(UtilTest, RandomPermNegative) { 47 | std::vector perm = kstd::RandomPerm(-1); 48 | ASSERT_EQ(perm.size(), 0); 49 | } 50 | 51 | TEST(UtilTest, UnixTimestamp) { 52 | int64_t timestamp = kstd::UnixTimestamp(); 53 | std::cout << timestamp << std::endl; 54 | ASSERT_GT(timestamp, 0); 55 | } 56 | 57 | TEST(UtilTest, UnixMilliTimestamp) { 58 | int64_t timestamp = kstd::UnixMilliTimestamp(); 59 | std::cout << timestamp << std::endl; 60 | ASSERT_GT(timestamp, 0); 61 | } 62 | 63 | TEST(UtilTest, UnixMicroTimestamp) { 64 | int64_t timestamp = kstd::UnixMicroTimestamp(); 65 | std::cout << timestamp << std::endl; 66 | ASSERT_GT(timestamp, 0); 67 | } 68 | 69 | TEST(UtilTest, UnixNanoTimestamp) { 70 | int64_t timestamp = kstd::UnixNanoTimestamp(); 71 | std::cout << timestamp << std::endl; 72 | ASSERT_GT(timestamp, 0); 73 | } 74 | 75 | int main(int argc, char** argv) { 76 | kstd::InitRandom(); 77 | ::testing::InitGoogleTest(&argc, argv); 78 | return RUN_ALL_TESTS(); 79 | } 80 | -------------------------------------------------------------------------------- /src/std/thread_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, arana-db Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace kstd { 20 | 21 | class ThreadPool final { 22 | public: 23 | ThreadPool(); 24 | ~ThreadPool(); 25 | 26 | ThreadPool(const ThreadPool&) = delete; 27 | void operator=(const ThreadPool&) = delete; 28 | 29 | template 30 | auto ExecuteTask(F&& f, Args&&... args) -> std::future>; 31 | 32 | void JoinAll(); 33 | void SetMaxIdleThread(unsigned int m); 34 | 35 | private: 36 | void CreateWorker(); 37 | void WorkerRoutine(); 38 | void MonitorRoutine(); 39 | 40 | std::thread monitor_; 41 | std::atomic maxIdleThread_; 42 | std::atomic pendingStopSignal_; 43 | 44 | static thread_local bool working_; 45 | std::deque worker_threads_; 46 | 47 | std::mutex mutex_; 48 | std::condition_variable cond_; 49 | unsigned waiters_; 50 | bool shutdown_; 51 | std::deque> tasks_; 52 | 53 | static const int kMaxThreads = 256; 54 | }; 55 | 56 | template 57 | auto ThreadPool::ExecuteTask(F&& f, Args&&... args) -> std::future> { 58 | using resultType = std::invoke_result_t; 59 | 60 | auto task = 61 | std::make_shared>(std::bind(std::forward(f), std::forward(args)...)); 62 | 63 | { 64 | std::unique_lock guard(mutex_); 65 | if (shutdown_) { 66 | return std::future(); 67 | } 68 | 69 | tasks_.emplace_back([=]() { (*task)(); }); 70 | if (waiters_ == 0) { 71 | CreateWorker(); 72 | } 73 | 74 | cond_.notify_one(); 75 | } 76 | 77 | return task->get_future(); 78 | } 79 | 80 | } // namespace kstd 81 | -------------------------------------------------------------------------------- /src/storage/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | FILE(GLOB STORAGE_SRC 7 | "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc" 8 | "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h" 9 | "${CMAKE_CURRENT_SOURCE_DIR}/include/storage/*.h" 10 | ) 11 | SET(LIBRARY_OUTPUT_PATH ${PLIB_INSTALL_DIR}) 12 | ADD_LIBRARY(storage ${STORAGE_SRC}) 13 | 14 | TARGET_INCLUDE_DIRECTORIES(storage 15 | PUBLIC ${CMAKE_SOURCE_DIR}/src 16 | PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} 17 | PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include 18 | PRIVATE ${ROCKSDB_SOURCES_DIR} 19 | PRIVATE ${ROCKSDB_SOURCES_DIR}/include 20 | PRIVATE ${PROTO_OUTPUT_DIR} 21 | PRIVATE ${PROTOBUF_INCLUDE_DIR} 22 | PRIVATE ${SPDLOG_INCLUDE_DIR} 23 | PRIVATE ${PSTD_INCLUDE_DIR} 24 | ) 25 | 26 | TARGET_LINK_LIBRARIES(storage 27 | PRIVATE kstd 28 | PRIVATE ${FMT_LIBRARIES} 29 | PRIVATE ${ZLIB_LIBRARIES} 30 | PRIVATE ${LEVELDB_LIBRARIES} 31 | PRIVATE ${GFLAGS_LIBRARIES} 32 | PRIVATE ${ROCKSDB_LIBRARIES} 33 | PRIVATE ${PROTOBUF_LIBRARY} 34 | PRIVATE ${SPDLOG_LIBRARIES} 35 | ) 36 | 37 | SET_TARGET_PROPERTIES(storage PROPERTIES LINKER_LANGUAGE CXX) 38 | 39 | ADD_SUBDIRECTORY(tests) 40 | 41 | ADD_DEPENDENCIES(storage 42 | kstd 43 | zlib 44 | leveldb 45 | gflags 46 | rocksdb 47 | protobuf 48 | spdlog 49 | ) 50 | -------------------------------------------------------------------------------- /src/storage/detect_environment: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | OUTPUT=$1 4 | if test -z "$OUTPUT"; then 5 | echo "usage: $0 " >&2 6 | exit 1 7 | fi 8 | 9 | # Delete existing output, if it exists 10 | rm -f "$OUTPUT" 11 | touch "$OUTPUT" 12 | 13 | if test -z "$CXX"; then 14 | CXX=g++ 15 | fi 16 | 17 | # Test whether Snappy library is installed 18 | # http://code.google.com/p/snappy/ 19 | $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < 21 | int main() {} 22 | EOF 23 | if [ "$?" = 0 ]; then 24 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lsnappy" 25 | fi 26 | 27 | # Test whether gflags library is installed 28 | # http://gflags.github.io/gflags/ 29 | # check if the namespace is gflags 30 | $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF 31 | #include 32 | using namespace gflags; 33 | int main() {} 34 | EOF 35 | if [ "$?" = 0 ]; then 36 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lgflags" 37 | else 38 | # check if namespace is google 39 | $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF 40 | #include 41 | using namespace google; 42 | int main() {} 43 | EOF 44 | if [ "$?" = 0 ]; then 45 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lgflags" 46 | fi 47 | fi 48 | 49 | # Test whether zlib library is installed 50 | $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < 52 | int main() {} 53 | EOF 54 | if [ "$?" = 0 ]; then 55 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lz" 56 | fi 57 | 58 | # Test whether bzip library is installed 59 | $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < 61 | int main() {} 62 | EOF 63 | if [ "$?" = 0 ]; then 64 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lbz2" 65 | fi 66 | 67 | # Test whether lz4 library is installed 68 | $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < 70 | #include 71 | int main() {} 72 | EOF 73 | if [ "$?" = 0 ]; then 74 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -llz4" 75 | fi 76 | 77 | # Test whether zstd library is installed 78 | $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < 80 | int main() {} 81 | EOF 82 | if [ "$?" = 0 ]; then 83 | ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lzstd" 84 | fi 85 | 86 | 87 | 88 | # Test processor nums 89 | PROCESSOR_NUMS=$(cat /proc/cpuinfo | grep processor | wc -l) 90 | 91 | echo "ROCKSDB_LDFLAGS=$ROCKSDB_LDFLAGS" >> "$OUTPUT" 92 | echo "PROCESSOR_NUMS=$PROCESSOR_NUMS" >> "$OUTPUT" 93 | -------------------------------------------------------------------------------- /src/storage/include/storage/build_version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef INCLUDE_STORAGE_BUILD_VERSION_H_ 7 | #define INCLUDE_STORAGE_BUILD_VERSION_H_ 8 | 9 | // this variable tells us about the git revision 10 | extern const char* floyd_build_git_sha; 11 | 12 | // Date on which the code was compiled: 13 | extern const char* floyd_build_compile_date; 14 | 15 | #endif // INCLUDE_STORAGE_BUILD_VERSION_H_ 16 | -------------------------------------------------------------------------------- /src/storage/include/storage/slot_indexer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef __SLOT_INDEXER_H__ 7 | #define __SLOT_INDEXER_H__ 8 | 9 | #include 10 | #include 11 | 12 | namespace storage { 13 | // Manage slots to rocksdb indexes 14 | // TODO(wangshaoyi): temporarily mock return 15 | class SlotIndexer { 16 | public: 17 | explicit SlotIndexer(int32_t inst_num) : inst_num_(inst_num) { assert(inst_num > 0); } 18 | SlotIndexer() = delete; 19 | ~SlotIndexer() {} 20 | uint32_t GetInstanceID(uint32_t slot_id) { return slot_id % inst_num_; } 21 | void ReshardSlots(const std::vector& slots) {} 22 | 23 | private: 24 | int32_t inst_num_ = 3; 25 | }; 26 | } // namespace storage 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/storage/include/storage/util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present The storage Authors. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef SRC_UTIL_H_ 7 | #define SRC_UTIL_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace storage { 17 | 18 | int Int64ToStr(char* dst, size_t dstlen, int64_t svalue); 19 | int StrToInt64(const char* s, size_t slen, int64_t* value); 20 | int StrToInt64Strict(const char* s, size_t slen, int64_t* value); 21 | int StringMatch(const char* pattern, uint64_t pattern_len, const char* string, uint64_t string_len, int nocase); 22 | int StrToLongDouble(const char* s, size_t slen, long double* ldval); 23 | int LongDoubleToStr(long double ldval, std::string* value); 24 | int do_mkdir(const char* path, mode_t mode); 25 | int mkpath(const char* path, mode_t mode); 26 | int delete_dir(const char* dirname); 27 | int is_dir(const char* filename); 28 | int CalculateStartAndEndKey(const std::string& key, std::string* start_key, std::string* end_key); 29 | bool isTailWildcard(const std::string& pattern); 30 | void GetFilepath(const char* path, const char* filename, char* filepath); 31 | bool DeleteFiles(const char* path); 32 | } // namespace storage 33 | 34 | #endif // SRC_UTIL_H_ 35 | -------------------------------------------------------------------------------- /src/storage/include/storage/version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #define STORAGE_MAJOR 1 9 | #define STORAGE_MINOR 0 10 | #define STORAGE_PATCH 0 11 | -------------------------------------------------------------------------------- /src/storage/src/build_version.cc.in: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "storage/build_version.h" 7 | const char* storage_build_git_sha = "storage_build_git_sha:@@GIT_SHA@@"; 8 | const char* storage_build_git_date = "storage_build_git_date:@@GIT_DATE_TIME@@"; 9 | const char* storage_build_compile_date = __DATE__; 10 | -------------------------------------------------------------------------------- /src/storage/src/debug.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #ifndef NDEBUG 9 | # define TRACE(M, ...) fprintf(stderr, "[TRACE] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 10 | #else 11 | # define TRACE(M, ...) \ 12 | {} 13 | #endif // NDEBUG 14 | -------------------------------------------------------------------------------- /src/storage/src/lock_mgr.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2024-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | #include "std/lock_mgr.h" 12 | 13 | #include "src/mutex.h" 14 | 15 | namespace storage { 16 | 17 | using LockMgr = kstd::lock::LockMgr; 18 | 19 | } // namespace storage 20 | -------------------------------------------------------------------------------- /src/storage/src/murmurhash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | /* 7 | Murmurhash from http://sites.google.com/site/murmurhash/ 8 | 9 | All code is released to the public domain. For business purposes, Murmurhash 10 | is under the MIT license. 11 | */ 12 | #pragma once 13 | 14 | #include 15 | #include "rocksdb/slice.h" 16 | 17 | #if defined(__x86_64__) 18 | # define MURMUR_HASH MurmurHash64A 19 | uint64_t MurmurHash64A(const void* key, int len, unsigned int seed); 20 | # define MurmurHash MurmurHash64A 21 | typedef uint64_t murmur_t; 22 | 23 | #elif defined(__i386__) 24 | # define MURMUR_HASH MurmurHash2 25 | unsigned int MurmurHash2(const void* key, int len, unsigned int seed); 26 | # define MurmurHash MurmurHash2 27 | typedef unsigned int murmur_t; 28 | 29 | #else 30 | # define MURMUR_HASH MurmurHashNeutral2 31 | unsigned int MurmurHashNeutral2(const void* key, int len, unsigned int seed); 32 | # define MurmurHash MurmurHashNeutral2 33 | using murmur_t = unsigned int; 34 | #endif 35 | 36 | // Allow slice to be hashable by murmur hash. 37 | namespace storage { 38 | using Slice = rocksdb::Slice; 39 | struct murmur_hash { 40 | size_t operator()(const Slice& slice) const { return MurmurHash(slice.data(), static_cast(slice.size()), 0); } 41 | }; 42 | } // namespace storage 43 | -------------------------------------------------------------------------------- /src/storage/src/mutex.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "rocksdb/status.h" 11 | 12 | #include "std/mutex.h" 13 | 14 | namespace storage { 15 | 16 | using Status = rocksdb::Status; 17 | 18 | using Mutex = kstd::lock::Mutex; 19 | using CondVar = kstd::lock::CondVar; 20 | using MutexFactory = kstd::lock::MutexFactory; 21 | 22 | } // namespace storage 23 | -------------------------------------------------------------------------------- /src/storage/src/mutex_impl.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include "src/mutex.h" 9 | 10 | #include "std/mutex_impl.h" 11 | 12 | #include 13 | 14 | namespace storage { 15 | 16 | using MutexFactoryImpl = kstd::lock::MutexFactoryImpl; 17 | 18 | } // namespace storage 19 | -------------------------------------------------------------------------------- /src/storage/src/options_helper.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "src/options_helper.h" 7 | 8 | #include 9 | 10 | namespace storage { 11 | 12 | // strToInt may throw exception 13 | static bool strToInt(const std::string& value, int* num, int base = 10) { 14 | size_t end; 15 | *num = std::stoi(value, &end, base); 16 | return end >= value.size(); 17 | } 18 | 19 | // strToUint64 may throw exception 20 | static bool strToUint64(const std::string& value, uint64_t* num, int base = 10) { 21 | size_t end; 22 | *num = std::stoull(value, &end, base); 23 | return end >= value.size(); 24 | } 25 | 26 | // strToUint32 may throw exception 27 | static bool strToUint32(const std::string& value, uint32_t* num, int base = 10) { 28 | uint64_t uint64Val; 29 | if (!strToUint64(value, &uint64Val)) { 30 | return false; 31 | } 32 | if ((uint64Val >> 32LL) == 0) { 33 | *num = static_cast(uint64Val); 34 | } else { 35 | throw std::out_of_range(value); 36 | } 37 | return true; 38 | } 39 | 40 | bool ParseOptionMember(const MemberType& member_type, const std::string& value, char* member_address) { 41 | switch (member_type) { 42 | case MemberType::kInt: { 43 | int intVal; 44 | if (!strToInt(value, &intVal)) { 45 | return false; 46 | } 47 | *reinterpret_cast(member_address) = intVal; 48 | break; 49 | } 50 | case MemberType::kUint: { 51 | uint32_t uint32Val; 52 | if (!strToUint32(value, &uint32Val)) { 53 | return false; 54 | } 55 | *reinterpret_cast(member_address) = static_cast(uint32Val); 56 | break; 57 | } 58 | case MemberType::kUint64T: { 59 | uint64_t uint64Val; 60 | if (!strToUint64(value, &uint64Val)) { 61 | return false; 62 | } 63 | *reinterpret_cast(member_address) = uint64Val; 64 | break; 65 | } 66 | case MemberType::kSizeT: { 67 | uint64_t uint64Val; 68 | if (!strToUint64(value, &uint64Val)) { 69 | return false; 70 | } 71 | *reinterpret_cast(member_address) = static_cast(uint64Val); 72 | break; 73 | } 74 | default: { 75 | return false; 76 | } 77 | } 78 | return true; 79 | } 80 | 81 | } // namespace storage 82 | -------------------------------------------------------------------------------- /src/storage/src/redis_hyperloglog.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef SRC_REDIS_HYPERLOGLOG_H_ 7 | #define SRC_REDIS_HYPERLOGLOG_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace storage { 15 | 16 | class HyperLogLog { 17 | public: 18 | HyperLogLog(uint8_t precision, std::string origin_register); 19 | ~HyperLogLog(); 20 | 21 | double Estimate() const; 22 | double FirstEstimate() const; 23 | uint32_t CountZero() const; 24 | double Alpha() const; 25 | uint8_t Nctz(uint32_t x, int b); 26 | 27 | std::string Add(const char* value, uint32_t len); 28 | std::string Merge(const HyperLogLog& hll); 29 | 30 | protected: 31 | uint32_t m_ = 0; // register bit width 32 | uint32_t b_ = 0; // regieter size 33 | double alpha_ = 0; 34 | std::unique_ptr register_; 35 | }; 36 | 37 | } // namespace storage 38 | 39 | #endif // SRC_REDIS_HYPERLOGLOG_H_ 40 | -------------------------------------------------------------------------------- /src/storage/src/scope_record_lock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "src/lock_mgr.h" 14 | #include "std/scope_record_lock.h" 15 | #include "storage/storage.h" 16 | 17 | namespace storage { 18 | 19 | using ScopeRecordLock = kstd::lock::ScopeRecordLock; 20 | using MultiScopeRecordLock = kstd::lock::MultiScopeRecordLock; 21 | 22 | } // namespace storage 23 | -------------------------------------------------------------------------------- /src/storage/src/scope_snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #pragma once 7 | 8 | #include "rocksdb/db.h" 9 | 10 | #include "std/noncopyable.h" 11 | 12 | namespace storage { 13 | class ScopeSnapshot : public kstd::noncopyable { 14 | public: 15 | ScopeSnapshot(rocksdb::DB* db, const rocksdb::Snapshot** snapshot) : db_(db), snapshot_(snapshot) { 16 | *snapshot_ = db_->GetSnapshot(); 17 | } 18 | ~ScopeSnapshot() override { db_->ReleaseSnapshot(*snapshot_); } 19 | 20 | private: 21 | rocksdb::DB* const db_; 22 | const rocksdb::Snapshot** snapshot_; 23 | }; 24 | 25 | } // namespace storage 26 | -------------------------------------------------------------------------------- /src/storage/src/strings_filter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #ifndef SRC_STRINGS_FILTER_H_ 7 | #define SRC_STRINGS_FILTER_H_ 8 | 9 | #include 10 | #include 11 | 12 | #include "rocksdb/compaction_filter.h" 13 | #include "src/debug.h" 14 | #include "src/strings_value_format.h" 15 | 16 | namespace storage { 17 | 18 | class StringsFilter : public rocksdb::CompactionFilter { 19 | public: 20 | StringsFilter() = default; 21 | bool Filter(int level, const rocksdb::Slice& key, const rocksdb::Slice& value, std::string* new_value, 22 | bool* value_changed) const override { 23 | int64_t unix_time; 24 | rocksdb::Env::Default()->GetCurrentTime(&unix_time); 25 | auto cur_time = static_cast(unix_time); 26 | ParsedStringsValue parsed_strings_value(value); 27 | TRACE("==========================START=========================="); 28 | TRACE("[StringsFilter], key: %s, value = %s, timestamp: %lu, cur_time: %d", key.ToString().c_str(), 29 | parsed_strings_value.UserValue().ToString().c_str(), parsed_strings_value.Etime(), cur_time); 30 | 31 | if (parsed_strings_value.Etime() != 0 && parsed_strings_value.Etime() < cur_time) { 32 | TRACE("Drop[Stale]"); 33 | return true; 34 | } else { 35 | TRACE("Reserve"); 36 | return false; 37 | } 38 | } 39 | 40 | const char* Name() const override { return "StringsFilter"; } 41 | }; 42 | 43 | class StringsFilterFactory : public rocksdb::CompactionFilterFactory { 44 | public: 45 | StringsFilterFactory() = default; 46 | std::unique_ptr CreateCompactionFilter( 47 | const rocksdb::CompactionFilter::Context& context) override { 48 | return std::unique_ptr(new StringsFilter()); 49 | } 50 | const char* Name() const override { return "StringsFilterFactory"; } 51 | }; 52 | 53 | } // namespace storage 54 | #endif // SRC_STRINGS_FILTER_H_ 55 | -------------------------------------------------------------------------------- /src/storage/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | # This source code is licensed under the BSD-style license found in the 3 | # LICENSE file in the root directory of this source tree. An additional grant 4 | # of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | INCLUDE(GoogleTest) 7 | 8 | FILE(GLOB_RECURSE TEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*test.cc") 9 | 10 | FOREACH (TEST_SOURCE ${TEST_SOURCES}) 11 | GET_FILENAME_COMPONENT(TEST_FILENAME ${TEST_SOURCE} NAME) 12 | STRING(REPLACE ".cc" "" TEST_NAME ${TEST_FILENAME}) 13 | 14 | ADD_EXECUTABLE(${TEST_NAME} ${TEST_SOURCE}) 15 | 16 | ADD_DEPENDENCIES(${TEST_NAME} storage) 17 | TARGET_INCLUDE_DIRECTORIES(${TEST_NAME} 18 | PRIVATE storage 19 | PRIVATE ${ROCKSDB_SOURCES_DIR} 20 | PRIVATE ${ROCKSDB_SOURCES_DIR}/include 21 | PRIVATE ${BRAFT_INCLUDE_DIR} 22 | PRIVATE ${GTEST_INCLUDE_DIR} 23 | PRIVATE ${PROTO_OUTPUT_DIR} 24 | PRIVATE ${PROTOBUF_INCLUDE_DIR} 25 | PRIVATE ${SPDLOG_INCLUDE_DIR} 26 | PRIVATE ${PSTD_INCLUDE_DIR} 27 | ) 28 | 29 | TARGET_LINK_LIBRARIES(${TEST_NAME} 30 | PRIVATE storage 31 | PRIVATE ${GTEST_LIBRARIES} 32 | PRIVATE ${GTEST_MAIN_LIBRARIES} 33 | PRIVATE ${FMT_LIBRARIES} 34 | PRIVATE ${SPDLOG_LIBRARIES} 35 | PRIVATE kstd 36 | PRIVATE ${ROCKSDB_LIBRARIES} 37 | PRIVATE ${Snappy_LIBRARIES} 38 | PRIVATE ${LZ4_LIBRARIES} 39 | PRIVATE ${zstd_LIBRARIES} 40 | PRIVATE ${PROTOBUF_LIBRARY} 41 | PRIVATE binlog_pb 42 | ) 43 | GTEST_DISCOVER_TESTS(${TEST_NAME}) 44 | ENDFOREACH () 45 | -------------------------------------------------------------------------------- /src/storage/tests/lock_mgr_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include 8 | 9 | #include "src/lock_mgr.h" 10 | #include "src/mutex_impl.h" 11 | 12 | using namespace storage; 13 | 14 | void Func(LockMgr* mgr, int id, const std::string& key) { 15 | mgr->TryLock(key); 16 | printf("thread %d TryLock %s success\n", id, key.c_str()); 17 | std::this_thread::sleep_for(std::chrono::seconds(3)); 18 | mgr->UnLock(key); 19 | printf("thread %d UnLock %s\n", id, key.c_str()); 20 | } 21 | 22 | TEST(LockMgr, LockMgrTest) { 23 | std::shared_ptr factory = std::make_shared(); 24 | LockMgr mgr(1, 3, factory); 25 | 26 | std::thread t1(Func, &mgr, 1, "key_1"); 27 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 28 | std::thread t2(Func, &mgr, 2, "key_2"); 29 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 30 | std::thread t3(Func, &mgr, 3, "key_3"); 31 | std::thread t4(Func, &mgr, 4, "key_4"); 32 | 33 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 34 | 35 | auto s = mgr.TryLock("key_1"); 36 | printf("thread main TryLock key_1 ret %s\n", s.ToString().c_str()); 37 | mgr.UnLock("key_1"); 38 | printf("thread main UnLock key_1\n"); 39 | 40 | t1.join(); 41 | t2.join(); 42 | t3.join(); 43 | t4.join(); 44 | } 45 | 46 | int main(int argc, char** argv) { 47 | ::testing::InitGoogleTest(&argc, argv); 48 | return RUN_ALL_TESTS(); 49 | } -------------------------------------------------------------------------------- /src/storage/tests/options_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include 8 | 9 | #include "storage/storage.h" 10 | 11 | using namespace storage; 12 | 13 | class StorageOptionsTest : public ::testing::Test { 14 | public: 15 | StorageOptionsTest() = default; 16 | 17 | ~StorageOptionsTest() override = default; 18 | 19 | static void SetUpTestSuite() {} 20 | 21 | static void TearDownTestSuite() {} 22 | 23 | StorageOptions storage_options; 24 | storage::Status s; 25 | }; 26 | 27 | // ResetOptions 28 | TEST_F(StorageOptionsTest, ResetOptionsTest) { 29 | std::unordered_map cf_options_map{{"write_buffer_size", "4096"}, 30 | {"max_write_buffer_number", "10"}}; 31 | s = storage_options.ResetOptions(OptionType::kColumnFamily, cf_options_map); 32 | ASSERT_TRUE(s.ok()); 33 | ASSERT_EQ(storage_options.options.write_buffer_size, 4096); 34 | ASSERT_EQ(storage_options.options.max_write_buffer_number, 10); 35 | 36 | std::unordered_map invalid_cf_options_map{{"write_buffer_size", "abc"}, 37 | {"max_write_buffer_number", "0x33"}}; 38 | s = storage_options.ResetOptions(OptionType::kColumnFamily, invalid_cf_options_map); 39 | ASSERT_FALSE(s.ok()); 40 | ASSERT_EQ(storage_options.options.write_buffer_size, 4096); 41 | ASSERT_EQ(storage_options.options.max_write_buffer_number, 10); 42 | 43 | std::unordered_map db_options_map{{"max_open_files", "16"}, 44 | {"max_background_compactions", "32"}}; 45 | s = storage_options.ResetOptions(OptionType::kDB, db_options_map); 46 | ASSERT_TRUE(s.ok()); 47 | ASSERT_EQ(storage_options.options.max_open_files, 16); 48 | ASSERT_EQ(storage_options.options.max_background_compactions, 32); 49 | 50 | std::unordered_map invalid_db_options_map{{"max_open_files", "a"}, 51 | {"max_background_compactions", "bac"}}; 52 | s = storage_options.ResetOptions(OptionType::kDB, invalid_db_options_map); 53 | ASSERT_FALSE(s.ok()); 54 | ASSERT_EQ(storage_options.options.max_open_files, 16); 55 | ASSERT_EQ(storage_options.options.max_background_compactions, 32); 56 | } 57 | 58 | int main(int argc, char **argv) { 59 | ::testing::InitGoogleTest(&argc, argv); 60 | return RUN_ALL_TESTS(); 61 | } 62 | -------------------------------------------------------------------------------- /src/storage/tests/strings_filter_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include 7 | #include 8 | 9 | #include "src/strings_filter.h" 10 | #include "storage/storage.h" 11 | 12 | using namespace storage; 13 | 14 | // Filter 15 | TEST(StringsFilterTest, FilterTest) { 16 | std::string new_value; 17 | bool is_stale; 18 | bool value_changed; 19 | auto filter = std::make_unique(); 20 | 21 | int64_t ttl = 1; 22 | StringsValue strings_value("FILTER_VALUE"); 23 | strings_value.SetRelativeTimestamp(ttl); 24 | is_stale = filter->Filter(0, "FILTER_KEY", strings_value.Encode(), &new_value, &value_changed); 25 | ASSERT_FALSE(is_stale); 26 | std::this_thread::sleep_for(std::chrono::milliseconds(2000)); 27 | is_stale = filter->Filter(0, "FILTER_KEY", strings_value.Encode(), &new_value, &value_changed); 28 | ASSERT_TRUE(is_stale); 29 | } 30 | 31 | int main(int argc, char **argv) { 32 | ::testing::InitGoogleTest(&argc, argv); 33 | return RUN_ALL_TESTS(); 34 | } 35 | -------------------------------------------------------------------------------- /src/store.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Implemented a set of functions and instructions for 8 | coordinating external commands and interfacing with RocksDB. 9 | */ 10 | 11 | #include "store.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "config.h" 18 | #include "db.h" 19 | #include "std/log.h" 20 | #include "std/std_string.h" 21 | 22 | namespace kiwi { 23 | 24 | Store::~Store() { INFO("STORE is closing..."); } 25 | 26 | Store& Store::Instance() { 27 | static Store store; 28 | return store; 29 | } 30 | 31 | void Store::Init(int db_number) { 32 | db_number_ = db_number; 33 | backends_.reserve(db_number_); 34 | for (int i = 0; i < db_number_; i++) { 35 | auto db = std::make_unique(i, g_config.db_path); 36 | db->Open(); 37 | backends_.push_back(std::move(db)); 38 | INFO("Open DB_{} success!", i); 39 | } 40 | INFO("STORE Init success!"); 41 | } 42 | 43 | void Store::HandleTaskSpecificDB(const TasksVector& tasks) { 44 | std::for_each(tasks.begin(), tasks.end(), [this](const auto& task) { 45 | if (task.db < 0 || task.db >= db_number_) { 46 | WARN("The database index is out of range."); 47 | return; 48 | } 49 | auto& db = backends_.at(task.db); 50 | switch (task.type) { 51 | case kCheckpoint: { 52 | if (auto s = task.args.find(kCheckpointPath); s == task.args.end()) { 53 | WARN("The critical parameter 'path' is missing for do a checkpoint."); 54 | return; 55 | } 56 | auto path = task.args.find(kCheckpointPath)->second; 57 | kstd::TrimSlash(path); 58 | db->CreateCheckpoint(path, task.sync); 59 | break; 60 | } 61 | case kLoadDBFromCheckpoint: { 62 | if (auto s = task.args.find(kCheckpointPath); s == task.args.end()) { 63 | WARN("The critical parameter 'path' is missing for load a checkpoint."); 64 | return; 65 | } 66 | auto path = task.args.find(kCheckpointPath)->second; 67 | kstd::TrimSlash(path); 68 | db->LoadDBFromCheckpoint(path, task.sync); 69 | break; 70 | } 71 | case kEmpty: { 72 | WARN("A empty task was passed in, not doing anything."); 73 | break; 74 | } 75 | default: 76 | break; 77 | } 78 | }); 79 | } 80 | } // namespace kiwi 81 | -------------------------------------------------------------------------------- /src/store.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | Connects external commands and coordinates the operation 8 | of RocksDB. 9 | */ 10 | 11 | #pragma once 12 | 13 | #define GLOG_NO_ABBREVIATED_SEVERITIES 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "common.h" 20 | #include "db.h" 21 | #include "storage/storage.h" 22 | 23 | namespace kiwi { 24 | 25 | enum TaskType { kCheckpoint = 0, kLoadDBFromCheckpoint, kEmpty }; 26 | 27 | enum TaskArg { 28 | kCheckpointPath = 0, 29 | }; 30 | 31 | constexpr const char* ErrTypeMessage = "WRONGTYPE"; 32 | 33 | struct TaskContext { 34 | TaskType type = kEmpty; 35 | int db = -1; 36 | std::map args; 37 | bool sync = false; 38 | TaskContext() = delete; 39 | TaskContext(TaskType t, bool s = false) : type(t), sync(s) {} 40 | TaskContext(TaskType t, int d, bool s = false) : type(t), db(d), sync(s) {} 41 | TaskContext(TaskType t, int d, const std::map& a, bool s = false) 42 | : type(t), db(d), args(a), sync(s) {} 43 | }; 44 | 45 | using TasksVector = std::vector; 46 | 47 | class Store { 48 | public: 49 | static Store& Instance(); 50 | 51 | Store(const Store&) = delete; 52 | void operator=(const Store&) = delete; 53 | ~Store(); 54 | 55 | void Init(int db_number); 56 | 57 | std::unique_ptr& GetBackend(int32_t index) { return backends_[index]; }; 58 | 59 | void HandleTaskSpecificDB(const TasksVector& tasks); 60 | 61 | int GetDBNumber() const { return db_number_; } 62 | 63 | private: 64 | Store() = default; 65 | int db_number_ = 0; 66 | std::vector> backends_; 67 | }; 68 | 69 | #define STORE_INST Store::Instance() 70 | 71 | } // namespace kiwi 72 | -------------------------------------------------------------------------------- /src/unbounded_buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023-present, arana-db Community. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory 5 | 6 | /* 7 | The definition part of the module that supports dynamically 8 | expandable buffer. 9 | */ 10 | 11 | #pragma once 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace kiwi { 18 | 19 | class UnboundedBuffer { 20 | public: 21 | UnboundedBuffer() = default; 22 | 23 | std::size_t PushDataAt(const void* pData, std::size_t nSize, std::size_t offset = 0); 24 | std::size_t PushData(const void* pData, std::size_t nSize); 25 | std::size_t PushData(const std::string_view& data); 26 | std::size_t Write(const void* pData, std::size_t nSize); 27 | void AdjustWritePtr(std::size_t nBytes) { writePos_ += nBytes; } 28 | 29 | std::size_t PeekDataAt(void* pBuf, std::size_t nSize, std::size_t offset = 0); 30 | std::size_t PeekData(void* pBuf, std::size_t nSize); 31 | void AdjustReadPtr(std::size_t nBytes) { readPos_ += nBytes; } 32 | 33 | char* ReadAddr() { return &buffer_[readPos_]; } 34 | char* WriteAddr() { return &buffer_[writePos_]; } 35 | 36 | bool IsEmpty() const { return ReadableSize() == 0; } 37 | std::size_t ReadableSize() const { return writePos_ - readPos_; } 38 | std::size_t WritableSize() const { return buffer_.size() - writePos_; } 39 | 40 | void Shrink(bool tight = false); 41 | void Clear(); 42 | void Swap(UnboundedBuffer& buf); 43 | 44 | std::string ToString(); 45 | 46 | static const std::size_t MAX_BUFFER_SIZE; 47 | 48 | private: 49 | void assureSpace(std::size_t size); 50 | std::size_t readPos_ = 0; 51 | std::size_t writePos_ = 0; 52 | std::vector buffer_; 53 | }; 54 | 55 | } // namespace kiwi 56 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ### kiwi test 2 | 3 | * 在 kiwi 目录下执行 `sh ./etc/script/Arana/Kiwitests.sh geo` 测试kiwi GEO命令 4 | * 如果是`unit/type`接口, 例如 SET, 执行 `./etc/script/kiwitests.sh type/set` 测试kiwi SET命令 -------------------------------------------------------------------------------- /tests/assets/encodings.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arana-db/kiwi/0fa4f1b95ff2f89873d06eb86de40532dd91d146/tests/assets/encodings.rdb -------------------------------------------------------------------------------- /tests/assets/hash-zipmap.rdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arana-db/kiwi/0fa4f1b95ff2f89873d06eb86de40532dd91d146/tests/assets/hash-zipmap.rdb -------------------------------------------------------------------------------- /tests/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/OpenAtomFoundation/kiwi/tests 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/onsi/ginkgo/v2 v2.13.2 7 | github.com/onsi/gomega v1.30.0 8 | github.com/redis/go-redis/v9 v9.6.1 9 | ) 10 | 11 | require ( 12 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 13 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 14 | github.com/go-logr/logr v1.3.0 // indirect 15 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 16 | github.com/google/go-cmp v0.6.0 // indirect 17 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect 18 | golang.org/x/net v0.23.0 // indirect 19 | golang.org/x/sys v0.18.0 // indirect 20 | golang.org/x/text v0.14.0 // indirect 21 | golang.org/x/tools v0.14.0 // indirect 22 | gopkg.in/yaml.v3 v3.0.1 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /tests/helpers/bg_complex_data.tcl: -------------------------------------------------------------------------------- 1 | source tests/support/redis.tcl 2 | source tests/support/util.tcl 3 | 4 | proc bg_complex_data {host port db ops} { 5 | set r [redis $host $port] 6 | $r select $db 7 | createComplexDataset $r $ops 8 | } 9 | 10 | bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] 11 | -------------------------------------------------------------------------------- /tests/helpers/gen_write_load.tcl: -------------------------------------------------------------------------------- 1 | source tests/support/redis.tcl 2 | 3 | proc gen_write_load {host port seconds} { 4 | set start_time [clock seconds] 5 | set r [redis $host $port 1] 6 | $r select 9 7 | while 1 { 8 | $r set [expr rand()] [expr rand()] 9 | if {[clock seconds]-$start_time > $seconds} { 10 | exit 0 11 | } 12 | } 13 | } 14 | 15 | gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] 16 | -------------------------------------------------------------------------------- /tests/kiwi_suite_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-present, Arana/Kiwi Community. All rights reserved. 3 | * This source code is licensed under the BSD-style license found in the 4 | * LICENSE file in the root directory of this source tree. An additional grant 5 | * of patent rights can be found in the PATENTS file in the same directory. 6 | */ 7 | 8 | package kiwi_test 9 | 10 | import ( 11 | "math" 12 | "testing" 13 | 14 | . "github.com/onsi/ginkgo/v2" 15 | . "github.com/onsi/gomega" 16 | ) 17 | 18 | const ( 19 | DefaultKey = "key" 20 | DefaultValue = "value" 21 | Max64 = math.MaxUint64 22 | Min64 = math.MinInt64 23 | OK = "OK" 24 | Nil = "" 25 | ) 26 | 27 | // cmd 28 | const ( 29 | kCmdSelect = "select" 30 | ) 31 | 32 | // err 33 | const ( 34 | kInvalidIndex = "ERR invalid DB index for 'select DB index is out of range'" 35 | ) 36 | 37 | func TestKiwi(t *testing.T) { 38 | RegisterFailHandler(Fail) 39 | RunSpecs(t, "Tests Suite") 40 | } 41 | -------------------------------------------------------------------------------- /tests/print_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | pwd=$(cd "$(dirname "$0")" && pwd) 4 | 5 | for file in "$pwd"/test_*.log; do 6 | if [ -f "$file" ]; then 7 | echo "\n\n\n============================================================================================" 8 | echo "File: $file" 9 | cat "$file" 10 | echo "" 11 | fi 12 | done -------------------------------------------------------------------------------- /tests/sentinel/run.tcl: -------------------------------------------------------------------------------- 1 | # Sentinel test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com 2 | # This software is released under the BSD License. See the COPYING file for 3 | # more information. 4 | 5 | cd tests/sentinel 6 | source ../instances.tcl 7 | 8 | set ::instances_count 5 ; # How many instances we use at max. 9 | 10 | proc main {} { 11 | parse_options 12 | spawn_instance sentinel $::sentinel_base_port $::instances_count 13 | spawn_instance redis $::redis_base_port $::instances_count 14 | run_tests 15 | cleanup 16 | } 17 | 18 | if {[catch main e]} { 19 | puts $::errorInfo 20 | cleanup 21 | exit 1 22 | } 23 | -------------------------------------------------------------------------------- /tests/sentinel/tests/01-conf-update.tcl: -------------------------------------------------------------------------------- 1 | # Test Sentinel configuration consistency after partitions heal. 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "We can failover with Sentinel 1 crashed" { 6 | set old_port [RI $master_id tcp_port] 7 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 8 | assert {[lindex $addr 1] == $old_port} 9 | 10 | # Crash Sentinel 1 11 | kill_instance sentinel 1 12 | 13 | kill_instance redis $master_id 14 | foreach_sentinel_id id { 15 | if {$id != 1} { 16 | wait_for_condition 1000 50 { 17 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 18 | } else { 19 | fail "Sentinel $id did not received failover info" 20 | } 21 | } 22 | } 23 | restart_instance redis $master_id 24 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 25 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] 26 | } 27 | 28 | test "After Sentinel 1 is restarted, its config gets updated" { 29 | restart_instance sentinel 1 30 | wait_for_condition 1000 50 { 31 | [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 32 | } else { 33 | fail "Restarted Sentinel did not received failover info" 34 | } 35 | } 36 | 37 | test "New master [join $addr {:}] role matches" { 38 | assert {[RI $master_id role] eq {master}} 39 | } 40 | -------------------------------------------------------------------------------- /tests/sentinel/tests/03-runtime-reconf.tcl: -------------------------------------------------------------------------------- 1 | # Test runtime reconfiguration command SENTINEL SET. 2 | -------------------------------------------------------------------------------- /tests/sentinel/tests/04-slave-selection.tcl: -------------------------------------------------------------------------------- 1 | # Test slave selection algorithm. 2 | # 3 | # This unit should test: 4 | # 1) That when there are no suitable slaves no failover is performed. 5 | # 2) That among the available slaves, the one with better offset is picked. 6 | -------------------------------------------------------------------------------- /tests/sentinel/tests/05-manual.tcl: -------------------------------------------------------------------------------- 1 | # Test manual failover 2 | 3 | source "../tests/includes/init-tests.tcl" 4 | 5 | test "Manual failover works" { 6 | set old_port [RI $master_id tcp_port] 7 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 8 | assert {[lindex $addr 1] == $old_port} 9 | S 0 SENTINEL FAILOVER mymaster 10 | foreach_sentinel_id id { 11 | wait_for_condition 1000 50 { 12 | [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port 13 | } else { 14 | fail "At least one Sentinel did not received failover info" 15 | } 16 | } 17 | set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 18 | set master_id [get_instance_id_by_port redis [lindex $addr 1]] 19 | } 20 | 21 | test "New master [join $addr {:}] role matches" { 22 | assert {[RI $master_id role] eq {master}} 23 | } 24 | 25 | test "All the other slaves now point to the new master" { 26 | foreach_redis_id id { 27 | if {$id != $master_id && $id != 0} { 28 | wait_for_condition 1000 50 { 29 | [RI $id master_port] == [lindex $addr 1] 30 | } else { 31 | fail "Redis ID $id not configured to replicate with new master" 32 | } 33 | } 34 | } 35 | } 36 | 37 | test "The old master eventually gets reconfigured as a slave" { 38 | wait_for_condition 1000 50 { 39 | [RI 0 master_port] == [lindex $addr 1] 40 | } else { 41 | fail "Old master not reconfigured as slave of new master" 42 | } 43 | } 44 | 45 | -------------------------------------------------------------------------------- /tests/sentinel/tests/includes/init-tests.tcl: -------------------------------------------------------------------------------- 1 | # Initialization tests -- most units will start including this. 2 | 3 | test "(init) Restart killed instances" { 4 | foreach type {redis sentinel} { 5 | foreach_${type}_id id { 6 | if {[get_instance_attrib $type $id pid] == -1} { 7 | puts -nonewline "$type/$id " 8 | flush stdout 9 | restart_instance $type $id 10 | } 11 | } 12 | } 13 | } 14 | 15 | test "(init) Remove old master entry from sentinels" { 16 | foreach_sentinel_id id { 17 | catch {S $id SENTINEL REMOVE mymaster} 18 | } 19 | } 20 | 21 | set redis_slaves 4 22 | test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { 23 | create_redis_master_slave_cluster [expr {$redis_slaves+1}] 24 | } 25 | set master_id 0 26 | 27 | test "(init) Sentinels can start monitoring a master" { 28 | set sentinels [llength $::sentinel_instances] 29 | set quorum [expr {$sentinels/2+1}] 30 | foreach_sentinel_id id { 31 | S $id SENTINEL MONITOR mymaster \ 32 | [get_instance_attrib redis $master_id host] \ 33 | [get_instance_attrib redis $master_id port] $quorum 34 | } 35 | foreach_sentinel_id id { 36 | assert {[S $id sentinel master mymaster] ne {}} 37 | S $id SENTINEL SET mymaster down-after-milliseconds 2000 38 | S $id SENTINEL SET mymaster failover-timeout 20000 39 | S $id SENTINEL SET mymaster parallel-syncs 10 40 | } 41 | } 42 | 43 | test "(init) Sentinels can talk with the master" { 44 | foreach_sentinel_id id { 45 | wait_for_condition 1000 50 { 46 | [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 47 | } else { 48 | fail "Sentinel $id can't talk with the master." 49 | } 50 | } 51 | } 52 | 53 | test "(init) Sentinels are able to auto-discover other sentinels" { 54 | set sentinels [llength $::sentinel_instances] 55 | foreach_sentinel_id id { 56 | wait_for_condition 1000 50 { 57 | [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) 58 | } else { 59 | fail "At least some sentinel can't detect some other sentinel" 60 | } 61 | } 62 | } 63 | 64 | test "(init) Sentinels are able to auto-discover slaves" { 65 | foreach_sentinel_id id { 66 | wait_for_condition 1000 50 { 67 | [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves 68 | } else { 69 | fail "At least some sentinel can't detect some slave" 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /tests/sentinel/tmp/.gitignore: -------------------------------------------------------------------------------- 1 | redis_* 2 | sentinel_* 3 | -------------------------------------------------------------------------------- /tests/support/tmpfile.tcl: -------------------------------------------------------------------------------- 1 | set ::tmpcounter 0 2 | set ::tmproot "./tests/tmp" 3 | file mkdir $::tmproot 4 | 5 | # returns a dirname unique to this process to write to 6 | proc tmpdir {basename} { 7 | set dir [file join $::tmproot $basename.[pid].[incr ::tmpcounter]] 8 | file mkdir $dir 9 | set _ $dir 10 | } 11 | 12 | # return a filename unique to this process to write to 13 | proc tmpfile {basename} { 14 | file join $::tmproot $basename.[pid].[incr ::tmpcounter] 15 | } 16 | -------------------------------------------------------------------------------- /tests/unit/auth.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"auth"}} { 2 | test {AUTH fails if there is no password configured server side} { 3 | catch {r auth foo} err 4 | set _ $err 5 | } {ERR*no password*} 6 | } 7 | 8 | start_server {tags {"auth"} overrides {requirepass foobar}} { 9 | # test {AUTH fails when a wrong password is given} { 10 | # catch {r auth wrong!} err 11 | # set _ $err 12 | # } {ERR*invalid password} 13 | 14 | # test {AUTH succeeds when the right password is given} { 15 | # r auth foobar 16 | # } {OK} 17 | # 18 | # test {Once AUTH succeeded we can actually send commands to the server} { 19 | # r set foo 100 20 | # r incr foo 21 | # } {101} 22 | } 23 | 24 | start_server {tags {"auth"} overrides {userpass foobar}} { 25 | # test {AUTH fails when a wrong password is given} { 26 | # catch {r auth wrong!} err 27 | # set _ $err 28 | # } {ERR*invalid password} 29 | # 30 | # test {Arbitrary command gives an error when AUTH is required} { 31 | # catch {r set foo bar} err 32 | # set _ $err 33 | # } {ERR*NOAUTH*} 34 | 35 | # test {AUTH succeeds when the right password is given} { 36 | # r auth foobar 37 | # } {OK} 38 | # 39 | # test {Once AUTH succeeded we can actually send commands to the server} { 40 | # r set foo 100 41 | # r incr foo 42 | # } {101} 43 | } 44 | -------------------------------------------------------------------------------- /tests/unit/command.tcl: -------------------------------------------------------------------------------- 1 | # kiwi does not support the docs command 2 | 3 | start_server {tags {"command"}} { 4 | test "Command docs supported." { 5 | set doc [r command docs set] 6 | # puts $doc 7 | assert [dict exists $doc set] 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tests/unit/introspection.tcl: -------------------------------------------------------------------------------- 1 | # kiwi does not support the client command 2 | 3 | start_server {tags {"introspection"}} { 4 | test {CLIENT LIST} { 5 | r client list 6 | } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} 7 | 8 | test {MONITOR can log executed commands} { 9 | set rd [redis_deferring_client] 10 | $rd monitor 11 | r set foo bar 12 | r get foo 13 | list [$rd read] [$rd read] [$rd read] 14 | } {*OK*"set" "foo"*"get" "foo"*} 15 | 16 | test {MONITOR can log commands issued by the scripting engine} { 17 | set rd [redis_deferring_client] 18 | $rd monitor 19 | r eval {redis.call('set',KEYS[1],ARGV[1])} 1 foo bar 20 | $rd read ;# Discard the OK 21 | assert_match {*eval*} [$rd read] 22 | assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] 23 | } 24 | 25 | test {CLIENT GETNAME should return NIL if name is not assigned} { 26 | r client getname 27 | } {} 28 | 29 | test {CLIENT LIST shows empty fields for unassigned names} { 30 | r client list 31 | } {*name= *} 32 | 33 | test {CLIENT SETNAME does not accept spaces} { 34 | catch {r client setname "foo bar"} e 35 | set e 36 | } {ERR*} 37 | 38 | test {CLIENT SETNAME can assign a name to this connection} { 39 | assert_equal [r client setname myname] {OK} 40 | r client list 41 | } {*name=myname*} 42 | 43 | test {CLIENT SETNAME can change the name of an existing connection} { 44 | assert_equal [r client setname someothername] {OK} 45 | r client list 46 | } {*name=someothername*} 47 | 48 | test {After CLIENT SETNAME, connection can still be closed} { 49 | set rd [redis_deferring_client] 50 | $rd client setname foobar 51 | assert_equal [$rd read] "OK" 52 | assert_match {*foobar*} [r client list] 53 | $rd close 54 | # Now the client should no longer be listed 55 | wait_for_condition 50 100 { 56 | [string match {*foobar*} [r client list]] == 0 57 | } else { 58 | fail "Client still listed in CLIENT LIST after SETNAME." 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /tests/unit/keys.tcl: -------------------------------------------------------------------------------- 1 | # TODO all param is invalid 2 | start_server {tags {"keys"}} { 3 | test {KEYS with pattern} { 4 | foreach key {key_x key_y key_z foo_a foo_b foo_c} { 5 | r set $key hello 6 | } 7 | assert_equal {foo_a foo_b foo_c} [r keys foo*] 8 | assert_equal {foo_a foo_b foo_c} [r keys f*] 9 | assert_equal {foo_a foo_b foo_c} [r keys f*o*] 10 | } 11 | 12 | test {KEYS to get all keys} { 13 | lsort [r keys *] 14 | } {foo_a foo_b foo_c key_x key_y key_z} 15 | 16 | test {KEYS select by type} { 17 | foreach key {key_x key_y key_z foo_a foo_b foo_c} { 18 | r del $key 19 | } 20 | r set kv_1 value 21 | r set kv_2 value 22 | r hset hash_1 hash_field 1 23 | r hset hash_2 hash_field 1 24 | r lpush list_1 value 25 | r lpush list_2 value 26 | r zadd zset_1 1 "a" 27 | r zadd zset_2 1 "a" 28 | r sadd set_1 "a" 29 | r sadd set_2 "a" 30 | assert_equal {kv_1 kv_2} [r keys * string] 31 | assert_equal {hash_1 hash_2} [r keys * hash] 32 | assert_equal {list_1 list_2} [r keys * list] 33 | assert_equal {zset_1 zset_2} [r keys * zset] 34 | assert_equal {set_1 set_2} [r keys * set] 35 | assert_equal {kv_1 kv_2 hash_1 hash_2 zset_1 zset_2 set_1 set_2 list_1 list_2} [r keys *] 36 | assert_equal {kv_1 kv_2} [r keys * STRING] 37 | assert_equal {hash_1 hash_2} [r keys * HASH] 38 | assert_equal {list_1 list_2} [r keys * LIST] 39 | assert_equal {zset_1 zset_2} [r keys * ZSET] 40 | assert_equal {set_1 set_2} [r keys * SET] 41 | } 42 | 43 | test {KEYS syntax error} { 44 | catch {r keys * a} e1 45 | catch {r keys * strings} e2 46 | catch {r keys * c d} e3 47 | catch {r keys} e4 48 | catch {r keys * set zset} e5 49 | assert_equal {ERR syntax error} [set e1] 50 | assert_equal {ERR syntax error} [set e2] 51 | assert_equal {ERR syntax error} [set e3] 52 | assert_equal {ERR wrong number of arguments for 'keys' command} [set e4] 53 | assert_equal {ERR syntax error} [set e5] 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /tests/unit/latency-monitor.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"latency-monitor"}} { 2 | # Set a threshold high enough to avoid spurious latency events. 3 | r config set latency-monitor-threshold 200 4 | r latency reset 5 | 6 | # This parameter is not available in Arana/Kiwi 7 | test {Test latency events logging} { 8 | r debug sleep 0.3 9 | after 1100 10 | r debug sleep 0.4 11 | after 1100 12 | r debug sleep 0.5 13 | assert {[r latency history command] >= 3} 14 | } 15 | 16 | # This parameter is not available in Arana/Kiwi 17 | test {LATENCY HISTORY output is ok} { 18 | set min 250 19 | set max 450 20 | foreach event [r latency history command] { 21 | lassign $event time latency 22 | assert {$latency >= $min && $latency <= $max} 23 | incr min 100 24 | incr max 100 25 | set last_time $time ; # Used in the next test 26 | } 27 | } 28 | 29 | # This parameter is not available in Arana/Kiwi 30 | test {LATENCY LATEST output is ok} { 31 | foreach event [r latency latest] { 32 | lassign $event eventname time latency max 33 | assert {$eventname eq "command"} 34 | assert {$max >= 450 & $max <= 650} 35 | assert {$time == $last_time} 36 | break 37 | } 38 | } 39 | 40 | # This parameter is not available in Arana/Kiwi 41 | test {LATENCY HISTORY / RESET with wrong event name is fine} { 42 | assert {[llength [r latency history blabla]] == 0} 43 | assert {[r latency reset blabla] == 0} 44 | } 45 | 46 | # This parameter is not available in Arana/Kiwi 47 | test {LATENCY DOCTOR produces some output} { 48 | assert {[string length [r latency doctor]] > 0} 49 | } 50 | 51 | # This parameter is not available in Arana/Kiwi 52 | test {LATENCY RESET is able to reset events} { 53 | assert {[r latency reset] > 0} 54 | assert {[r latency latest] eq {}} 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tests/unit/limits.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"limits"} overrides {maxclients 10}} { 2 | test {Check if maxclients works refusing connections} { 3 | set c 0 4 | catch { 5 | while {$c < 50} { 6 | incr c 7 | set rd [redis_deferring_client] 8 | $rd ping 9 | $rd read 10 | after 100 11 | } 12 | } e 13 | assert {$c > 8 && $c <= 10} 14 | set e 15 | } {*ERR max*reached*} 16 | } 17 | -------------------------------------------------------------------------------- /tests/unit/memefficiency.tcl: -------------------------------------------------------------------------------- 1 | proc test_memory_efficiency {range} { 2 | r flushall 3 | set rd [redis_deferring_client] 4 | set base_mem [s used_memory] 5 | set written 0 6 | for {set j 0} {$j < 10000} {incr j} { 7 | set key key:$j 8 | set val [string repeat A [expr {int(rand()*$range)}]] 9 | $rd set $key $val 10 | incr written [string length $key] 11 | incr written [string length $val] 12 | incr written 2 ;# A separator is the minimum to store key-value data. 13 | } 14 | for {set j 0} {$j < 10000} {incr j} { 15 | $rd read ; # Discard replies 16 | } 17 | 18 | set current_mem [s used_memory] 19 | set used [expr {$current_mem-$base_mem}] 20 | set efficiency [expr {double($written)/$used}] 21 | return $efficiency 22 | } 23 | 24 | start_server {tags {"memefficiency"}} { 25 | foreach {size_range expected_min_efficiency} { 26 | 32 0.15 27 | 64 0.25 28 | 128 0.35 29 | 1024 0.75 30 | 16384 0.82 31 | } { 32 | test "Memory efficiency with values in range $size_range" { 33 | set efficiency [test_memory_efficiency $size_range] 34 | assert {$efficiency >= $expected_min_efficiency} 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tests/unit/obuf-limits.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"obuf-limits"}} { 2 | test {Client output buffer hard limit is enforced} { 3 | r config set client-output-buffer-limit {pubsub 100000 0 0} 4 | set rd1 [redis_deferring_client] 5 | 6 | $rd1 subscribe foo 7 | set reply [$rd1 read] 8 | assert {$reply eq "subscribe foo 1"} 9 | 10 | set omem 0 11 | while 1 { 12 | r publish foo bar 13 | set clients [split [r client list] "\r\n"] 14 | set c [split [lindex $clients 1] " "] 15 | if {![regexp {omem=([0-9]+)} $c - omem]} break 16 | if {$omem > 200000} break 17 | } 18 | assert {$omem >= 90000 && $omem < 200000} 19 | $rd1 close 20 | } 21 | 22 | test {Client output buffer soft limit is not enforced if time is not overreached} { 23 | r config set client-output-buffer-limit {pubsub 0 100000 10} 24 | set rd1 [redis_deferring_client] 25 | 26 | $rd1 subscribe foo 27 | set reply [$rd1 read] 28 | assert {$reply eq "subscribe foo 1"} 29 | 30 | set omem 0 31 | set start_time 0 32 | set time_elapsed 0 33 | while 1 { 34 | r publish foo bar 35 | set clients [split [r client list] "\r\n"] 36 | set c [split [lindex $clients 1] " "] 37 | if {![regexp {omem=([0-9]+)} $c - omem]} break 38 | if {$omem > 100000} { 39 | if {$start_time == 0} {set start_time [clock seconds]} 40 | set time_elapsed [expr {[clock seconds]-$start_time}] 41 | if {$time_elapsed >= 5} break 42 | } 43 | } 44 | assert {$omem >= 100000 && $time_elapsed >= 5 && $time_elapsed <= 10} 45 | $rd1 close 46 | } 47 | 48 | test {Client output buffer soft limit is enforced if time is overreached} { 49 | r config set client-output-buffer-limit {pubsub 0 100000 3} 50 | set rd1 [redis_deferring_client] 51 | 52 | $rd1 subscribe foo 53 | set reply [$rd1 read] 54 | assert {$reply eq "subscribe foo 1"} 55 | 56 | set omem 0 57 | set start_time 0 58 | set time_elapsed 0 59 | while 1 { 60 | r publish foo bar 61 | set clients [split [r client list] "\r\n"] 62 | set c [split [lindex $clients 1] " "] 63 | if {![regexp {omem=([0-9]+)} $c - omem]} break 64 | if {$omem > 100000} { 65 | if {$start_time == 0} {set start_time [clock seconds]} 66 | set time_elapsed [expr {[clock seconds]-$start_time}] 67 | if {$time_elapsed >= 10} break 68 | } 69 | } 70 | assert {$omem >= 100000 && $time_elapsed < 6} 71 | $rd1 close 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /tests/unit/printver.tcl: -------------------------------------------------------------------------------- 1 | start_server {} { 2 | set i [r info] 3 | regexp {Arana/Kiwi_version:(.*?)\r\n} $i - version 4 | regexp {Arana/Kiwi_git_sha:(.*?)\r\n} $i - sha1 5 | puts "Testing Arana/Kiwi version $version ($sha1)" 6 | } 7 | -------------------------------------------------------------------------------- /tests/unit/quit.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"quit"}} { 2 | proc format_command {args} { 3 | set cmd "*[llength $args]\r\n" 4 | foreach a $args { 5 | append cmd "$[string length $a]\r\n$a\r\n" 6 | } 7 | set _ $cmd 8 | } 9 | 10 | # test "QUIT returns OK" { 11 | # reconnect 12 | # assert_equal OK [r quit] 13 | # assert_error * {r ping} 14 | # } 15 | 16 | # test "Pipelined commands after QUIT must not be executed" { 17 | # reconnect 18 | # r write [format_command quit] 19 | # r write [format_command set foo bar] 20 | # r flush 21 | # assert_equal OK [r read] 22 | # assert_error * {r read} 23 | 24 | # reconnect 25 | # assert_equal {} [r get foo] 26 | # } 27 | 28 | # test "Pipelined commands after QUIT that exceed read buffer size" { 29 | # reconnect 30 | # r write [format_command quit] 31 | # r write [format_command set foo [string repeat "x" 1024]] 32 | # r flush 33 | # assert_equal OK [r read] 34 | # assert_error * {r read} 35 | # 36 | # reconnect 37 | # assert_equal {} [r get foo] 38 | # 39 | # } 40 | } 41 | -------------------------------------------------------------------------------- /tests/unit/slowlog.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { 2 | test {SLOWLOG - check that it starts with an empty log} { 3 | r slowlog len 4 | } {0} 5 | 6 | # test {SLOWLOG - only logs commands taking more time than specified} { 7 | # r config set slowlog-log-slower-than 100000 8 | # r ping 9 | # assert_equal [r slowlog len] 0 10 | # r debug sleep 0.2 11 | # assert_equal [r slowlog len] 1 12 | # } 13 | 14 | test {SLOWLOG - max entries is correctly handled} { 15 | r config set slowlog-log-slower-than 0 16 | r config set slowlog-max-len 10 17 | for {set i 0} {$i < 100} {incr i} { 18 | r ping 19 | } 20 | r slowlog len 21 | } {10} 22 | 23 | test {SLOWLOG - GET optional argument to limit output len works} { 24 | llength [r slowlog get 5] 25 | } {5} 26 | 27 | test {SLOWLOG - RESET subcommand works} { 28 | r config set slowlog-log-slower-than 100000 29 | r slowlog reset 30 | r slowlog len 31 | } {0} 32 | 33 | # test {SLOWLOG - logged entry sanity check} { 34 | # r debug sleep 0.2 35 | # set e [lindex [r slowlog get] 0] 36 | # assert_equal [llength $e] 4 37 | # assert_equal [lindex $e 0] 105 38 | # assert_equal [expr {[lindex $e 2] > 100000}] 1 39 | # assert_equal [lindex $e 3] {debug sleep 0.2} 40 | # } 41 | 42 | test {SLOWLOG - commands with too many arguments are trimmed} { 43 | r config set slowlog-log-slower-than 0 44 | r slowlog reset 45 | r sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 46 | set e [lindex [r slowlog get] 0] 47 | lindex $e 3 48 | } {sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 {... (2 more arguments)}} 49 | 50 | test {SLOWLOG - too long arguments are trimmed} { 51 | r config set slowlog-log-slower-than 0 52 | r slowlog reset 53 | set arg [string repeat A 129] 54 | r sadd set foo $arg 55 | set e [lindex [r slowlog get] 0] 56 | lindex $e 3 57 | } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} 58 | 59 | # test {SLOWLOG - EXEC is not logged, just executed commands} { 60 | # r config set slowlog-log-slower-than 100000 61 | # r slowlog reset 62 | # assert_equal [r slowlog len] 0 63 | # r multi 64 | # r debug sleep 0.2 65 | # r exec 66 | # assert_equal [r slowlog len] 1 67 | # set e [lindex [r slowlog get] 0] 68 | # assert_equal [lindex $e 3] {debug sleep 0.2} 69 | # } 70 | } 71 | -------------------------------------------------------------------------------- /tests/unit/tcl/aof-race.tcl: -------------------------------------------------------------------------------- 1 | set defaults { appendonly {yes} appendfilename {appendonly.aof} } 2 | set server_path [tmpdir server.aof] 3 | set aof_path "$server_path/appendonly.aof" 4 | 5 | proc start_server_aof {overrides code} { 6 | upvar defaults defaults srv srv server_path server_path 7 | set config [concat $defaults $overrides] 8 | start_server [list overrides $config] $code 9 | } 10 | 11 | tags {"aof"} { 12 | # Specific test for a regression where internal buffers were not properly 13 | # cleaned after a child responsible for an AOF rewrite exited. This buffer 14 | # was subsequently appended to the new AOF, resulting in duplicate commands. 15 | start_server_aof [list dir $server_path] { 16 | set client [redis [srv host] [srv port]] 17 | set bench [open "|src/redis-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] 18 | after 100 19 | 20 | # Benchmark should be running by now: start background rewrite 21 | $client bgrewriteaof 22 | 23 | # Read until benchmark pipe reaches EOF 24 | while {[string length [read $bench]] > 0} {} 25 | 26 | # Check contents of foo 27 | assert_equal 20000 [$client get foo] 28 | } 29 | 30 | # Restart server to replay AOF 31 | start_server_aof [list dir $server_path] { 32 | set client [redis [srv host] [srv port]] 33 | assert_equal 20000 [$client get foo] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/unit/tcl/convert-zipmap-hash-on-load.tcl: -------------------------------------------------------------------------------- 1 | # Copy RDB with zipmap encoded hash to server path 2 | set server_path [tmpdir "server.convert-zipmap-hash-on-load"] 3 | 4 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 5 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { 6 | test "RDB load zipmap hash: converts to ziplist" { 7 | r select 0 8 | 9 | assert_match "*ziplist*" [r debug object hash] 10 | assert_equal 2 [r hlen hash] 11 | assert_match {v1 v2} [r hmget hash f1 f2] 12 | } 13 | } 14 | 15 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 16 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-entries" 1]] { 17 | test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-entries is exceeded" { 18 | r select 0 19 | 20 | assert_match "*hashtable*" [r debug object hash] 21 | assert_equal 2 [r hlen hash] 22 | assert_match {v1 v2} [r hmget hash f1 f2] 23 | } 24 | } 25 | 26 | exec cp -f tests/assets/hash-zipmap.rdb $server_path 27 | start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-value" 1]] { 28 | test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-value is exceeded" { 29 | r select 0 30 | 31 | assert_match "*hashtable*" [r debug object hash] 32 | assert_equal 2 [r hlen hash] 33 | assert_match {v1 v2} [r hmget hash f1 f2] 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/unit/type.tcl: -------------------------------------------------------------------------------- 1 | start_server {tags {"type"}} { 2 | 3 | test "type command" { 4 | r flushdb 5 | 6 | r set key1 key1 7 | assert_equal string [r type key1] 8 | 9 | r hset key2 key key2 10 | assert_equal hash [r type key2] 11 | 12 | r lpush key3 key3 13 | assert_equal list [r type key3] 14 | 15 | r zadd key4 100 key4 16 | assert_equal zset [r type key4] 17 | 18 | r sadd key5 key5 19 | assert_equal set [r type key5] 20 | } 21 | } -------------------------------------------------------------------------------- /tests/unit/type/list-2.tcl: -------------------------------------------------------------------------------- 1 | start_server { 2 | tags {"list"} 3 | overrides { 4 | "list-max-ziplist-value" 16 5 | "list-max-ziplist-entries" 256 6 | } 7 | } { 8 | source "tests/unit/type/list-common.tcl" 9 | 10 | foreach {type large} [array get largevalue] { 11 | tags {"slow"} { 12 | test "LTRIM stress testing - $type" { 13 | set mylist {} 14 | set startlen 32 15 | r del mylist 16 | 17 | # Start with the large value to ensure the 18 | # right encoding is used. 19 | r rpush mylist $large 20 | lappend mylist $large 21 | 22 | for {set i 0} {$i < $startlen} {incr i} { 23 | set str [randomInt 9223372036854775807] 24 | r rpush mylist $str 25 | lappend mylist $str 26 | } 27 | 28 | for {set i 0} {$i < 1000} {incr i} { 29 | set min [expr {int(rand()*$startlen)}] 30 | set max [expr {$min+int(rand()*$startlen)}] 31 | set mylist [lrange $mylist $min $max] 32 | r ltrim mylist $min $max 33 | assert_equal $mylist [r lrange mylist 0 -1] 34 | 35 | for {set j [r llen mylist]} {$j < $startlen} {incr j} { 36 | set str [randomInt 9223372036854775807] 37 | r rpush mylist $str 38 | lappend mylist $str 39 | } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/unit/type/list-common.tcl: -------------------------------------------------------------------------------- 1 | # We need a value larger than list-max-ziplist-value to make sure 2 | # the list has the right encoding when it is swapped in again. 3 | array set largevalue {} 4 | set largevalue(ziplist) "hello" 5 | set largevalue(linkedlist) [string repeat "hello" 4] 6 | --------------------------------------------------------------------------------