├── .git-ftp-include ├── .gitattributes ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── Cargo.toml ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── config ├── nextest.toml ├── pd.toml └── tikv.toml ├── doc └── 1.0-roadmap.md ├── examples ├── common │ └── mod.rs ├── pessimistic.rs ├── raw.rs └── transaction.rs ├── getting-started.md ├── proto-build ├── Cargo.toml └── src │ └── main.rs ├── proto ├── autoid.proto ├── brpb.proto ├── cdcpb.proto ├── configpb.proto ├── coprocessor.proto ├── deadlock.proto ├── debugpb.proto ├── diagnosticspb.proto ├── disaggregated.proto ├── disk_usage.proto ├── encryptionpb.proto ├── enginepb.proto ├── errorpb.proto ├── gcpb.proto ├── import_kvpb.proto ├── import_sstpb.proto ├── include │ ├── eraftpb.proto │ ├── gogoproto │ │ └── gogo.proto │ ├── google │ │ ├── api │ │ │ ├── annotations.proto │ │ │ └── http.proto │ │ └── protobuf │ │ │ ├── any.proto │ │ │ ├── api.proto │ │ │ ├── compiler │ │ │ └── plugin.proto │ │ │ ├── descriptor.proto │ │ │ ├── duration.proto │ │ │ ├── empty.proto │ │ │ ├── field_mask.proto │ │ │ ├── source_context.proto │ │ │ ├── struct.proto │ │ │ ├── timestamp.proto │ │ │ ├── type.proto │ │ │ └── wrappers.proto │ └── rustproto.proto ├── keyspacepb.proto ├── kvrpcpb.proto ├── logbackuppb.proto ├── meta_storagepb.proto ├── metapb.proto ├── mpp.proto ├── pdpb.proto ├── raft_cmdpb.proto ├── raft_serverpb.proto ├── recoverdatapb.proto ├── replication_modepb.proto ├── resource_manager.proto ├── resource_usage_agent.proto ├── schedulingpb.proto ├── tikvpb.proto ├── tracepb.proto └── tsopb.proto ├── rust-toolchain.toml ├── rustfmt.toml ├── src ├── backoff.rs ├── common │ ├── errors.rs │ ├── mod.rs │ └── security.rs ├── compat.rs ├── config.rs ├── generated │ ├── autoid.rs │ ├── backup.rs │ ├── cdcpb.rs │ ├── configpb.rs │ ├── coprocessor.rs │ ├── deadlock.rs │ ├── debugpb.rs │ ├── diagnosticspb.rs │ ├── disaggregated.rs │ ├── disk_usage.rs │ ├── encryptionpb.rs │ ├── enginepb.rs │ ├── eraftpb.rs │ ├── errorpb.rs │ ├── gcpb.rs │ ├── google.api.rs │ ├── import_kvpb.rs │ ├── import_sstpb.rs │ ├── keyspacepb.rs │ ├── kvrpcpb.rs │ ├── logbackup.rs │ ├── meta_storagepb.rs │ ├── metapb.rs │ ├── mod.rs │ ├── mpp.rs │ ├── pdpb.rs │ ├── raft_cmdpb.rs │ ├── raft_serverpb.rs │ ├── recover_data.rs │ ├── replication_modepb.rs │ ├── resource_manager.rs │ ├── resource_usage_agent.rs │ ├── schedulingpb.rs │ ├── span.rs │ ├── tikvpb.rs │ ├── tracepb.rs │ └── tsopb.rs ├── kv │ ├── bound_range.rs │ ├── codec.rs │ ├── key.rs │ ├── kvpair.rs │ ├── mod.rs │ └── value.rs ├── lib.rs ├── mock.rs ├── pd │ ├── client.rs │ ├── cluster.rs │ ├── mod.rs │ ├── retry.rs │ └── timestamp.rs ├── proptests │ ├── mod.rs │ └── raw.rs ├── proto.rs ├── raw │ ├── client.rs │ ├── lowering.rs │ ├── mod.rs │ └── requests.rs ├── region.rs ├── region_cache.rs ├── request │ ├── keyspace.rs │ ├── mod.rs │ ├── plan.rs │ ├── plan_builder.rs │ └── shard.rs ├── stats.rs ├── store │ ├── client.rs │ ├── errors.rs │ ├── mod.rs │ └── request.rs ├── timestamp.rs ├── transaction │ ├── buffer.rs │ ├── client.rs │ ├── lock.rs │ ├── lowering.rs │ ├── mod.rs │ ├── requests.rs │ ├── snapshot.rs │ └── transaction.rs └── util │ ├── iter.rs │ └── mod.rs ├── taplo.toml └── tests ├── common ├── ctl.rs └── mod.rs ├── failpoint_tests.rs └── integration_tests.rs /.git-ftp-include: -------------------------------------------------------------------------------- 1 | !target/doc/ 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | src/generated/ diff=false 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | push: 4 | branches: 5 | - master 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 8 | cancel-in-progress: true 9 | 10 | env: 11 | CARGO_TERM_COLOR: always 12 | 13 | name: CI 14 | 15 | jobs: 16 | check: 17 | name: check 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Install Protoc 22 | uses: arduino/setup-protoc@v1 23 | with: 24 | version: '3.x' 25 | repo-token: ${{ secrets.GITHUB_TOKEN }} 26 | - name: Rust Cache 27 | uses: Swatinem/rust-cache@v2 28 | - name: make check 29 | run: make check 30 | - name: Catch unexpected changes in the generated code 31 | run: | 32 | git diff --exit-code 33 | 34 | unit-test: 35 | name: unit test 36 | env: 37 | CARGO_INCREMENTAL: 0 38 | NEXTEST_PROFILE: ci 39 | runs-on: ubuntu-latest 40 | steps: 41 | - uses: actions/checkout@v4 42 | - name: Install Protoc 43 | uses: arduino/setup-protoc@v1 44 | with: 45 | version: '3.x' 46 | repo-token: ${{ secrets.GITHUB_TOKEN }} 47 | - name: Rust Cache 48 | uses: Swatinem/rust-cache@v2 49 | - name: Install latest nextest release 50 | uses: taiki-e/install-action@nextest 51 | - name: unit test 52 | run: make unit-test 53 | 54 | integration-test: 55 | name: integration test 56 | strategy: 57 | fail-fast: false 58 | matrix: 59 | case: ["integration-test-txn", "integration-test-raw"] 60 | env: 61 | CARGO_INCREMENTAL: 0 62 | NEXTEST_PROFILE: ci 63 | TIKV_VERSION: v8.5.1 64 | RUST_LOG: info 65 | runs-on: ubuntu-latest 66 | steps: 67 | - uses: actions/checkout@v4 68 | - name: Install Protoc 69 | uses: arduino/setup-protoc@v1 70 | with: 71 | version: '3.x' 72 | repo-token: ${{ secrets.GITHUB_TOKEN }} 73 | - name: Rust Cache 74 | uses: Swatinem/rust-cache@v2 75 | - name: install tiup 76 | run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh 77 | - name: start tiup playground 78 | run: | 79 | # use latest stable version 80 | ~/.tiup/bin/tiup install tikv:${{ env.TIKV_VERSION }} pd:${{ env.TIKV_VERSION }} 81 | ~/.tiup/bin/tiup playground ${{ env.TIKV_VERSION }} --mode tikv-slim --kv 3 --tag cluster --without-monitor --kv.config config/tikv.toml --pd.config config/pd.toml & 82 | while :; do 83 | echo "waiting cluster to be ready" 84 | [[ "$(curl -I http://127.0.0.1:2379/pd/api/v1/regions 2>/dev/null | head -n 1 | cut -d$' ' -f2)" -ne "405" ]] || break 85 | sleep 1 86 | done 87 | - name: Install latest nextest release 88 | uses: taiki-e/install-action@nextest 89 | - name: Integration test 90 | run: MULTI_REGION=1 make ${{ matrix.case }} 91 | - name: Upload logs 92 | if: failure() 93 | uses: actions/upload-artifact@v4 94 | with: 95 | name: cluster-logs 96 | path: | 97 | ~/.tiup/data/cluster/tikv*/*.log 98 | ~/.tiup/data/cluster/pd*/*.log 99 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # OSX leaves these everywhere on SMB shares 3 | ._* 4 | 5 | # OSX trash 6 | .DS_Store 7 | 8 | # Eclipse files 9 | .classpath 10 | .project 11 | .settings/** 12 | 13 | # Vim swap files 14 | *.swp 15 | 16 | # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA 17 | .idea/ 18 | *.iml 19 | out/ 20 | 21 | # Vscode files 22 | .vscode/** 23 | 24 | target 25 | tmp 26 | /bin 27 | 28 | Cargo.lock 29 | *.rs.bk 30 | *.rs.fmt 31 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tikv-client" 3 | version = "0.3.0" 4 | keywords = ["TiKV", "KV", "distributed-systems"] 5 | license = "Apache-2.0" 6 | authors = ["The TiKV Project Authors"] 7 | repository = "https://github.com/tikv/client-rust" 8 | description = "The Rust language implementation of TiKV client." 9 | edition = "2021" 10 | 11 | [features] 12 | default = ["prometheus"] 13 | prometheus = ["prometheus/push", "prometheus/process"] 14 | # Enable integration tests with a running TiKV and PD instance. 15 | # Use $PD_ADDRS, comma separated, to set the addresses the tests use. 16 | integration-tests = [] 17 | 18 | [lib] 19 | name = "tikv_client" 20 | 21 | [workspace] 22 | members = [ 23 | ".", 24 | "proto-build", 25 | ] 26 | 27 | [dependencies] 28 | async-recursion = "0.3" 29 | async-trait = "0.1" 30 | derive-new = "0.5" 31 | either = "1.6" 32 | fail = "0.4" 33 | futures = { version = "0.3" } 34 | lazy_static = "1" 35 | log = "0.4" 36 | pin-project = "1" 37 | prometheus = { version = "0.13", default-features = false } 38 | prost = "0.12" 39 | rand = "0.8" 40 | regex = "1" 41 | semver = "1.0" 42 | serde = "1.0" 43 | serde_derive = "1.0" 44 | serde_json = "1" 45 | take_mut = "0.2.2" 46 | thiserror = "1" 47 | tokio = { version = "1", features = ["sync", "rt-multi-thread", "macros"] } 48 | tonic = { version = "0.10", features = ["tls"] } 49 | 50 | [dev-dependencies] 51 | clap = "2" 52 | env_logger = "0.10" 53 | fail = { version = "0.4", features = ["failpoints"] } 54 | proptest = "1" 55 | proptest-derive = "0.5.1" 56 | reqwest = { version = "0.11", features = ["json", "native-tls-vendored"] } 57 | rstest = "0.18.2" 58 | serde_json = "1" 59 | serial_test = "0.5.0" 60 | tempfile = "3.6" 61 | tokio = { version = "1", features = ["sync", "rt-multi-thread", "macros"] } 62 | 63 | [[test]] 64 | name = "failpoint_tests" 65 | path = "tests/failpoint_tests.rs" 66 | required-features = ["fail/failpoints"] 67 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export RUSTFLAGS=-Dwarnings 2 | 3 | .PHONY: default check unit-test generate integration-tests integration-tests-txn integration-tests-raw test doc docker-pd docker-kv docker all 4 | 5 | export PD_ADDRS ?= 127.0.0.1:2379 6 | export MULTI_REGION ?= 1 7 | 8 | ALL_FEATURES := integration-tests 9 | 10 | NEXTEST_ARGS := --config-file $(shell pwd)/config/nextest.toml 11 | 12 | INTEGRATION_TEST_ARGS := --features "integration-tests" --test-threads 1 13 | 14 | RUN_INTEGRATION_TEST := cargo nextest run ${NEXTEST_ARGS} --all ${INTEGRATION_TEST_ARGS} 15 | 16 | default: check 17 | 18 | generate: 19 | cargo run -p tikv-client-proto-build 20 | 21 | check: generate 22 | cargo check --all --all-targets --features "${ALL_FEATURES}" 23 | cargo fmt -- --check 24 | cargo clippy --all-targets --features "${ALL_FEATURES}" -- -D clippy::all 25 | 26 | unit-test: generate 27 | cargo nextest run ${NEXTEST_ARGS} --all --no-default-features 28 | 29 | integration-test: integration-test-txn integration-test-raw 30 | 31 | integration-test-txn: generate 32 | $(RUN_INTEGRATION_TEST) txn_ 33 | 34 | integration-test-raw: generate 35 | $(RUN_INTEGRATION_TEST) raw_ 36 | 37 | test: unit-test integration-test 38 | 39 | doc: 40 | cargo doc --workspace --exclude tikv-client-proto --document-private-items --no-deps 41 | 42 | tiup: 43 | tiup playground nightly --mode tikv-slim --kv 3 --without-monitor --kv.config $(shell pwd)/config/tikv.toml --pd.config $(shell pwd)/config/pd.toml & 44 | 45 | all: generate check doc test 46 | 47 | clean: 48 | cargo clean 49 | rm -rf target 50 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | approvers: 3 | - 5kbpers 4 | - AndreMouche 5 | - andylokandy 6 | - breezewish 7 | - brson 8 | - bufferflies 9 | - BusyJay 10 | - cfzjywxk 11 | - Connor1996 12 | - coocood 13 | - crazycs520 14 | - disksing 15 | - ekexium 16 | - gengliqi 17 | - glorv 18 | - hicqu 19 | - hunterlxt 20 | - imtbkcat 21 | - innerr 22 | - iosmanthus 23 | - jackysp 24 | - kennytm 25 | - Little-Wallace 26 | - liuzix 27 | - lonng 28 | - LykxSassinator 29 | - lysu 30 | - marsishandsome 31 | - MyonKeminta 32 | - niedhui 33 | - NingLin-P 34 | - nrc 35 | - overvenus 36 | - pingyu 37 | - skyzh 38 | - SpadeA-Tang 39 | - sticnarf 40 | - sunxiaoguang 41 | - tabokie 42 | - TennyZhuang 43 | - tonyxuqqi 44 | - v01dstar 45 | - yiwu-arbug 46 | - you06 47 | - youjiali1995 48 | - YuJuncen 49 | - zhangjinpeng87 50 | - zhongzc 51 | - zhouqiang-cl 52 | - zyguan 53 | reviewers: 54 | - 3AceShowHand 55 | - 3pointer 56 | - CalvinNeo 57 | - ethercflow 58 | - fredchenbj 59 | - Fullstop000 60 | - gozssky 61 | - haojinming 62 | - hbisheng 63 | - hhwyt 64 | - HuSharp 65 | - jayzhan211 66 | - Jibbow 67 | - JmPotato 68 | - Leavrth 69 | - lhy1024 70 | - longfangsong 71 | - lzmhhh123 72 | - Mossaka 73 | - MrCroxx 74 | - nolouch 75 | - rleungx 76 | - Rustin170506 77 | - tier-cap 78 | - wjhuang2016 79 | - wshwsh12 80 | - Xuanwo 81 | -------------------------------------------------------------------------------- /config/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.ci] 2 | retries = 0 3 | fail-fast = false 4 | slow-timeout = { period = "60s", terminate-after = 10 } # Timeout 10m. TODO: speed up the slow tests. 5 | failure-output = "final" 6 | 7 | [profile.ci.junit] 8 | path = "junit.xml" 9 | -------------------------------------------------------------------------------- /config/pd.toml: -------------------------------------------------------------------------------- 1 | [schedule] 2 | max-merge-region-size = 1 3 | max-merge-region-keys = 3 4 | -------------------------------------------------------------------------------- /config/tikv.toml: -------------------------------------------------------------------------------- 1 | [coprocessor] 2 | region-max-keys = 10 3 | region-split-keys = 7 4 | batch-split-limit = 100 5 | 6 | [raftstore] 7 | region-split-check-diff = "1B" 8 | pd-heartbeat-tick-interval = "2s" 9 | pd-store-heartbeat-tick-interval = "5s" 10 | split-region-check-tick-interval = "1s" 11 | raft-entry-max-size = "1MB" 12 | 13 | [rocksdb] 14 | max-open-files = 10000 15 | 16 | [raftdb] 17 | max-open-files = 10000 18 | 19 | [storage] 20 | api-version = 2 21 | enable-ttl = true 22 | reserve-space = "0MiB" 23 | -------------------------------------------------------------------------------- /doc/1.0-roadmap.md: -------------------------------------------------------------------------------- 1 | # client-rust 1.0 roadmap 2 | 3 | The client-rust project complements the TiKV ecosystem by providing a reliable Rust client. It can be a foundation for implementing clients in other languages. There have been preliminary works in [client-py](https://github.com/tikv/client-py) and [client-cpp](https://github.com/tikv/client-cpp). There are also plans for general-purpose client-java and [client-node](https://github.com/tikv/tikv/issues/10054). 4 | 5 | The document describes our plan to push client-rust towards its general availability (1.0). 6 | 7 | ## Deliverable 8 | 9 | client-rust 1.0 that 10 | - enables new users to start using it in 10 mins 11 | - has been heavily tested under simulated and real workload 12 | - is easy to use, debug and seek help from documentation 13 | - is performant 14 | - supports the most-wanted features: GC, async commit, etc. 15 | 16 | 17 | ## Milestones 18 | 19 | There are several milestones when we make important progress. We may release new versions with these changes, but it depends. The version numbers are tentative and subject to change. 20 | 21 | #### 0.1 - initial release 22 | 23 | The initial release that is functional but may be weak in performance and user-friendliness. It supports: 24 | raw mode, including CAS 25 | transactional mode, including optimistic and pessimistic 26 | 27 | #### 0.2 - feature complete 28 | 29 | 0.2 should be feature complete 30 | - Region cache 31 | - Large transaction 32 | - Async commit 33 | - (Optional) GC, depending on the progress of the new mechanism 34 | 35 | #### 0.3 - user-friendly and high-quality 36 | 37 | 0.3 should be a relatively mature version. It is easy to use and have fewer bugs. 38 | 39 | The tasks include better logging, error handling, compatibility with the Rust async ecosystem, better unit test and integration test. 40 | 41 | #### 1.0 - production ready 42 | 43 | The gap between 0.3 and 1.0 may include 44 | - Documentation 45 | - Test coverage 46 | - Performance test / benchmark 47 | - Usability improvement based on user feedback 48 | 49 | ## Estimated Timeline 50 | 51 | The timeline is a rough estimate. It is subject to change if we find new issues, underestimate the workload, or have insufficient contributors’ time. 52 | 53 | ##### May 2021 54 | Release 0.1.0 55 | 56 | ##### Aug 2021 57 | Release 0.2.0 58 | 59 | ##### Oct 2021 60 | Release 0.3.0 61 | 62 | ##### Jan 2022 63 | Release 1.0 64 | 65 | We will take a relative longer time to let users try the tikv-client, and improve based on their feedback. 66 | 67 | 68 | 69 | ## Risks 70 | 71 | - Inaccurate estimated workload. Considering the complexity of the client in TiDB, we may underestimate the workload. 72 | - Lacking staff. Regular contributors include Ziqian Qin and Andy Lok. 73 | - Too much or too little external contribution - too much means core contributors don’t have time to focus on code and design, or that we add too many new features which do not get us to 1.0. Too little means we can’t iterate effectively based on user experience. 74 | 75 | 76 | ---- 77 | 78 | ## Tasks 79 | 80 | Tasks listed here are collected from https://github.com/tikv/client-rust/discussions/272. 81 | 82 | **Note**: They are preliminary, and we should adapt them according to user feedback. 83 | 84 | 85 | ### 0.2 86 | 87 | - https://github.com/tikv/client-rust/issues/299 Add region cache 88 | - https://github.com/tikv/client-rust/issues/209 Use CheckTxnStatus rather than Cleanup 89 | - https://github.com/tikv/client-rust/issues/239 Better experience when "dropping active transaction" 90 | - https://github.com/tikv/client-rust/issues/287 Support 1PC 91 | - https://github.com/tikv/client-rust/issues/189 Support large transaction 92 | - https://github.com/tikv/client-rust/issues/287 Support async commit 93 | 94 | ### 0.3 95 | 96 | 97 | - https://github.com/tikv/client-rust/issues/267 Better logging 98 | - https://github.com/tikv/client-rust/issues/284 Test with nemesis and fault injection 99 | - https://github.com/tikv/client-rust/issues/246 Compatibility with any async reactor (don't bind to tokio) 100 | - https://github.com/tikv/client-rust/issues/285 Improve unit test coverage 101 | - https://github.com/tikv/client-rust/issues/286 TiKV version compatibility check 102 | - https://github.com/tikv/client-rust/issues/284 Test with various workload: simulated / real / extreme 103 | 104 | ### 1.0 105 | 106 | 107 | - https://github.com/tikv/client-rust/issues/288 Benchmarking script/tool 108 | - https://github.com/tikv/client-rust/issues/283 Parallelize multi-region request 109 | - https://github.com/tikv/client-rust/issues/289 Synchronous API 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /examples/common/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::path::PathBuf; 4 | 5 | use clap::crate_version; 6 | use clap::App; 7 | use clap::Arg; 8 | 9 | pub struct CommandArgs { 10 | pub pd: Vec, 11 | pub ca: Option, 12 | pub cert: Option, 13 | pub key: Option, 14 | } 15 | 16 | pub fn parse_args(app_name: &str) -> CommandArgs { 17 | let matches = App::new(app_name) 18 | .version(crate_version!()) 19 | .author("The TiKV Project Authors") 20 | .arg( 21 | Arg::with_name("pd") 22 | .long("pd") 23 | .aliases(&["pd-endpoint", "pd-endpoints"]) 24 | .value_name("PD_URL") 25 | .help("Sets PD endpoints") 26 | .long_help("Sets PD endpoints. Uses `,` to separate multiple PDs") 27 | .takes_value(true) 28 | .multiple(true) 29 | .value_delimiter(",") 30 | .required(false) 31 | .default_value("localhost:2379"), 32 | ) 33 | // A cyclic dependency between CA, cert and key is made 34 | // to ensure that no security options are missing. 35 | .arg( 36 | Arg::with_name("ca") 37 | .long("ca") 38 | .value_name("CA_PATH") 39 | .help("Sets the CA") 40 | .long_help("Sets the CA. Must be used with --cert and --key") 41 | .takes_value(true) 42 | .requires("cert"), 43 | ) 44 | .arg( 45 | Arg::with_name("cert") 46 | .long("cert") 47 | .value_name("CERT_PATH") 48 | .help("Sets the certificate") 49 | .long_help("Sets the certificate. Must be used with --ca and --key") 50 | .takes_value(true) 51 | .requires("key"), 52 | ) 53 | .arg( 54 | Arg::with_name("key") 55 | .long("key") 56 | .alias("private-key") 57 | .value_name("KEY_PATH") 58 | .help("Sets the private key") 59 | .long_help("Sets the private key. Must be used with --ca and --cert") 60 | .takes_value(true) 61 | .requires("ca"), 62 | ) 63 | .get_matches(); 64 | 65 | CommandArgs { 66 | pd: matches.values_of("pd").unwrap().map(String::from).collect(), 67 | ca: matches.value_of("ca").map(PathBuf::from), 68 | cert: matches.value_of("cert").map(PathBuf::from), 69 | key: matches.value_of("key").map(PathBuf::from), 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /examples/pessimistic.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | mod common; 4 | 5 | use tikv_client::Config; 6 | use tikv_client::Key; 7 | use tikv_client::TransactionClient as Client; 8 | use tikv_client::TransactionOptions; 9 | use tikv_client::Value; 10 | 11 | use crate::common::parse_args; 12 | 13 | #[tokio::main] 14 | async fn main() { 15 | env_logger::init(); 16 | 17 | // You can try running this example by passing your pd endpoints 18 | // (and SSL options if necessary) through command line arguments. 19 | let args = parse_args("txn"); 20 | 21 | // Create a configuration to use for the example. 22 | // Optionally encrypt the traffic. 23 | let config = if let (Some(ca), Some(cert), Some(key)) = (args.ca, args.cert, args.key) { 24 | Config::default().with_security(ca, cert, key) 25 | } else { 26 | Config::default() 27 | } 28 | // This example uses the default keyspace, so api-v2 must be enabled on the server. 29 | .with_default_keyspace(); 30 | 31 | // init 32 | let client = Client::new_with_config(args.pd, config) 33 | .await 34 | .expect("Could not connect to tikv"); 35 | 36 | let key1: Key = b"key01".to_vec().into(); 37 | let value1: Value = b"value1".to_vec(); 38 | let key2: Key = b"key02".to_vec().into(); 39 | let value2: Value = b"value2".to_vec(); 40 | let mut txn0 = client 41 | .begin_optimistic() 42 | .await 43 | .expect("Could not begin a transaction"); 44 | for (key, value) in [(key1.clone(), value1), (key2, value2)] { 45 | txn0.put(key, value).await.expect("Could not set key value"); 46 | } 47 | txn0.commit().await.expect("Could not commit"); 48 | drop(txn0); 49 | let mut txn1 = client 50 | .begin_pessimistic() 51 | .await 52 | .expect("Could not begin a transaction"); 53 | // lock the key 54 | let value = txn1 55 | .get_for_update(key1.clone()) 56 | .await 57 | .expect("Could not get_and_lock the key"); 58 | println!("{:?}", (&key1, value)); 59 | { 60 | // another txn cannot write to the locked key 61 | let mut txn2 = client 62 | .begin_with_options(TransactionOptions::new_optimistic().no_resolve_locks()) 63 | .await 64 | .expect("Could not begin a transaction"); 65 | let value2: Value = b"value2".to_vec(); 66 | txn2.put(key1.clone(), value2).await.unwrap(); 67 | let result = txn2.commit().await; 68 | assert!(result.is_err()); 69 | } 70 | // while this txn can still write it 71 | let value3: Value = b"value3".to_vec(); 72 | txn1.put(key1.clone(), value3).await.unwrap(); 73 | txn1.commit().await.unwrap(); 74 | let mut txn3 = client 75 | .begin_optimistic() 76 | .await 77 | .expect("Could not begin a transaction"); 78 | let result = txn3.get(key1.clone()).await.unwrap().unwrap(); 79 | txn3.commit() 80 | .await 81 | .expect("Committing read-only transaction should not fail"); 82 | println!("{:?}", (key1, result)); 83 | } 84 | -------------------------------------------------------------------------------- /examples/raw.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | #![type_length_limit = "8165158"] 4 | 5 | mod common; 6 | 7 | use tikv_client::Config; 8 | use tikv_client::IntoOwnedRange; 9 | use tikv_client::Key; 10 | use tikv_client::KvPair; 11 | use tikv_client::RawClient as Client; 12 | use tikv_client::Result; 13 | use tikv_client::Value; 14 | 15 | use crate::common::parse_args; 16 | 17 | const KEY: &str = "TiKV"; 18 | const VALUE: &str = "Rust"; 19 | 20 | #[tokio::main] 21 | async fn main() -> Result<()> { 22 | env_logger::init(); 23 | 24 | // You can try running this example by passing your pd endpoints 25 | // (and SSL options if necessary) through command line arguments. 26 | let args = parse_args("raw"); 27 | 28 | // Create a configuration to use for the example. 29 | // Optionally encrypt the traffic. 30 | let config = if let (Some(ca), Some(cert), Some(key)) = (args.ca, args.cert, args.key) { 31 | Config::default().with_security(ca, cert, key) 32 | } else { 33 | Config::default() 34 | } 35 | // This example uses the default keyspace, so api-v2 must be enabled on the server. 36 | .with_default_keyspace(); 37 | 38 | // When we first create a client we receive a `Connect` structure which must be resolved before 39 | // the client is actually connected and usable. 40 | let client = Client::new_with_config(args.pd, config).await?; 41 | 42 | // Requests are created from the connected client. These calls return structures which 43 | // implement `Future`. This means the `Future` must be resolved before the action ever takes 44 | // place. 45 | // 46 | // Here we set the key `TiKV` to have the value `Rust` associated with it. 47 | client.put(KEY.to_owned(), VALUE.to_owned()).await.unwrap(); // Returns a `tikv_client::Error` on failure. 48 | println!("Put key {KEY:?}, value {VALUE:?}."); 49 | 50 | // Unlike a standard Rust HashMap all calls take owned values. This is because under the hood 51 | // protobufs must take ownership of the data. If we only took a borrow we'd need to internally 52 | // clone it. This is against Rust API guidelines, so you must manage this yourself. 53 | // 54 | // Above, you saw we can use a `&'static str`, this is primarily for making examples short. 55 | // This type is practical to use for real things, and usage forces an internal copy. 56 | // 57 | // It is best to pass a `Vec` in terms of explicitness and speed. `String`s and a few other 58 | // types are supported as well, but it all ends up as `Vec` in the end. 59 | let value: Option = client.get(KEY.to_owned()).await?; 60 | assert_eq!(value, Some(Value::from(VALUE.to_owned()))); 61 | println!("Get key `{KEY}` returned value {value:?}."); 62 | 63 | // You can also set the `ColumnFamily` used by the request. 64 | // This is *advanced usage* and should have some special considerations. 65 | client 66 | .delete(KEY.to_owned()) 67 | .await 68 | .expect("Could not delete value"); 69 | println!("Key: `{KEY}` deleted"); 70 | 71 | // Here we check if the key has been deleted from the key-value store. 72 | let value: Option = client 73 | .get(KEY.to_owned()) 74 | .await 75 | .expect("Could not get just deleted entry"); 76 | assert!(value.is_none()); 77 | 78 | // You can ask to write multiple key-values at the same time, it is much more 79 | // performant because it is passed in one request to the key-value store. 80 | let pairs = vec![ 81 | KvPair::from(("k1".to_owned(), "v1".to_owned())), 82 | KvPair::from(("k2".to_owned(), "v2".to_owned())), 83 | KvPair::from(("k3".to_owned(), "v3".to_owned())), 84 | ]; 85 | client.batch_put(pairs).await.expect("Could not put pairs"); 86 | 87 | // Same thing when you want to retrieve multiple values. 88 | let keys = vec![Key::from("k1".to_owned()), Key::from("k2".to_owned())]; 89 | let values = client 90 | .batch_get(keys.clone()) 91 | .await 92 | .expect("Could not get values"); 93 | println!("Found values: {values:?} for keys: {keys:?}"); 94 | 95 | // Scanning a range of keys is also possible giving it two bounds 96 | // it will returns all entries between these two. 97 | let start = "k1"; 98 | let end = "k2"; 99 | let pairs = client 100 | .scan((start..=end).into_owned(), 10) 101 | .await 102 | .expect("Could not scan"); 103 | 104 | let keys: Vec<_> = pairs.into_iter().map(|p| p.key().clone()).collect(); 105 | assert_eq!( 106 | &keys, 107 | &[Key::from("k1".to_owned()), Key::from("k2".to_owned()),] 108 | ); 109 | println!("Scanning from {start:?} to {end:?} gives: {keys:?}"); 110 | 111 | let k1 = "k1"; 112 | let k2 = "k2"; 113 | let k3 = "k3"; 114 | let batch_scan_keys = vec![ 115 | (k1.to_owned()..=k2.to_owned()), 116 | (k2.to_owned()..=k3.to_owned()), 117 | (k1.to_owned()..=k3.to_owned()), 118 | ]; 119 | let kv_pairs = client 120 | .batch_scan(batch_scan_keys.to_owned(), 10) 121 | .await 122 | .expect("Could not batch scan"); 123 | let vals: Vec<_> = kv_pairs 124 | .into_iter() 125 | .map(|p| String::from_utf8(p.1).unwrap()) 126 | .collect(); 127 | assert_eq!( 128 | &vals, 129 | &[ 130 | "v1".to_owned(), 131 | "v2".to_owned(), 132 | "v2".to_owned(), 133 | "v3".to_owned(), 134 | "v1".to_owned(), 135 | "v2".to_owned(), 136 | "v3".to_owned() 137 | ] 138 | ); 139 | println!("Scanning batch scan from {batch_scan_keys:?} gives: {vals:?}"); 140 | 141 | // Delete all keys in the whole range. 142 | client.delete_range("".to_owned().."".to_owned()).await?; 143 | 144 | Ok(()) 145 | } 146 | -------------------------------------------------------------------------------- /examples/transaction.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | mod common; 4 | 5 | use tikv_client::BoundRange; 6 | use tikv_client::Config; 7 | use tikv_client::Key; 8 | use tikv_client::KvPair; 9 | use tikv_client::TransactionClient as Client; 10 | use tikv_client::Value; 11 | 12 | use crate::common::parse_args; 13 | 14 | async fn puts(client: &Client, pairs: impl IntoIterator>) { 15 | let mut txn = client 16 | .begin_optimistic() 17 | .await 18 | .expect("Could not begin a transaction"); 19 | for pair in pairs { 20 | let (key, value) = pair.into().into(); 21 | txn.put(key, value).await.expect("Could not set key value"); 22 | } 23 | txn.commit().await.expect("Could not commit transaction"); 24 | } 25 | 26 | async fn get(client: &Client, key: Key) -> Option { 27 | let mut txn = client 28 | .begin_optimistic() 29 | .await 30 | .expect("Could not begin a transaction"); 31 | let res = txn.get(key).await.expect("Could not get value"); 32 | txn.commit() 33 | .await 34 | .expect("Committing read-only transaction should not fail"); 35 | res 36 | } 37 | 38 | async fn key_exists(client: &Client, key: Key) -> bool { 39 | let mut txn = client 40 | .begin_optimistic() 41 | .await 42 | .expect("Could not begin a transaction"); 43 | let res = txn 44 | .key_exists(key) 45 | .await 46 | .expect("Could not check key exists"); 47 | txn.commit() 48 | .await 49 | .expect("Committing read-only transaction should not fail"); 50 | res 51 | } 52 | 53 | async fn scan(client: &Client, range: impl Into, limit: u32) { 54 | let mut txn = client 55 | .begin_optimistic() 56 | .await 57 | .expect("Could not begin a transaction"); 58 | txn.scan(range, limit) 59 | .await 60 | .expect("Could not scan key-value pairs in range") 61 | .for_each(|pair| println!("{pair:?}")); 62 | txn.commit().await.expect("Could not commit transaction"); 63 | } 64 | 65 | async fn dels(client: &Client, keys: impl IntoIterator) { 66 | let mut txn = client 67 | .begin_optimistic() 68 | .await 69 | .expect("Could not begin a transaction"); 70 | for key in keys { 71 | txn.delete(key).await.expect("Could not delete the key"); 72 | } 73 | txn.commit().await.expect("Could not commit transaction"); 74 | } 75 | 76 | #[tokio::main] 77 | async fn main() { 78 | env_logger::init(); 79 | 80 | // You can try running this example by passing your pd endpoints 81 | // (and SSL options if necessary) through command line arguments. 82 | let args = parse_args("txn"); 83 | 84 | // Create a configuration to use for the example. 85 | // Optionally encrypt the traffic. 86 | let config = if let (Some(ca), Some(cert), Some(key)) = (args.ca, args.cert, args.key) { 87 | Config::default().with_security(ca, cert, key) 88 | } else { 89 | Config::default() 90 | } 91 | // This example uses the default keyspace, so api-v2 must be enabled on the server. 92 | .with_default_keyspace(); 93 | 94 | let txn = Client::new_with_config(args.pd, config) 95 | .await 96 | .expect("Could not connect to tikv"); 97 | 98 | // set 99 | let key1: Key = b"key1".to_vec().into(); 100 | let value1: Value = b"value1".to_vec(); 101 | let key2: Key = b"key2".to_vec().into(); 102 | let value2: Value = b"value2".to_vec(); 103 | puts(&txn, vec![(key1, value1), (key2, value2)]).await; 104 | 105 | // get 106 | let key1: Key = b"key1".to_vec().into(); 107 | let value1 = get(&txn, key1.clone()).await; 108 | println!("{:?}", (key1, value1)); 109 | 110 | // check key exists 111 | let key1: Key = b"key1".to_vec().into(); 112 | let key1_exists = key_exists(&txn, key1.clone()).await; 113 | let key2: Key = b"key_not_exist".to_vec().into(); 114 | let key2_exists = key_exists(&txn, key2.clone()).await; 115 | println!( 116 | "check exists {:?}", 117 | vec![(key1, key1_exists), (key2, key2_exists)] 118 | ); 119 | 120 | // scan 121 | let key1: Key = b"key1".to_vec().into(); 122 | scan(&txn, key1.., 10).await; 123 | 124 | // delete 125 | let key1: Key = b"key1".to_vec().into(); 126 | let key2: Key = b"key2".to_vec().into(); 127 | dels(&txn, vec![key1, key2]).await; 128 | } 129 | -------------------------------------------------------------------------------- /getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | The TiKV client is a Rust library (crate). To use this crate in your project, add the following dependencies to your `Cargo.toml`: 4 | 5 | ```toml 6 | [dependencies] 7 | tikv-client = "0.1" 8 | tokio = { version = "1.5", features = ["full"] } 9 | ``` 10 | 11 | Note that you need to use Tokio. The TiKV client has an async API and therefore you need an async runtime in your program to use it. At the moment, Tokio is used internally in the client and so you must use Tokio in your code too. We plan to become more flexible in future versions. 12 | 13 | The general flow of using the client crate is to create either a raw or transaction client object (which can be configured) then send commands using the client object, or use it to create transactions objects. In the latter case, the transaction is built up using various commands and then committed (or rolled back). 14 | 15 | ## Examples 16 | 17 | To use the client in your program, use code like the following. 18 | 19 | Raw mode: 20 | 21 | ```rust 22 | use tikv_client::RawClient; 23 | 24 | let client = RawClient::new(vec!["127.0.0.1:2379"], None).await?; 25 | client.put("key".to_owned(), "value".to_owned()).await?; 26 | let value = client.get("key".to_owned()).await?; 27 | ``` 28 | 29 | Transactional mode: 30 | 31 | ```rust 32 | use tikv_client::TransactionClient; 33 | 34 | let txn_client = TransactionClient::new(vec!["127.0.0.1:2379"], None).await?; 35 | let mut txn = txn_client.begin_optimistic().await?; 36 | txn.put("key".to_owned(), "value".to_owned()).await?; 37 | let value = txn.get("key".to_owned()).await?; 38 | txn.commit().await?; 39 | ``` 40 | 41 | To make an example which builds and runs, 42 | 43 | ```rust 44 | use tikv_client::{TransactionClient, Error}; 45 | 46 | async fn run() -> Result<(), Error> { 47 | let txn_client = TransactionClient::new(vec!["127.0.0.1:2379"], None).await?; 48 | let mut txn = txn_client.begin_optimistic().await?; 49 | txn.put("key".to_owned(), "value".to_owned()).await?; 50 | let value = txn.get("key".to_owned()).await?; 51 | println!("value: {:?}", value); 52 | txn.commit().await?; 53 | Ok(()) 54 | } 55 | 56 | #[tokio::main] 57 | async fn main() { 58 | run().await.unwrap(); 59 | } 60 | ``` 61 | 62 | For more examples, see the [examples](examples) directory. 63 | 64 | ## A TiKV cluster 65 | 66 | To use the client, you'll need a TiKV instance to communicate with. In production, this should be a cluster of dedicated servers which are accessed via the network. To get started, you can run a TiKV 'cluster' on your local machine. 67 | 68 | A TiKV cluster consists of TiKV nodes and PD nodes. For normal use, you need at least three of each; there is no maximum. For testing etc., you need at least one TiKV node 69 | and one PD node. For more details, see the [TiKV docs](https://tikv.org/docs/dev/concepts/architecture/). 70 | 71 | The easiest way to manage a TiKV cluster (locally or on multiple machines) is to use [TiUP](https://github.com/pingcap/tiup). To install it on your computer, use 72 | 73 | ``` 74 | curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh 75 | ``` 76 | 77 | then, to start a local TiKV 'cluster' for testing, 78 | 79 | ``` 80 | tiup playground nightly --mode tikv-slim 81 | ``` 82 | 83 | For more information about TiUP, see their [docs](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup). 84 | 85 | You can also build and/or run TiKV and PD instances manually. See [this blog post](https://ncameron.org/blog/building-running-and-benchmarking-tikv-and-tidb/) for details, note that you don't need the TiDB nodes. 86 | -------------------------------------------------------------------------------- /proto-build/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tikv-client-proto-build" 3 | version = "0.0.0" 4 | publish = false 5 | 6 | keywords = ["TiKV", "KV", "distributed-systems"] 7 | license = "Apache-2.0" 8 | authors = ["The TiKV Project Authors"] 9 | repository = "https://github.com/tikv/client-rust" 10 | description = "The Rust language implementation of TiKV client." 11 | edition = "2021" 12 | 13 | [dependencies] 14 | glob = "0.3" 15 | tonic-build = { version = "0.10", features = ["cleanup-markdown"] } 16 | -------------------------------------------------------------------------------- /proto-build/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | fn main() { 4 | tonic_build::configure() 5 | .emit_rerun_if_changed(false) 6 | .build_server(false) 7 | .include_file("mod.rs") 8 | .out_dir("src/generated") 9 | .compile( 10 | &glob::glob("proto/*.proto") 11 | .unwrap() 12 | .collect::, _>>() 13 | .unwrap(), 14 | &["proto/include", "proto"], 15 | ) 16 | .unwrap(); 17 | } 18 | -------------------------------------------------------------------------------- /proto/autoid.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package autoid; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | 7 | option (gogoproto.sizer_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option (rustproto.lite_runtime_all) = true; 11 | 12 | option java_package = "org.tikv.kvproto"; 13 | 14 | message AutoIDRequest { 15 | int64 dbID = 1; 16 | int64 tblID = 2; 17 | bool isUnsigned = 3; 18 | uint64 n = 4; 19 | int64 increment = 5; 20 | int64 offset = 6; 21 | uint32 keyspaceID = 7; 22 | } 23 | 24 | message AutoIDResponse { 25 | int64 min = 1; 26 | int64 max = 2; 27 | 28 | bytes errmsg = 3; 29 | } 30 | 31 | message RebaseRequest { 32 | int64 dbID = 1; 33 | int64 tblID = 2; 34 | bool isUnsigned = 3; 35 | int64 base = 4; 36 | bool force = 5; 37 | } 38 | 39 | message RebaseResponse { 40 | bytes errmsg = 1; 41 | } 42 | 43 | service AutoIDAlloc { 44 | rpc AllocAutoID(AutoIDRequest) returns (AutoIDResponse) {} 45 | rpc Rebase(RebaseRequest) returns (RebaseResponse) {} 46 | } 47 | 48 | -------------------------------------------------------------------------------- /proto/cdcpb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package cdcpb; 3 | 4 | import "raft_cmdpb.proto"; 5 | import "metapb.proto"; 6 | import "errorpb.proto"; 7 | import "kvrpcpb.proto"; 8 | 9 | import "gogoproto/gogo.proto"; 10 | import "rustproto.proto"; 11 | 12 | option(gogoproto.sizer_all) = true; 13 | option(gogoproto.marshaler_all) = true; 14 | option(gogoproto.unmarshaler_all) = true; 15 | option(rustproto.lite_runtime_all) = true; 16 | 17 | option java_package = "org.tikv.kvproto"; 18 | 19 | message Header { 20 | uint64 cluster_id = 1; 21 | string ticdc_version = 2; 22 | } 23 | 24 | message DuplicateRequest { 25 | uint64 region_id = 1; 26 | } 27 | 28 | message Compatibility { 29 | string required_version = 1; 30 | } 31 | 32 | // ClusterIDMismatch is an error variable that 33 | // tells people that the cluster ID of the request does not match the TiKV cluster ID. 34 | message ClusterIDMismatch { 35 | // The current tikv cluster ID. 36 | uint64 current = 1; 37 | // The cluster ID of the TiCDC request. 38 | uint64 request = 2; 39 | } 40 | 41 | message Error { 42 | errorpb.NotLeader not_leader = 1; 43 | errorpb.RegionNotFound region_not_found = 2; 44 | errorpb.EpochNotMatch epoch_not_match = 3; 45 | DuplicateRequest duplicate_request = 4; 46 | Compatibility compatibility = 5; 47 | ClusterIDMismatch cluster_id_mismatch = 6; 48 | errorpb.ServerIsBusy server_is_busy = 7; 49 | } 50 | 51 | message TxnInfo { 52 | uint64 start_ts = 1; 53 | bytes primary = 2; 54 | } 55 | 56 | message TxnStatus { 57 | uint64 start_ts = 1; 58 | uint64 min_commit_ts = 2; 59 | uint64 commit_ts = 3; 60 | bool is_rolled_back = 4; 61 | } 62 | 63 | message Event { 64 | enum LogType { 65 | UNKNOWN = 0; 66 | PREWRITE = 1; 67 | COMMIT = 2; 68 | ROLLBACK = 3; 69 | COMMITTED = 4; 70 | INITIALIZED = 5; 71 | } 72 | 73 | message Row { 74 | uint64 start_ts = 1; 75 | uint64 commit_ts = 2; 76 | LogType type = 3; 77 | 78 | enum OpType { 79 | UNKNOWN = 0; 80 | PUT = 1; 81 | DELETE = 2; 82 | } 83 | OpType op_type = 4; 84 | bytes key = 5; 85 | bytes value = 6; 86 | bytes old_value = 7; 87 | // expire_ts_unix_secs is used for RawKV (see `ChangeDataRequest.KvApi`), 88 | // and represents the expiration time of this row. 89 | // Absolute time, seconds since Unix epoch. 90 | uint64 expire_ts_unix_secs = 8; 91 | // The source of this row. 92 | uint64 txn_source = 9; 93 | } 94 | 95 | message Entries { 96 | repeated Row entries = 1; 97 | } 98 | 99 | message Admin { 100 | raft_cmdpb.AdminRequest admin_request = 1; 101 | raft_cmdpb.AdminResponse admin_response = 2; 102 | } 103 | 104 | message LongTxn { 105 | repeated TxnInfo txn_info = 1; 106 | } 107 | 108 | uint64 region_id = 1; 109 | uint64 index = 2; 110 | uint64 request_id = 7; 111 | oneof event { 112 | Entries entries = 3; 113 | Admin admin = 4; 114 | Error error = 5; 115 | uint64 resolved_ts = 6 [deprecated=true]; 116 | // Note that field 7 is taken by request_id. 117 | LongTxn long_txn = 8; 118 | // More region level events ... 119 | } 120 | } 121 | 122 | // NOTE: events and resolved_ts won't appear simultaneously in one ChangeDataEvent. 123 | message ChangeDataEvent { 124 | repeated Event events = 1; 125 | ResolvedTs resolved_ts = 2; 126 | // More store level events ... 127 | } 128 | 129 | message ResolvedTs { 130 | repeated uint64 regions = 1; 131 | uint64 ts = 2; 132 | uint64 request_id = 3; 133 | } 134 | 135 | message ChangeDataRequest { 136 | message Register {} 137 | 138 | message Deregister {} 139 | 140 | message NotifyTxnStatus { 141 | repeated TxnStatus txn_status = 1; 142 | } 143 | 144 | Header header = 1; 145 | uint64 region_id = 2; 146 | metapb.RegionEpoch region_epoch = 3; 147 | 148 | uint64 checkpoint_ts = 4; 149 | bytes start_key = 5; 150 | bytes end_key = 6; 151 | // Used for CDC to identify events corresponding to different requests. 152 | // Generally in one call stream, a region can be subscribed multiple 153 | // times with different `request_id`s. 154 | uint64 request_id = 7; 155 | 156 | kvrpcpb.ExtraOp extra_op = 8; 157 | 158 | oneof request { 159 | // A normal request that trying to register change data feed on a region. 160 | Register register = 9; 161 | 162 | // Notify the region that some of the running transactions on the region has a pushed 163 | // min_commit_ts so that the resolved_ts can be advanced. 164 | NotifyTxnStatus notify_txn_status = 10; 165 | 166 | Deregister deregister = 13; 167 | } 168 | 169 | // KvAPI specifies to capture data written by different KV API. 170 | // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. 171 | enum KvAPI { 172 | TiDB = 0; 173 | RawKV = 1; 174 | TxnKV = 2; 175 | } 176 | 177 | KvAPI kv_api = 11; 178 | 179 | // Whether to filter out the value write by cdc itself. 180 | bool filter_loop = 12; 181 | } 182 | 183 | service ChangeData { 184 | rpc EventFeed(stream ChangeDataRequest) returns(stream ChangeDataEvent); 185 | 186 | // EventFeedV2 is like EventFeed, with some new changes: 187 | // * clients send requested features in HTTP/2 headers; 188 | // * if servers meets unsupported feature request, 189 | // it can fail the stream with an UNIMPLEMENTED error. 190 | rpc EventFeedV2(stream ChangeDataRequest) returns (stream ChangeDataEvent); 191 | } 192 | -------------------------------------------------------------------------------- /proto/configpb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package configpb; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | import "google/api/annotations.proto"; 7 | 8 | option (gogoproto.sizer_all) = true; 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (rustproto.lite_runtime_all) = true; 12 | 13 | service Config { 14 | rpc Create(CreateRequest) returns (CreateResponse) {} 15 | rpc GetAll(GetAllRequest) returns (GetAllResponse) {} 16 | rpc Get(GetRequest) returns (GetResponse) { 17 | option (google.api.http) = { 18 | get: "/component" 19 | }; 20 | } 21 | rpc Update(UpdateRequest) returns (UpdateResponse) { 22 | option (google.api.http) = { 23 | post: "/component" 24 | body: "*" 25 | }; 26 | } 27 | rpc Delete(DeleteRequest) returns (DeleteResponse) { 28 | option (google.api.http) = { 29 | delete: "/component" 30 | }; 31 | } 32 | } 33 | 34 | enum StatusCode { 35 | UNKNOWN = 0; 36 | OK = 1; 37 | WRONG_VERSION = 2; 38 | NOT_CHANGE = 3; 39 | COMPONENT_NOT_FOUND = 4; 40 | COMPONENT_ID_NOT_FOUND = 5; 41 | } 42 | 43 | message Status { 44 | StatusCode code = 1; 45 | string message = 2; 46 | } 47 | 48 | // The version is used to tell the configuration which can be shared 49 | // or not apart. 50 | // Global version represents the version of these configuration 51 | // which can be shared, each kind of component only have one. 52 | // For local version, every component will have one to represent 53 | // the version of these configuration which cannot be shared. 54 | message Version { 55 | uint64 local = 1; 56 | uint64 global = 2; 57 | } 58 | 59 | message Local { 60 | string component_id = 1; 61 | } 62 | 63 | message Global { 64 | string component = 1; 65 | } 66 | 67 | message ConfigKind { 68 | oneof kind { 69 | Local local = 1; 70 | Global global = 2; 71 | } 72 | } 73 | 74 | message ConfigEntry { 75 | string name = 1; 76 | string value = 2; 77 | } 78 | 79 | message LocalConfig { 80 | Version version = 1; 81 | string component = 2; 82 | string component_id = 3; 83 | string config = 4; 84 | } 85 | 86 | message Header { 87 | uint64 cluster_id = 1; 88 | } 89 | 90 | message CreateRequest { 91 | Header header = 1; 92 | Version version = 2; 93 | string component = 3; 94 | string component_id = 4; 95 | string config = 5; 96 | } 97 | 98 | message CreateResponse { 99 | Header header = 1; 100 | Status status = 2; 101 | Version version = 3; 102 | string config = 4; 103 | } 104 | 105 | message GetAllRequest { 106 | Header header = 1; 107 | } 108 | 109 | message GetAllResponse { 110 | Header header = 1; 111 | Status status = 2; 112 | repeated LocalConfig local_configs = 3; 113 | } 114 | 115 | message GetRequest { 116 | Header header = 1; 117 | Version version = 2; 118 | string component = 3; 119 | string component_id = 4; 120 | } 121 | 122 | message GetResponse { 123 | Header header = 1; 124 | Status status = 2; 125 | Version version = 3; 126 | string config = 4; 127 | } 128 | 129 | message UpdateRequest { 130 | Header header = 1; 131 | Version version = 2; 132 | ConfigKind kind = 3; 133 | repeated ConfigEntry entries = 4; 134 | } 135 | 136 | message UpdateResponse { 137 | Header header = 1; 138 | Status status = 2; 139 | Version version = 3; 140 | string config = 4; 141 | } 142 | 143 | message DeleteRequest { 144 | Header header = 1; 145 | Version version = 2; 146 | ConfigKind kind = 3; 147 | } 148 | 149 | message DeleteResponse { 150 | Header header = 1; 151 | Status status = 2; 152 | Version version = 3; 153 | } 154 | -------------------------------------------------------------------------------- /proto/coprocessor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package coprocessor; 3 | 4 | import "errorpb.proto"; 5 | import "kvrpcpb.proto"; 6 | import "gogoproto/gogo.proto"; 7 | import "rustproto.proto"; 8 | import "metapb.proto"; 9 | 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.sizer_all) = true; 12 | option (gogoproto.unmarshaler_all) = true; 13 | option (rustproto.lite_runtime_all) = true; 14 | 15 | option java_package = "org.tikv.kvproto"; 16 | 17 | 18 | // [start, end) 19 | message KeyRange { 20 | bytes start = 1; 21 | bytes end = 2; 22 | } 23 | 24 | message Request { 25 | kvrpcpb.Context context = 1; 26 | int64 tp = 2; 27 | bytes data = 3; 28 | uint64 start_ts = 7; 29 | repeated KeyRange ranges = 4; 30 | 31 | // If cache is enabled, TiKV returns cache hit instead of data if 32 | // its last version matches this `cache_if_match_version`. 33 | bool is_cache_enabled = 5; 34 | uint64 cache_if_match_version = 6; 35 | // Any schema-ful storage to validate schema correctness if necessary. 36 | int64 schema_ver = 8; 37 | bool is_trace_enabled = 9; 38 | // paging_size is 0 when it's disabled, otherwise, it should be a positive number. 39 | uint64 paging_size = 10; 40 | // tasks stores the batched coprocessor tasks sent to the same tikv store. 41 | repeated StoreBatchTask tasks = 11; 42 | uint64 connection_id = 12; // This is the session id between a client and tidb 43 | string connection_alias = 13; // This is the session alias between a client and tidb 44 | } 45 | 46 | message Response { 47 | bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; 48 | errorpb.Error region_error = 2; 49 | kvrpcpb.LockInfo locked = 3; 50 | string other_error = 4; 51 | KeyRange range = 5; 52 | 53 | // This field is always filled for compatibility consideration. However 54 | // newer TiDB should respect `exec_details_v2` field instead. 55 | kvrpcpb.ExecDetails exec_details = 6; 56 | // This field is provided in later versions, containing more detailed 57 | // information. 58 | kvrpcpb.ExecDetailsV2 exec_details_v2 = 11; 59 | 60 | bool is_cache_hit = 7; 61 | uint64 cache_last_version = 8; 62 | bool can_be_cached = 9; 63 | 64 | reserved 10; 65 | 66 | // Contains the latest buckets version of the region. 67 | // Clients should query PD to update buckets in cache if its is stale. 68 | uint64 latest_buckets_version = 12; 69 | 70 | // StoreBatchTaskResponse is the collection of batch task responses. 71 | repeated StoreBatchTaskResponse batch_responses = 13; 72 | } 73 | 74 | message RegionInfo { 75 | uint64 region_id = 1; 76 | metapb.RegionEpoch region_epoch = 2; 77 | repeated KeyRange ranges = 3; 78 | } 79 | 80 | message TableRegions { 81 | int64 physical_table_id = 1; 82 | repeated RegionInfo regions = 2; 83 | } 84 | 85 | message BatchRequest { 86 | kvrpcpb.Context context = 1; 87 | int64 tp = 2; 88 | bytes data = 3; 89 | repeated RegionInfo regions = 4; 90 | uint64 start_ts = 5; 91 | // Any schema-ful storage to validate schema correctness if necessary. 92 | int64 schema_ver = 6; 93 | // Used for partition table scan 94 | repeated TableRegions table_regions = 7; 95 | string log_id = 8; 96 | uint64 connection_id = 9; // This is the session id between a client and tidb 97 | string connection_alias = 10; // This is the session alias between a client and tidb 98 | } 99 | 100 | message BatchResponse { 101 | bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; 102 | string other_error = 2; 103 | kvrpcpb.ExecDetails exec_details = 3; 104 | repeated metapb.Region retry_regions = 4; 105 | } 106 | 107 | message StoreBatchTask { 108 | uint64 region_id = 1; 109 | metapb.RegionEpoch region_epoch = 2; 110 | metapb.Peer peer = 3; 111 | repeated KeyRange ranges = 4; 112 | uint64 task_id = 5; 113 | } 114 | 115 | message StoreBatchTaskResponse { 116 | bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; 117 | errorpb.Error region_error = 2; 118 | kvrpcpb.LockInfo locked = 3; 119 | string other_error = 4; 120 | uint64 task_id = 5; 121 | kvrpcpb.ExecDetailsV2 exec_details_v2 = 6; 122 | } 123 | -------------------------------------------------------------------------------- /proto/deadlock.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package deadlock; 3 | 4 | import "gogoproto/gogo.proto"; 5 | 6 | message WaitForEntriesRequest { 7 | } 8 | 9 | message WaitForEntriesResponse { 10 | repeated WaitForEntry entries = 1 [(gogoproto.nullable) = false]; 11 | } 12 | 13 | message WaitForEntry { 14 | // The transaction id that is waiting. 15 | uint64 txn = 1; 16 | // The transaction id that is being waited for. 17 | uint64 wait_for_txn = 2; 18 | // The hash value of the key is being waited for. 19 | uint64 key_hash = 3; 20 | // The key the current txn is trying to lock. 21 | bytes key = 4; 22 | // The tag came from the lock request's context. 23 | bytes resource_group_tag = 5; 24 | // Milliseconds it has been waits. 25 | uint64 wait_time = 6; 26 | } 27 | 28 | enum DeadlockRequestType { 29 | Detect = 0; 30 | // CleanUpWaitFor cleans a single entry the transaction is waiting. 31 | CleanUpWaitFor = 1; 32 | // CleanUp cleans all entries the transaction is waiting. 33 | CleanUp = 2; 34 | } 35 | 36 | message DeadlockRequest { 37 | DeadlockRequestType tp = 1; 38 | WaitForEntry entry = 2 [(gogoproto.nullable) = false]; 39 | } 40 | 41 | message DeadlockResponse { 42 | // The same entry sent by DeadlockRequest, identifies the sender. 43 | WaitForEntry entry = 1 [(gogoproto.nullable) = false]; 44 | // The key hash of the lock that is hold by the waiting transaction. 45 | uint64 deadlock_key_hash = 2; 46 | // The other entries of the dead lock circle. The current entry is in `entry` field and not 47 | // included in this field. 48 | repeated WaitForEntry wait_chain = 3; 49 | } 50 | 51 | service Deadlock { 52 | // Get local wait for entries, should be handle by every node. 53 | // The owner should sent this request to all members to build the complete wait for graph. 54 | rpc GetWaitForEntries(WaitForEntriesRequest) returns (WaitForEntriesResponse) {} 55 | 56 | // Detect should only sent to the owner. only be handled by the owner. 57 | // The DeadlockResponse is sent back only if there is deadlock detected. 58 | // CleanUpWaitFor and CleanUp doesn't return responses. 59 | rpc Detect(stream DeadlockRequest) returns (stream DeadlockResponse) {} 60 | } 61 | -------------------------------------------------------------------------------- /proto/diagnosticspb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package diagnosticspb; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | 7 | option (gogoproto.sizer_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option (rustproto.lite_runtime_all) = true; 11 | 12 | option java_package = "org.tikv.kvproto"; 13 | 14 | // Diagnostics service for TiDB cluster components. 15 | service Diagnostics { 16 | // Searchs log in the target node 17 | rpc search_log(SearchLogRequest) returns (stream SearchLogResponse) {}; 18 | // Retrieves server info in the target node 19 | rpc server_info(ServerInfoRequest) returns (ServerInfoResponse) {}; 20 | } 21 | 22 | enum LogLevel { 23 | UNKNOWN = 0; 24 | Debug = 1; 25 | Info = 2; 26 | Warn = 3; 27 | Trace = 4; 28 | Critical = 5; 29 | Error = 6; 30 | } 31 | 32 | message SearchLogRequest { 33 | enum Target { 34 | Normal = 0; 35 | Slow = 1; 36 | } 37 | int64 start_time = 1; 38 | int64 end_time = 2; 39 | repeated LogLevel levels = 3; 40 | // We use a string array to represent multiple CNF pattern sceniaor like: 41 | // SELECT * FROM t WHERE c LIKE '%s%' and c REGEXP '.*a.*' because 42 | // Golang and Rust don't support perl-like (?=re1)(?=re2) 43 | repeated string patterns = 4; 44 | Target target = 5; 45 | } 46 | 47 | message SearchLogResponse { 48 | repeated LogMessage messages = 1; 49 | } 50 | 51 | message LogMessage { 52 | int64 time = 1; 53 | LogLevel level = 2; 54 | string message = 3; 55 | } 56 | 57 | enum ServerInfoType { 58 | All = 0; 59 | HardwareInfo = 1; 60 | SystemInfo = 2; 61 | LoadInfo = 3; 62 | } 63 | 64 | message ServerInfoRequest { 65 | ServerInfoType tp = 1; 66 | } 67 | 68 | message ServerInfoPair { 69 | string key = 1; 70 | string value = 2; 71 | } 72 | 73 | message ServerInfoItem { 74 | // cpu, memory, disk, network ... 75 | string tp = 1; 76 | // eg. network: lo1/eth0, cpu: core1/core2, disk: sda1/sda2 77 | string name = 2; 78 | // all key-value pairs for specified item, e.g: 79 | // ServerInfoItem { 80 | // tp = "network" 81 | // name = "eth0" 82 | // paris = [ 83 | // ServerInfoPair { key = "readbytes", value = "4k"}, 84 | // ServerInfoPair { key = "writebytes", value = "1k"}, 85 | // ] 86 | // } 87 | repeated ServerInfoPair pairs = 3; 88 | } 89 | 90 | message ServerInfoResponse { 91 | repeated ServerInfoItem items = 1; 92 | } 93 | -------------------------------------------------------------------------------- /proto/disaggregated.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package disaggregated; 3 | 4 | import "coprocessor.proto"; 5 | import "kvrpcpb.proto"; 6 | 7 | option java_package = "org.tikv.kvproto"; 8 | 9 | /// For S3 lock service /// 10 | 11 | message S3LockResult { 12 | oneof error { 13 | Success success = 1; 14 | NotOwner not_owner = 2; 15 | Conflict conflict = 3; 16 | } 17 | } 18 | message Success {} 19 | // Error caused by S3GC owner changed 20 | // client should retry 21 | message NotOwner{ 22 | } 23 | // Error caused by concurrency conflict, 24 | // request cancel 25 | message Conflict { 26 | string reason = 1; 27 | } 28 | 29 | message TryAddLockRequest { 30 | // The data file key to add lock 31 | bytes data_file_key = 1; 32 | // The lock store id 33 | uint64 lock_store_id = 3; 34 | // The upload sequence number of lock store 35 | uint64 lock_seq = 4; 36 | } 37 | 38 | message TryAddLockResponse { 39 | S3LockResult result = 1; 40 | } 41 | 42 | message TryMarkDeleteRequest { 43 | // The data file key to be marked as deleted 44 | bytes data_file_key = 1; 45 | } 46 | 47 | message TryMarkDeleteResponse { 48 | S3LockResult result = 1; 49 | } 50 | 51 | /// For disagg compute node init its disagg configuration /// 52 | 53 | message GetDisaggConfigRequest { 54 | } 55 | message DisaggS3Config { 56 | string bucket = 1; 57 | string root = 2; 58 | string endpoint = 3; 59 | } 60 | message GetDisaggConfigResponse { 61 | DisaggS3Config s3_config = 1; 62 | } 63 | 64 | /// For compute task dispatch and data exchange /// 65 | 66 | message DisaggTaskMeta { 67 | uint64 start_ts = 1; // start ts of a query 68 | // gather_id + query_ts + server_id + local_query_id to represent a global unique query. 69 | int64 gather_id = 9; // used to distinguish different gathers in the mpp query 70 | uint64 query_ts = 2; // timestamp when start to execute query, used for TiFlash miniTSO schedule. 71 | uint64 server_id = 3; // TiDB server id 72 | uint64 local_query_id = 4; // unique local query_id if tidb don't restart. 73 | int64 task_id = 5; // if task id is -1 , it indicates a tidb task. 74 | string executor_id = 6; // the exectuor id 75 | uint32 keyspace_id = 7; // keyspace id of the request 76 | kvrpcpb.APIVersion api_version = 8; // API version of the request 77 | uint64 connection_id = 10; // This is the session id between a client and tidb 78 | string connection_alias = 11; // This is the session alias between a client and tidb 79 | } 80 | 81 | message DisaggReadError { 82 | int32 code = 1; 83 | string msg = 2; 84 | } 85 | 86 | message EstablishDisaggTaskError { 87 | oneof errors { 88 | ErrorRegion error_region = 1; 89 | ErrorLocked error_locked = 2; 90 | 91 | ErrorOther error_other = 99; 92 | } 93 | } 94 | 95 | message ErrorRegion { 96 | string msg = 1; 97 | // The read node needs to update its region cache about these regions. 98 | repeated uint64 region_ids = 2; 99 | } 100 | 101 | message ErrorLocked { 102 | string msg = 1; 103 | // The read node needs to resolve these locks. 104 | repeated kvrpcpb.LockInfo locked = 2; 105 | } 106 | 107 | message ErrorOther { 108 | int32 code = 1; 109 | string msg = 2; 110 | } 111 | 112 | message EstablishDisaggTaskRequest { 113 | DisaggTaskMeta meta = 1; 114 | string address = 2; // target address of this task. 115 | // The write node needs to ensure that subsequent 116 | // FetchDisaggPagesRequest can be processed within timeout_s. 117 | // unit: seconds 118 | int64 timeout_s = 3; 119 | // The key ranges, Region meta that read node need to execute TableScan 120 | repeated coprocessor.RegionInfo regions = 4; 121 | int64 schema_ver = 5; 122 | // Used for PartitionTableScan 123 | repeated coprocessor.TableRegions table_regions = 6; 124 | // The encoded TableScan/PartitionTableScan + Selection. 125 | bytes encoded_plan = 7; 126 | } 127 | 128 | message EstablishDisaggTaskResponse { 129 | EstablishDisaggTaskError error = 1; 130 | 131 | // Write node maintains a snapshot with a lease time. 132 | // Read node should read the delta pages 133 | // (ColumnFileInMemory and ColumnFileTiny) 134 | // along with this store_id and snapshot_id. 135 | uint64 store_id = 3; // metapb.Store.id 136 | DisaggTaskMeta snapshot_id = 4; 137 | // Serialized disaggregated tasks (per physical table) 138 | repeated bytes tables = 5; 139 | } 140 | 141 | message CancelDisaggTaskRequest { 142 | DisaggTaskMeta meta = 1; 143 | } 144 | 145 | message CancelDisaggTaskResponse {} 146 | 147 | message FetchDisaggPagesRequest { 148 | // The snapshot id to fetch pages 149 | DisaggTaskMeta snapshot_id = 1; 150 | int64 table_id = 2; 151 | uint64 segment_id = 3; 152 | // It must be a subset of the delta pages ids returned 153 | // in EstablishDisaggTaskResponse.segments 154 | repeated uint64 page_ids = 4; 155 | } 156 | 157 | message PagesPacket { 158 | DisaggReadError error = 1; 159 | 160 | // Serialized column file data 161 | // * ColumnFilePersisted alone with its schema, page data, field offsets 162 | repeated bytes pages = 2; 163 | // * ColumnFileInMemory alone with its serialized block 164 | repeated bytes chunks = 3; 165 | 166 | // Return tipb.SelectResponse.execution_summaries in the 167 | // last packet 168 | repeated bytes summaries = 4; 169 | } 170 | -------------------------------------------------------------------------------- /proto/disk_usage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package disk_usage; 3 | 4 | import "rustproto.proto"; 5 | 6 | option (rustproto.lite_runtime_all) = true; 7 | 8 | option java_package = "org.tikv.kvproto"; 9 | 10 | enum DiskUsage { 11 | Normal = 0; 12 | AlmostFull = 1; 13 | AlreadyFull = 2; 14 | } 15 | -------------------------------------------------------------------------------- /proto/encryptionpb.proto: -------------------------------------------------------------------------------- 1 | // These encryption protobufs are not sent over the network. 2 | // Protobufs are used to define a stable backwards compatible persistent storage format. 3 | // These definitions are used by both PD and TiKV to keep their implementations similar. 4 | 5 | syntax = "proto3"; 6 | package encryptionpb; 7 | 8 | import "gogoproto/gogo.proto"; 9 | import "rustproto.proto"; 10 | 11 | option (gogoproto.sizer_all) = true; 12 | option (gogoproto.marshaler_all) = true; 13 | option (gogoproto.unmarshaler_all) = true; 14 | option (rustproto.lite_runtime_all) = true; 15 | 16 | option java_package = "org.tikv.kvproto"; 17 | 18 | // General encryption metadata for any data type. 19 | message EncryptionMeta { 20 | // ID of the key used to encrypt the data. 21 | uint64 key_id = 1; 22 | // Initialization vector (IV) of the data. 23 | bytes iv = 2; 24 | } 25 | 26 | // Information about an encrypted file. 27 | message FileInfo { 28 | // ID of the key used to encrypt the file. 29 | uint64 key_id = 1; 30 | // Initialization vector (IV) of the file. 31 | bytes iv = 2; 32 | // Method of encryption algorithm used to encrypted the file. 33 | EncryptionMethod method = 3; 34 | } 35 | 36 | message FileDictionary { 37 | // A map of file name to file info. 38 | map files = 1; 39 | } 40 | 41 | enum EncryptionMethod { 42 | UNKNOWN = 0; 43 | PLAINTEXT = 1; 44 | AES128_CTR = 2; 45 | AES192_CTR = 3; 46 | AES256_CTR = 4; 47 | SM4_CTR = 5; 48 | } 49 | 50 | // The key used to encrypt the user data. 51 | message DataKey { 52 | // A sequence of secret bytes used to encrypt data. 53 | bytes key = 1; 54 | // Method of encryption algorithm used to encrypted data. 55 | EncryptionMethod method = 2; 56 | // Creation time of the key. 57 | uint64 creation_time = 3; 58 | // A flag for the key have ever been exposed. 59 | bool was_exposed = 4; 60 | } 61 | 62 | message KeyDictionary { 63 | // A map of key ID to dat key. 64 | map keys = 1; 65 | // ID of a key currently in use. 66 | uint64 current_key_id = 2; 67 | } 68 | 69 | // Master key config. 70 | message MasterKey { 71 | oneof backend { 72 | MasterKeyPlaintext plaintext = 1; 73 | MasterKeyFile file = 2; 74 | MasterKeyKms kms = 3; 75 | } 76 | } 77 | 78 | // MasterKeyPlaintext indicates content is stored as plaintext. 79 | message MasterKeyPlaintext {} 80 | 81 | // MasterKeyFile is a master key backed by a file containing encryption key in human-readable 82 | // hex format. 83 | message MasterKeyFile { 84 | // Local file path. 85 | string path = 1; 86 | } 87 | 88 | // MasterKeyKms is a master key backed by KMS service that manages the encryption key, 89 | // and provide API to encrypt and decrypt a data key, which is used to encrypt the content. 90 | message MasterKeyKms { 91 | // KMS vendor. 92 | string vendor = 1; 93 | // KMS key id. 94 | string key_id = 2; 95 | // KMS region. 96 | string region = 3; 97 | // KMS endpoint. Normally not needed. 98 | string endpoint = 4; 99 | } 100 | 101 | message EncryptedContent { 102 | // Metadata of the encrypted content. 103 | // Eg. IV, method and KMS key ID 104 | // It is preferred to define new fields for extra metadata than using this metadata map. 105 | map metadata = 1; 106 | // Encrypted content. 107 | bytes content = 2; 108 | // Master key used to encrypt the content. 109 | MasterKey master_key = 3; 110 | // Initilization vector (IV) used. 111 | bytes iv = 4; 112 | // Encrypted data key generated by KMS and used to actually encrypt data. 113 | // Valid only when KMS is used. 114 | bytes ciphertext_key = 5; 115 | } 116 | -------------------------------------------------------------------------------- /proto/enginepb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package enginepb; 3 | 4 | import "metapb.proto"; 5 | import "raft_cmdpb.proto"; 6 | import "raft_serverpb.proto"; 7 | 8 | message CommandRequestHeader { 9 | uint64 region_id = 1; 10 | uint64 index = 2; 11 | uint64 term = 3; 12 | 13 | // Flush in-memory data to disk. 14 | bool sync_log = 4; 15 | // Destroy the region. 16 | bool destroy = 5; 17 | 18 | // Additional information for the request. 19 | bytes context = 6; 20 | } 21 | 22 | message CommandRequest { 23 | CommandRequestHeader header = 1; 24 | 25 | // We don't enclose normal requests and administrator request 26 | // at same time. 27 | 28 | // kv put / delete 29 | repeated raft_cmdpb.Request requests = 2; 30 | 31 | // region metadata manipulation command. 32 | raft_cmdpb.AdminRequest admin_request = 3; 33 | // region metadata manipulation result. 34 | raft_cmdpb.AdminResponse admin_response = 4; 35 | } 36 | 37 | message CommandRequestBatch { 38 | repeated CommandRequest requests = 1; 39 | } 40 | 41 | message CommandResponseHeader { 42 | uint64 region_id = 1; 43 | // Region is destroyed. 44 | bool destroyed = 2; 45 | } 46 | 47 | message CommandResponse { 48 | CommandResponseHeader header = 1; 49 | 50 | raft_serverpb.RaftApplyState apply_state = 2; 51 | uint64 applied_term = 3; 52 | } 53 | 54 | message CommandResponseBatch { 55 | repeated CommandResponse responses = 1; 56 | } 57 | 58 | message SnapshotState { 59 | metapb.Region region = 1; 60 | metapb.Peer peer = 2; 61 | raft_serverpb.RaftApplyState apply_state = 3; 62 | } 63 | 64 | message SnapshotData { 65 | string cf = 1; 66 | uint32 checksum = 2; 67 | repeated raft_serverpb.KeyValue data = 3; 68 | } 69 | 70 | message SnapshotRequest { 71 | oneof chunk { 72 | // The first message for snapshots. 73 | // It contains the latest region information after applied snapshot. 74 | SnapshotState state = 1; 75 | 76 | // Following messages are always data. 77 | SnapshotData data = 2; 78 | } 79 | } 80 | 81 | message SnapshotDone {} 82 | 83 | service Engine { 84 | rpc ApplyCommandBatch(stream CommandRequestBatch) returns (stream CommandResponseBatch) {} 85 | rpc ApplySnapshot(stream SnapshotRequest) returns (SnapshotDone) {} 86 | } 87 | -------------------------------------------------------------------------------- /proto/gcpb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package gcpb; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | 7 | option (gogoproto.sizer_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option (rustproto.lite_runtime_all) = true; 11 | 12 | option java_package = "org.tikv.kvproto"; 13 | 14 | service GC { 15 | rpc ListKeySpaces(ListKeySpacesRequest) returns (ListKeySpacesResponse) {} 16 | 17 | rpc GetMinServiceSafePoint(GetMinServiceSafePointRequest) returns (GetMinServiceSafePointResponse) {} 18 | 19 | rpc UpdateGCSafePoint(UpdateGCSafePointRequest) returns (UpdateGCSafePointResponse) {} 20 | 21 | rpc UpdateServiceSafePoint(UpdateServiceSafePointRequest) returns (UpdateServiceSafePointResponse) {} 22 | } 23 | 24 | message RequestHeader { 25 | // cluster_id is the ID of the cluster which be sent to. 26 | uint64 cluster_id = 1; 27 | // sender_id is the ID of the sender server, also member ID or etcd ID. 28 | uint64 sender_id = 2; 29 | } 30 | 31 | message ResponseHeader { 32 | // cluster_id is the ID of the cluster which sent the response. 33 | uint64 cluster_id = 1; 34 | Error error = 2; 35 | } 36 | 37 | enum ErrorType { 38 | OK = 0; 39 | UNKNOWN = 1; 40 | NOT_BOOTSTRAPPED = 2; 41 | // revision supplied does not match the current etcd revision 42 | REVISION_MISMATCH = 3; 43 | // if the proposed safe point is earlier than old safe point or gc safe point 44 | SAFEPOINT_ROLLBACK = 4; 45 | } 46 | 47 | message Error { 48 | ErrorType type = 1; 49 | string message = 2; 50 | } 51 | 52 | message KeySpace { 53 | bytes space_id = 1; 54 | uint64 gc_safe_point = 2; 55 | } 56 | 57 | message ListKeySpacesRequest { 58 | RequestHeader header = 1; 59 | // set with_gc_safe_point to true to also receive gc safe point for each key space 60 | bool with_gc_safe_point = 2; 61 | } 62 | 63 | message ListKeySpacesResponse { 64 | ResponseHeader header = 1; 65 | repeated KeySpace key_spaces = 2; 66 | } 67 | 68 | message GetMinServiceSafePointRequest { 69 | RequestHeader header = 1; 70 | bytes space_id = 2; 71 | } 72 | 73 | message GetMinServiceSafePointResponse { 74 | ResponseHeader header = 1; 75 | uint64 safe_point = 2; 76 | // revision here is to safeguard the validity of the obtained min, 77 | // preventing cases where new services register their safe points after min is obtained by gc worker 78 | int64 revision = 3; 79 | } 80 | 81 | message UpdateGCSafePointRequest { 82 | RequestHeader header = 1; 83 | bytes space_id = 2; 84 | uint64 safe_point = 3; 85 | // here client need to provide the revision obtained from GetMinServiceSafePoint, 86 | // so server can check if it's still valid 87 | int64 revision = 4; 88 | } 89 | 90 | message UpdateGCSafePointResponse { 91 | ResponseHeader header = 1; 92 | // update will be successful if revision is valid and new safepoint > old safe point 93 | // if failed, previously obtained min might be incorrect, should retry from GetMinServiceSafePoint 94 | bool succeeded = 2; 95 | uint64 new_safe_point = 3; 96 | } 97 | 98 | message UpdateServiceSafePointRequest { 99 | RequestHeader header = 1; 100 | bytes space_id = 2; 101 | bytes service_id = 3; 102 | // safe point will be set to expire on (PD Server time + TTL) 103 | // pass in a ttl < 0 to remove target safe point 104 | // pass in MAX_INT64 to set a safe point that never expire 105 | int64 TTL = 4; 106 | uint64 safe_point = 5; 107 | } 108 | 109 | message UpdateServiceSafePointResponse { 110 | ResponseHeader header = 1; 111 | // update will be successful if ttl < 0 (a removal request) 112 | // or if new safe point >= old safe point and new safe point >= gc safe point 113 | bool succeeded = 2; 114 | uint64 gc_safe_point = 3; 115 | uint64 old_safe_point = 4; 116 | uint64 new_safe_point = 5; 117 | } 118 | -------------------------------------------------------------------------------- /proto/import_kvpb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package import_kvpb; 4 | 5 | import "import_sstpb.proto"; 6 | import "gogoproto/gogo.proto"; 7 | import "rustproto.proto"; 8 | 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (rustproto.lite_runtime_all) = true; 13 | 14 | option java_package = "org.tikv.kvproto"; 15 | 16 | // ImportKV provides a service to import key-value pairs to TiKV. 17 | // 18 | // In order to import key-value pairs to TiKV, the user should: 19 | // 1. Open an engine identified by an UUID. 20 | // 2. Open write streams to write key-value batches to the opened engine. 21 | // Different streams/clients can write to the same engine concurrently. 22 | // 3. Close the engine after all write batches have been finished. An 23 | // engine can only be closed when all write streams are closed. An 24 | // engine can only be closed once, and it can not be opened again 25 | // once it is closed. 26 | // 4. Import the data in the engine to the target cluster. Note that 27 | // the import process is not atomic, it requires the data to be 28 | // idempotent on retry. An engine can only be imported after it is 29 | // closed. An engine can be imported multiple times, but can not be 30 | // imported concurrently. 31 | // 5. Clean up the engine after it has been imported. Delete all data 32 | // in the engine. An engine can not be cleaned up when it is 33 | // writing or importing. 34 | service ImportKV { 35 | // Switch the target cluster to normal/import mode. 36 | rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} 37 | // Open an engine. 38 | rpc OpenEngine(OpenEngineRequest) returns (OpenEngineResponse) {} 39 | // Open a write stream to the engine. 40 | rpc WriteEngine(stream WriteEngineRequest) returns (WriteEngineResponse) {} 41 | // Write to engine, single message version 42 | rpc WriteEngineV3(WriteEngineV3Request) returns (WriteEngineResponse) {} 43 | // Close the engine. 44 | rpc CloseEngine(CloseEngineRequest) returns (CloseEngineResponse) {} 45 | // Import the engine to the target cluster. 46 | rpc ImportEngine(ImportEngineRequest) returns (ImportEngineResponse) {} 47 | // Clean up the engine. 48 | rpc CleanupEngine(CleanupEngineRequest) returns (CleanupEngineResponse) {} 49 | // Compact the target cluster for better performance. 50 | rpc CompactCluster(CompactClusterRequest) returns (CompactClusterResponse) {} 51 | // Get current version and commit hash 52 | rpc GetVersion(GetVersionRequest) returns (GetVersionResponse) {} 53 | // Get importer metrics 54 | rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} 55 | } 56 | 57 | message SwitchModeRequest { 58 | string pd_addr = 1; 59 | import_sstpb.SwitchModeRequest request = 2; 60 | } 61 | 62 | message SwitchModeResponse { 63 | } 64 | 65 | message OpenEngineRequest { 66 | bytes uuid = 1; 67 | bytes key_prefix = 2; 68 | } 69 | 70 | message OpenEngineResponse { 71 | } 72 | 73 | message WriteHead { 74 | bytes uuid = 1; 75 | } 76 | 77 | message Mutation { 78 | enum OP { 79 | Put = 0; 80 | } 81 | OP op = 1; 82 | bytes key = 2; 83 | bytes value = 3; 84 | } 85 | 86 | message WriteBatch { 87 | uint64 commit_ts = 1; 88 | repeated Mutation mutations = 2; 89 | } 90 | 91 | message WriteEngineRequest { 92 | oneof chunk { 93 | WriteHead head = 1; 94 | WriteBatch batch = 2; 95 | } 96 | } 97 | 98 | message KVPair { 99 | bytes key = 1; 100 | bytes value = 2; 101 | } 102 | 103 | message WriteEngineV3Request { 104 | bytes uuid = 1; 105 | uint64 commit_ts = 2; 106 | repeated KVPair pairs = 3; 107 | } 108 | 109 | message WriteEngineResponse { 110 | Error error = 1; 111 | } 112 | 113 | message CloseEngineRequest { 114 | bytes uuid = 1; 115 | } 116 | 117 | message CloseEngineResponse { 118 | Error error = 1; 119 | } 120 | 121 | message ImportEngineRequest { 122 | bytes uuid = 1; 123 | string pd_addr = 2; 124 | } 125 | 126 | message ImportEngineResponse { 127 | } 128 | 129 | message CleanupEngineRequest { 130 | bytes uuid = 1; 131 | } 132 | 133 | message CleanupEngineResponse { 134 | } 135 | 136 | message CompactClusterRequest { 137 | string pd_addr = 1; 138 | import_sstpb.CompactRequest request = 2; 139 | } 140 | 141 | message CompactClusterResponse { 142 | } 143 | 144 | message GetVersionRequest { 145 | } 146 | 147 | message GetVersionResponse { 148 | string version = 1; 149 | string commit = 2; 150 | } 151 | 152 | message GetMetricsRequest { 153 | } 154 | 155 | message GetMetricsResponse { 156 | string prometheus = 1; 157 | } 158 | 159 | message Error { 160 | message EngineNotFound { 161 | bytes uuid = 1; 162 | } 163 | // This can happen if the client hasn't opened the engine, or the server 164 | // restarts while the client is writing or closing. An unclosed engine will 165 | // be removed on server restart, so the client should not continue but 166 | // restart the previous job in that case. 167 | EngineNotFound engine_not_found = 1; 168 | } 169 | -------------------------------------------------------------------------------- /proto/include/gogoproto/gogo.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers for Go with Gadgets 2 | // 3 | // Copyright (c) 2013, The GoGo Authors. All rights reserved. 4 | // http://github.com/gogo/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | syntax = "proto2"; 30 | package gogoproto; 31 | 32 | import "google/protobuf/descriptor.proto"; 33 | 34 | option java_package = "com.google.protobuf"; 35 | option java_outer_classname = "GoGoProtos"; 36 | option go_package = "github.com/gogo/protobuf/gogoproto"; 37 | 38 | extend google.protobuf.EnumOptions { 39 | optional bool goproto_enum_prefix = 62001; 40 | optional bool goproto_enum_stringer = 62021; 41 | optional bool enum_stringer = 62022; 42 | optional string enum_customname = 62023; 43 | optional bool enumdecl = 62024; 44 | } 45 | 46 | extend google.protobuf.EnumValueOptions { 47 | optional string enumvalue_customname = 66001; 48 | } 49 | 50 | extend google.protobuf.FileOptions { 51 | optional bool goproto_getters_all = 63001; 52 | optional bool goproto_enum_prefix_all = 63002; 53 | optional bool goproto_stringer_all = 63003; 54 | optional bool verbose_equal_all = 63004; 55 | optional bool face_all = 63005; 56 | optional bool gostring_all = 63006; 57 | optional bool populate_all = 63007; 58 | optional bool stringer_all = 63008; 59 | optional bool onlyone_all = 63009; 60 | 61 | optional bool equal_all = 63013; 62 | optional bool description_all = 63014; 63 | optional bool testgen_all = 63015; 64 | optional bool benchgen_all = 63016; 65 | optional bool marshaler_all = 63017; 66 | optional bool unmarshaler_all = 63018; 67 | optional bool stable_marshaler_all = 63019; 68 | 69 | optional bool sizer_all = 63020; 70 | 71 | optional bool goproto_enum_stringer_all = 63021; 72 | optional bool enum_stringer_all = 63022; 73 | 74 | optional bool unsafe_marshaler_all = 63023; 75 | optional bool unsafe_unmarshaler_all = 63024; 76 | 77 | optional bool goproto_extensions_map_all = 63025; 78 | optional bool goproto_unrecognized_all = 63026; 79 | optional bool gogoproto_import = 63027; 80 | optional bool protosizer_all = 63028; 81 | optional bool compare_all = 63029; 82 | optional bool typedecl_all = 63030; 83 | optional bool enumdecl_all = 63031; 84 | 85 | optional bool goproto_registration = 63032; 86 | optional bool messagename_all = 63033; 87 | } 88 | 89 | extend google.protobuf.MessageOptions { 90 | optional bool goproto_getters = 64001; 91 | optional bool goproto_stringer = 64003; 92 | optional bool verbose_equal = 64004; 93 | optional bool face = 64005; 94 | optional bool gostring = 64006; 95 | optional bool populate = 64007; 96 | optional bool stringer = 67008; 97 | optional bool onlyone = 64009; 98 | 99 | optional bool equal = 64013; 100 | optional bool description = 64014; 101 | optional bool testgen = 64015; 102 | optional bool benchgen = 64016; 103 | optional bool marshaler = 64017; 104 | optional bool unmarshaler = 64018; 105 | optional bool stable_marshaler = 64019; 106 | 107 | optional bool sizer = 64020; 108 | 109 | optional bool unsafe_marshaler = 64023; 110 | optional bool unsafe_unmarshaler = 64024; 111 | 112 | optional bool goproto_extensions_map = 64025; 113 | optional bool goproto_unrecognized = 64026; 114 | 115 | optional bool protosizer = 64028; 116 | optional bool compare = 64029; 117 | 118 | optional bool typedecl = 64030; 119 | 120 | optional bool messagename = 64033; 121 | } 122 | 123 | extend google.protobuf.FieldOptions { 124 | optional bool nullable = 65001; 125 | optional bool embed = 65002; 126 | optional string customtype = 65003; 127 | optional string customname = 65004; 128 | optional string jsontag = 65005; 129 | optional string moretags = 65006; 130 | optional string casttype = 65007; 131 | optional string castkey = 65008; 132 | optional string castvalue = 65009; 133 | 134 | optional bool stdtime = 65010; 135 | optional bool stdduration = 65011; 136 | } 137 | -------------------------------------------------------------------------------- /proto/include/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } 32 | -------------------------------------------------------------------------------- /proto/include/google/protobuf/duration.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option cc_enable_arenas = true; 37 | option go_package = "types"; 38 | option java_package = "com.google.protobuf"; 39 | option java_outer_classname = "DurationProto"; 40 | option java_multiple_files = true; 41 | option objc_class_prefix = "GPB"; 42 | 43 | // A Duration represents a signed, fixed-length span of time represented 44 | // as a count of seconds and fractions of seconds at nanosecond 45 | // resolution. It is independent of any calendar and concepts like "day" 46 | // or "month". It is related to Timestamp in that the difference between 47 | // two Timestamp values is a Duration and it can be added or subtracted 48 | // from a Timestamp. Range is approximately +-10,000 years. 49 | // 50 | // # Examples 51 | // 52 | // Example 1: Compute Duration from two Timestamps in pseudo code. 53 | // 54 | // Timestamp start = ...; 55 | // Timestamp end = ...; 56 | // Duration duration = ...; 57 | // 58 | // duration.seconds = end.seconds - start.seconds; 59 | // duration.nanos = end.nanos - start.nanos; 60 | // 61 | // if (duration.seconds < 0 && duration.nanos > 0) { 62 | // duration.seconds += 1; 63 | // duration.nanos -= 1000000000; 64 | // } else if (durations.seconds > 0 && duration.nanos < 0) { 65 | // duration.seconds -= 1; 66 | // duration.nanos += 1000000000; 67 | // } 68 | // 69 | // Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. 70 | // 71 | // Timestamp start = ...; 72 | // Duration duration = ...; 73 | // Timestamp end = ...; 74 | // 75 | // end.seconds = start.seconds + duration.seconds; 76 | // end.nanos = start.nanos + duration.nanos; 77 | // 78 | // if (end.nanos < 0) { 79 | // end.seconds -= 1; 80 | // end.nanos += 1000000000; 81 | // } else if (end.nanos >= 1000000000) { 82 | // end.seconds += 1; 83 | // end.nanos -= 1000000000; 84 | // } 85 | // 86 | // Example 3: Compute Duration from datetime.timedelta in Python. 87 | // 88 | // td = datetime.timedelta(days=3, minutes=10) 89 | // duration = Duration() 90 | // duration.FromTimedelta(td) 91 | // 92 | // # JSON Mapping 93 | // 94 | // In JSON format, the Duration type is encoded as a string rather than an 95 | // object, where the string ends in the suffix "s" (indicating seconds) and 96 | // is preceded by the number of seconds, with nanoseconds expressed as 97 | // fractional seconds. For example, 3 seconds with 0 nanoseconds should be 98 | // encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should 99 | // be expressed in JSON format as "3.000000001s", and 3 seconds and 1 100 | // microsecond should be expressed in JSON format as "3.000001s". 101 | // 102 | // 103 | message Duration { 104 | 105 | // Signed seconds of the span of time. Must be from -315,576,000,000 106 | // to +315,576,000,000 inclusive. Note: these bounds are computed from: 107 | // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years 108 | int64 seconds = 1; 109 | 110 | // Signed fractions of a second at nanosecond resolution of the span 111 | // of time. Durations less than one second are represented with a 0 112 | // `seconds` field and a positive or negative `nanos` field. For durations 113 | // of one second or more, a non-zero value for the `nanos` field must be 114 | // of the same sign as the `seconds` field. Must be from -999,999,999 115 | // to +999,999,999 inclusive. 116 | int32 nanos = 2; 117 | } 118 | -------------------------------------------------------------------------------- /proto/include/google/protobuf/empty.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option go_package = "types"; 37 | option java_package = "com.google.protobuf"; 38 | option java_outer_classname = "EmptyProto"; 39 | option java_multiple_files = true; 40 | option objc_class_prefix = "GPB"; 41 | option cc_enable_arenas = true; 42 | 43 | // A generic empty message that you can re-use to avoid defining duplicated 44 | // empty messages in your APIs. A typical example is to use it as the request 45 | // or the response type of an API method. For instance: 46 | // 47 | // service Foo { 48 | // rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); 49 | // } 50 | // 51 | // The JSON representation for `Empty` is empty JSON object `{}`. 52 | message Empty {} 53 | -------------------------------------------------------------------------------- /proto/include/google/protobuf/source_context.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option java_package = "com.google.protobuf"; 37 | option java_outer_classname = "SourceContextProto"; 38 | option java_multiple_files = true; 39 | option objc_class_prefix = "GPB"; 40 | option go_package = "types"; 41 | 42 | // `SourceContext` represents information about the source of a 43 | // protobuf element, like the file in which it is defined. 44 | message SourceContext { 45 | // The path-qualified name of the .proto file that contained the associated 46 | // protobuf element. For example: `"google/protobuf/source_context.proto"`. 47 | string file_name = 1; 48 | } 49 | -------------------------------------------------------------------------------- /proto/include/google/protobuf/struct.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option cc_enable_arenas = true; 37 | option go_package = "types"; 38 | option java_package = "com.google.protobuf"; 39 | option java_outer_classname = "StructProto"; 40 | option java_multiple_files = true; 41 | option objc_class_prefix = "GPB"; 42 | 43 | 44 | // `Struct` represents a structured data value, consisting of fields 45 | // which map to dynamically typed values. In some languages, `Struct` 46 | // might be supported by a native representation. For example, in 47 | // scripting languages like JS a struct is represented as an 48 | // object. The details of that representation are described together 49 | // with the proto support for the language. 50 | // 51 | // The JSON representation for `Struct` is JSON object. 52 | message Struct { 53 | // Unordered map of dynamically typed values. 54 | map fields = 1; 55 | } 56 | 57 | // `Value` represents a dynamically typed value which can be either 58 | // null, a number, a string, a boolean, a recursive struct value, or a 59 | // list of values. A producer of value is expected to set one of that 60 | // variants, absence of any variant indicates an error. 61 | // 62 | // The JSON representation for `Value` is JSON value. 63 | message Value { 64 | // The kind of value. 65 | oneof kind { 66 | // Represents a null value. 67 | NullValue null_value = 1; 68 | // Represents a double value. 69 | double number_value = 2; 70 | // Represents a string value. 71 | string string_value = 3; 72 | // Represents a boolean value. 73 | bool bool_value = 4; 74 | // Represents a structured value. 75 | Struct struct_value = 5; 76 | // Represents a repeated `Value`. 77 | ListValue list_value = 6; 78 | } 79 | } 80 | 81 | // `NullValue` is a singleton enumeration to represent the null value for the 82 | // `Value` type union. 83 | // 84 | // The JSON representation for `NullValue` is JSON `null`. 85 | enum NullValue { 86 | // Null value. 87 | NULL_VALUE = 0; 88 | } 89 | 90 | // `ListValue` is a wrapper around a repeated field of values. 91 | // 92 | // The JSON representation for `ListValue` is JSON array. 93 | message ListValue { 94 | // Repeated field of dynamically typed values. 95 | repeated Value values = 1; 96 | } 97 | -------------------------------------------------------------------------------- /proto/include/google/protobuf/wrappers.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | // Wrappers for primitive (non-message) types. These types are useful 32 | // for embedding primitives in the `google.protobuf.Any` type and for places 33 | // where we need to distinguish between the absence of a primitive 34 | // typed field and its default value. 35 | 36 | syntax = "proto3"; 37 | 38 | package google.protobuf; 39 | 40 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 41 | option cc_enable_arenas = true; 42 | option go_package = "types"; 43 | option java_package = "com.google.protobuf"; 44 | option java_outer_classname = "WrappersProto"; 45 | option java_multiple_files = true; 46 | option objc_class_prefix = "GPB"; 47 | 48 | // Wrapper message for `double`. 49 | // 50 | // The JSON representation for `DoubleValue` is JSON number. 51 | message DoubleValue { 52 | // The double value. 53 | double value = 1; 54 | } 55 | 56 | // Wrapper message for `float`. 57 | // 58 | // The JSON representation for `FloatValue` is JSON number. 59 | message FloatValue { 60 | // The float value. 61 | float value = 1; 62 | } 63 | 64 | // Wrapper message for `int64`. 65 | // 66 | // The JSON representation for `Int64Value` is JSON string. 67 | message Int64Value { 68 | // The int64 value. 69 | int64 value = 1; 70 | } 71 | 72 | // Wrapper message for `uint64`. 73 | // 74 | // The JSON representation for `UInt64Value` is JSON string. 75 | message UInt64Value { 76 | // The uint64 value. 77 | uint64 value = 1; 78 | } 79 | 80 | // Wrapper message for `int32`. 81 | // 82 | // The JSON representation for `Int32Value` is JSON number. 83 | message Int32Value { 84 | // The int32 value. 85 | int32 value = 1; 86 | } 87 | 88 | // Wrapper message for `uint32`. 89 | // 90 | // The JSON representation for `UInt32Value` is JSON number. 91 | message UInt32Value { 92 | // The uint32 value. 93 | uint32 value = 1; 94 | } 95 | 96 | // Wrapper message for `bool`. 97 | // 98 | // The JSON representation for `BoolValue` is JSON `true` and `false`. 99 | message BoolValue { 100 | // The bool value. 101 | bool value = 1; 102 | } 103 | 104 | // Wrapper message for `string`. 105 | // 106 | // The JSON representation for `StringValue` is JSON string. 107 | message StringValue { 108 | // The string value. 109 | string value = 1; 110 | } 111 | 112 | // Wrapper message for `bytes`. 113 | // 114 | // The JSON representation for `BytesValue` is JSON string. 115 | message BytesValue { 116 | // The bytes value. 117 | bytes value = 1; 118 | } 119 | -------------------------------------------------------------------------------- /proto/include/rustproto.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | import "google/protobuf/descriptor.proto"; 4 | 5 | // see https://github.com/gogo/protobuf/blob/master/gogoproto/gogo.proto 6 | // for the original idea 7 | 8 | package rustproto; 9 | 10 | extend google.protobuf.FileOptions { 11 | // When true, oneof field is generated public 12 | optional bool expose_oneof_all = 17001; 13 | // When true all fields are public, and not accessors generated 14 | optional bool expose_fields_all = 17003; 15 | // When false, `get_`, `set_`, `mut_` etc. accessors are not generated 16 | optional bool generate_accessors_all = 17004; 17 | // Use `bytes::Bytes` for `bytes` fields 18 | optional bool carllerche_bytes_for_bytes_all = 17011; 19 | // Use `bytes::Bytes` for `string` fields 20 | optional bool carllerche_bytes_for_string_all = 17012; 21 | // When true, will only generate codes that works with lite runtime. 22 | optional bool lite_runtime_all = 17035; 23 | } 24 | 25 | extend google.protobuf.MessageOptions { 26 | // When true, oneof field is generated public 27 | optional bool expose_oneof = 17001; 28 | // When true all fields are public, and not accessors generated 29 | optional bool expose_fields = 17003; 30 | // When false, `get_`, `set_`, `mut_` etc. accessors are not generated 31 | optional bool generate_accessors = 17004; 32 | // Use `bytes::Bytes` for `bytes` fields 33 | optional bool carllerche_bytes_for_bytes = 17011; 34 | // Use `bytes::Bytes` for `string` fields 35 | optional bool carllerche_bytes_for_string = 17012; 36 | } 37 | 38 | extend google.protobuf.FieldOptions { 39 | // When true all fields are public, and not accessors generated 40 | optional bool expose_fields_field = 17003; 41 | // When false, `get_`, `set_`, `mut_` etc. accessors are not generated 42 | optional bool generate_accessors_field = 17004; 43 | // Use `bytes::Bytes` for `bytes` fields 44 | optional bool carllerche_bytes_for_bytes_field = 17011; 45 | // Use `bytes::Bytes` for `string` fields 46 | optional bool carllerche_bytes_for_string_field = 17012; 47 | } -------------------------------------------------------------------------------- /proto/keyspacepb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package keyspacepb; 3 | 4 | import "pdpb.proto"; 5 | 6 | import "gogoproto/gogo.proto"; 7 | import "rustproto.proto"; 8 | 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (rustproto.lite_runtime_all) = true; 13 | 14 | option java_package = "org.tikv.kvproto"; 15 | 16 | // Keyspace provides services to manage keyspaces. 17 | service Keyspace { 18 | rpc LoadKeyspace (LoadKeyspaceRequest) returns (LoadKeyspaceResponse) {} 19 | // WatchKeyspaces first return all current keyspaces' metadata as its first response. 20 | // Then, it returns responses containing keyspaces that had their metadata changed. 21 | rpc WatchKeyspaces (WatchKeyspacesRequest) returns (stream WatchKeyspacesResponse) {} 22 | rpc UpdateKeyspaceState(UpdateKeyspaceStateRequest) returns (UpdateKeyspaceStateResponse) {} 23 | rpc GetAllKeyspaces(GetAllKeyspacesRequest) returns (GetAllKeyspacesResponse) {} 24 | } 25 | 26 | message KeyspaceMeta { 27 | uint32 id = 1; 28 | string name = 2; 29 | KeyspaceState state = 3; 30 | int64 created_at = 4; 31 | int64 state_changed_at = 5; 32 | map config = 7; 33 | } 34 | 35 | enum KeyspaceState { 36 | ENABLED = 0; 37 | DISABLED = 1; 38 | ARCHIVED = 2; 39 | TOMBSTONE = 3; 40 | } 41 | 42 | message LoadKeyspaceRequest { 43 | pdpb.RequestHeader header = 1; 44 | string name = 2; 45 | } 46 | 47 | message LoadKeyspaceResponse { 48 | pdpb.ResponseHeader header = 1; 49 | KeyspaceMeta keyspace = 2; 50 | } 51 | 52 | message WatchKeyspacesRequest { 53 | pdpb.RequestHeader header = 1; 54 | } 55 | 56 | message WatchKeyspacesResponse { 57 | pdpb.ResponseHeader header = 1; 58 | repeated KeyspaceMeta keyspaces = 2; 59 | } 60 | 61 | message UpdateKeyspaceStateRequest{ 62 | pdpb.RequestHeader header = 1; 63 | uint32 id = 2; 64 | KeyspaceState state = 3; 65 | } 66 | 67 | message UpdateKeyspaceStateResponse{ 68 | pdpb.ResponseHeader header = 1; 69 | KeyspaceMeta keyspace = 2; 70 | } 71 | 72 | message GetAllKeyspacesRequest{ 73 | pdpb.RequestHeader header = 1; 74 | uint32 start_id = 2; 75 | uint32 limit = 3; 76 | } 77 | 78 | message GetAllKeyspacesResponse{ 79 | pdpb.ResponseHeader header = 1; 80 | repeated KeyspaceMeta keyspaces = 2; 81 | } 82 | -------------------------------------------------------------------------------- /proto/logbackuppb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package logbackup; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | import "errorpb.proto"; 7 | 8 | option (gogoproto.sizer_all) = true; 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (rustproto.lite_runtime_all) = true; 12 | 13 | option java_package = "org.tikv.kvproto"; 14 | 15 | // The minimal information for identify a region. 16 | message RegionIdentity { 17 | uint64 id = 1; 18 | uint64 epoch_version = 2; 19 | // We omitted epoch_conf_version because config change won't make range change. 20 | } 21 | 22 | // The last flush ts with region information. 23 | message RegionCheckpoint { 24 | errorpb.Error err = 1; 25 | RegionIdentity region = 2; 26 | uint64 checkpoint = 3; 27 | } 28 | 29 | message GetLastFlushTSOfRegionRequest { 30 | repeated RegionIdentity regions = 1; 31 | } 32 | 33 | message GetLastFlushTSOfRegionResponse { 34 | repeated RegionCheckpoint checkpoints = 1; 35 | } 36 | 37 | message SubscribeFlushEventRequest { 38 | string client_id = 1; 39 | } 40 | 41 | message SubscribeFlushEventResponse { 42 | repeated FlushEvent events = 1; 43 | } 44 | 45 | message FlushEvent { 46 | bytes start_key = 1; 47 | bytes end_key = 2; 48 | uint64 checkpoint = 3; 49 | } 50 | 51 | // The log backup service. 52 | // Generally, most essential interfaces of log backup (say, checkpoint management, task management) are 53 | // provided by adding some key in the embed etcd of PD. 54 | // This interface is mainly provided for the checkpoint advancer and debug usage. 55 | service LogBackup { 56 | rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {} 57 | rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {} 58 | } 59 | -------------------------------------------------------------------------------- /proto/meta_storagepb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package meta_storagepb; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | 7 | option (gogoproto.sizer_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option (rustproto.lite_runtime_all) = true; 11 | 12 | option java_package = "org.tikv.kvproto"; 13 | 14 | // MetaStorage is the meta storage service. 15 | service MetaStorage { 16 | rpc Watch(WatchRequest) returns (stream WatchResponse) {} 17 | 18 | // Get is the same as etcd Range which might be implemented in a more common way 19 | // so that we can use other storages to replace etcd in the future. 20 | rpc Get(GetRequest) returns (GetResponse) {} 21 | 22 | rpc Put(PutRequest) returns (PutResponse) {} 23 | } 24 | 25 | enum ErrorType { 26 | OK = 0; 27 | UNKNOWN = 1; 28 | // required watch revision is smaller than current compact/min revision. 29 | DATA_COMPACTED = 2; 30 | } 31 | 32 | message Error { 33 | ErrorType type = 1; 34 | string message = 2; 35 | } 36 | 37 | message RequestHeader { 38 | // cluster_id is the ID of the cluster which be sent to. 39 | uint64 cluster_id = 1; 40 | // source is the source of the request. 41 | string source = 2; 42 | } 43 | 44 | message ResponseHeader { 45 | // cluster_id is the ID of the cluster which sent the response. 46 | uint64 cluster_id = 1; 47 | Error error = 2; 48 | int64 revision = 3; 49 | } 50 | 51 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 52 | message WatchRequest { 53 | RequestHeader header = 1; 54 | bytes key = 2; 55 | bytes range_end = 3; 56 | int64 start_revision = 4; 57 | bool prev_kv = 5; 58 | } 59 | 60 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 61 | message WatchResponse { 62 | ResponseHeader header = 1; 63 | int64 compact_revision = 2; 64 | repeated Event events = 3; 65 | } 66 | 67 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 68 | message GetRequest { 69 | RequestHeader header = 1; 70 | bytes key = 2; 71 | bytes range_end = 3; 72 | int64 limit = 4; 73 | int64 revision = 5; 74 | } 75 | 76 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 77 | message GetResponse { 78 | ResponseHeader header = 1; 79 | repeated KeyValue kvs = 2; 80 | bool more = 3; 81 | int64 count = 4; 82 | } 83 | 84 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 85 | message PutRequest { 86 | RequestHeader header = 1; 87 | bytes key = 2; 88 | bytes value = 3; 89 | int64 lease = 4; 90 | bool prev_kv = 5; 91 | } 92 | 93 | // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto 94 | message PutResponse { 95 | ResponseHeader header = 1; 96 | KeyValue prev_kv = 2; 97 | } 98 | 99 | // copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto 100 | message KeyValue { 101 | // key is the key in bytes. An empty key is not allowed. 102 | bytes key = 1; 103 | // create_revision is the revision of last creation on this key. 104 | int64 create_revision = 2; 105 | // mod_revision is the revision of last modification on this key. 106 | int64 mod_revision = 3; 107 | // version is the version of the key. A deletion resets 108 | // the version to zero and any modification of the key 109 | // increases its version. 110 | int64 version = 4; 111 | // value is the value held by the key, in bytes. 112 | bytes value = 5; 113 | // lease is the ID of the lease that attached to key. 114 | // When the attached lease expires, the key will be deleted. 115 | // If lease is 0, then no lease is attached to the key. 116 | int64 lease = 6; 117 | } 118 | 119 | // copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto 120 | message Event { 121 | enum EventType { 122 | PUT = 0; 123 | DELETE = 1; 124 | } 125 | // type is the kind of event. If type is a PUT, it indicates 126 | // new data has been stored to the key. If type is a DELETE, 127 | // it indicates the key was deleted. 128 | EventType type = 1; 129 | // kv holds the KeyValue for the event. 130 | // A PUT event contains current kv pair. 131 | // A PUT event with kv.Version=1 indicates the creation of a key. 132 | // A DELETE/EXPIRE event contains the deleted key with 133 | // its modification revision set to the revision of deletion. 134 | KeyValue kv = 2; 135 | 136 | // prev_kv holds the key-value pair before the event happens. 137 | KeyValue prev_kv = 3; 138 | } 139 | -------------------------------------------------------------------------------- /proto/metapb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package metapb; 3 | 4 | import "encryptionpb.proto"; 5 | import "gogoproto/gogo.proto"; 6 | import "rustproto.proto"; 7 | 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (rustproto.lite_runtime_all) = true; 12 | 13 | option java_package = "org.tikv.kvproto"; 14 | 15 | message Cluster { 16 | uint64 id = 1; 17 | // max peer count for a region. 18 | // pd will do the auto-balance if region peer count mismatches. 19 | uint32 max_peer_count = 2; 20 | // more attributes...... 21 | } 22 | 23 | enum StoreState { 24 | Up = 0; 25 | Offline = 1; 26 | Tombstone = 2; 27 | } 28 | 29 | // NodeState is going to replace StoreState to make the state concept more clear. 30 | // "Up" is devided into "Preparing" and "Serving" stages so that we can better describe the online process. 31 | // "Removing" is just like previous `Offline` which is more accurate. 32 | // "Removed" has the same meaning with `Tombstone`. 33 | enum NodeState { 34 | Preparing = 0; 35 | Serving = 1; 36 | Removing = 2; 37 | Removed = 3; 38 | } 39 | 40 | // Case insensitive key/value for replica constraints. 41 | message StoreLabel { 42 | string key = 1; 43 | string value = 2; 44 | } 45 | 46 | message Store { 47 | uint64 id = 1; 48 | // Address to handle client requests (kv, cop, etc.) 49 | string address = 2; 50 | StoreState state = 3; 51 | repeated StoreLabel labels = 4; 52 | string version = 5; 53 | // Address to handle peer requests (raft messages from other store). 54 | // Empty means same as address. 55 | string peer_address = 6; 56 | // Status address provides the HTTP service for external components 57 | string status_address = 7; 58 | string git_hash = 8; 59 | // The start timestamp of the current store 60 | int64 start_timestamp = 9; 61 | string deploy_path = 10; 62 | // The last heartbeat timestamp of the store. 63 | int64 last_heartbeat = 11; 64 | // If the store is physically destroyed, which means it can never up again. 65 | bool physically_destroyed = 12; 66 | // NodeState is used to replace StoreState which will be deprecated in the future. 67 | NodeState node_state = 13; 68 | } 69 | 70 | message RegionEpoch { 71 | // Conf change version, auto increment when add or remove peer 72 | uint64 conf_ver = 1; 73 | // Region version, auto increment when split or merge 74 | uint64 version = 2; 75 | } 76 | 77 | message BucketStats { 78 | // total read in bytes of each bucket 79 | repeated uint64 read_bytes = 1; 80 | 81 | // total write in bytes of each bucket 82 | repeated uint64 write_bytes = 2; 83 | 84 | // total read qps of each bucket 85 | repeated uint64 read_qps = 3; 86 | 87 | // total write qps of each bucket 88 | repeated uint64 write_qps = 4; 89 | 90 | // total read keys of each bucket 91 | repeated uint64 read_keys = 5; 92 | 93 | // total write keys of each bucket 94 | repeated uint64 write_keys = 6; 95 | } 96 | 97 | message Buckets { 98 | uint64 region_id = 1; 99 | 100 | // A hint indicate if keys have changed. 101 | uint64 version = 2; 102 | 103 | // keys of buckets, include start/end key of region 104 | repeated bytes keys = 3; 105 | 106 | // bucket stats 107 | BucketStats stats = 4; 108 | 109 | // The period in milliseconds that stats are collected with in 110 | uint64 period_in_ms = 5; 111 | } 112 | 113 | message Region { 114 | uint64 id = 1; 115 | // Region key range [start_key, end_key). 116 | bytes start_key = 2; 117 | bytes end_key = 3; 118 | RegionEpoch region_epoch = 4; 119 | repeated Peer peers = 5; 120 | // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. 121 | // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). 122 | // The field is only used by PD and should be ignored otherwise. 123 | // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. 124 | encryptionpb.EncryptionMeta encryption_meta = 6; 125 | // The flashback state indicates whether this region is in the flashback state. 126 | // TODO: only check by `flashback_start_ts` in the future. Keep for compatibility now. 127 | bool is_in_flashback = 7; 128 | // The start_ts that the current flashback progress is using. 129 | uint64 flashback_start_ts = 8; 130 | } 131 | 132 | enum PeerRole { 133 | // Voter -> Voter 134 | Voter = 0; 135 | // Learner/None -> Learner 136 | Learner = 1; 137 | // Learner/None -> Voter 138 | IncomingVoter = 2; 139 | // Voter -> Learner 140 | DemotingVoter = 3; 141 | // We forbid Voter -> None, it can introduce unavailability as discussed in 142 | // etcd-io/etcd#7625 143 | // Learner -> None can be apply directly, doesn't need to be stored as 144 | // joint state. 145 | } 146 | 147 | message Peer { 148 | uint64 id = 1; 149 | uint64 store_id = 2; 150 | PeerRole role = 3; 151 | bool is_witness = 4; 152 | } 153 | -------------------------------------------------------------------------------- /proto/mpp.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package mpp; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "coprocessor.proto"; 6 | import "metapb.proto"; 7 | import "kvrpcpb.proto"; 8 | 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.sizer_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | 13 | option java_package = "org.tikv.kvproto"; 14 | 15 | // TaskMeta contains meta of a mpp plan, including query's ts and task address. 16 | message TaskMeta { 17 | uint64 start_ts = 1; // start ts of a query 18 | int64 task_id = 2; // if task id is -1 , it indicates a tidb task. 19 | int64 partition_id = 3; // Only used for hash partition 20 | string address = 4; // target address of this task. 21 | uint64 gather_id = 5; // used to distinguish different gathers in the mpp query. 22 | uint64 query_ts = 6; // timestamp when start to execute query, used for TiFlash miniTSO schedule. 23 | uint64 local_query_id = 7; // unique local query_id if tidb don't restart. So we can use gather_id + query_ts + local_query_id + server_id to represent a global unique query. 24 | uint64 server_id = 8; // TiDB server id 25 | int64 mpp_version = 9; // mpp version 26 | uint32 keyspace_id = 10; // keyspace id of the request 27 | string coordinator_address = 11; // coordinator_address of this query 28 | bool report_execution_summary = 12; // Only when coordinator_address is not empty, this flag can be true. When set to true, TiFlash only report execution summary through ReportMPPTaskStatus service, don't include summaries in MppDataPacket 29 | kvrpcpb.APIVersion api_version = 16; // API version of the request 30 | string resource_group_name = 17; 31 | uint64 connection_id = 18; // This is the session id between a client and tidb 32 | string connection_alias = 19; // This is the session alias between a client and tidb 33 | } 34 | 35 | message IsAliveRequest { 36 | } 37 | 38 | message IsAliveResponse { 39 | bool available = 1; 40 | int64 mpp_version = 2; 41 | } 42 | 43 | // Dipsatch the task request to different tiflash servers. 44 | message DispatchTaskRequest { 45 | TaskMeta meta = 1; 46 | bytes encoded_plan = 2; 47 | int64 timeout = 3; 48 | repeated coprocessor.RegionInfo regions = 4; 49 | // If this task contains table scan, we still need their region info. 50 | int64 schema_ver = 5; 51 | // Used for partition table scan 52 | repeated coprocessor.TableRegions table_regions = 6; 53 | } 54 | 55 | // Get response of DispatchTaskRequest. 56 | message DispatchTaskResponse { 57 | Error error = 1; 58 | repeated metapb.Region retry_regions = 2; 59 | } 60 | 61 | // CancelTaskRequest closes the execution of a task. 62 | message CancelTaskRequest { 63 | TaskMeta meta = 1; 64 | Error error = 2; 65 | } 66 | 67 | message CancelTaskResponse { 68 | Error error = 1; 69 | } 70 | 71 | // ReportTaskStatus reports the execution status of a task. 72 | // when TiFlash reports status to TiDB, ReportTaskStatusRequest serialize tipb.TiFlashExecutionInfo into data; 73 | message ReportTaskStatusRequest { 74 | TaskMeta meta = 1; 75 | bytes data = 2; 76 | Error error = 3; 77 | } 78 | 79 | message ReportTaskStatusResponse { 80 | Error error = 1; 81 | } 82 | 83 | // build connection between different tasks. Data is sent by the tasks that are closer to the data sources. 84 | message EstablishMPPConnectionRequest { 85 | TaskMeta sender_meta = 1; // node closer to the source 86 | TaskMeta receiver_meta = 2; // node closer to the tidb mpp gather. 87 | } 88 | 89 | // when TiFlash sends data to TiDB, Data packets wrap tipb.SelectResponse, i.e., serialize tipb.SelectResponse into data; 90 | // when TiFlash sends data to TiFlash, data blocks are serialized into chunks, and the execution_summaries in tipb.SelectResponse are serialized into data only for the last packet. 91 | message MPPDataPacket { 92 | bytes data = 1; 93 | Error error = 2; 94 | repeated bytes chunks = 3; 95 | repeated uint64 stream_ids = 4; 96 | int64 version = 5; // version of data packet format 97 | } 98 | 99 | message Error { 100 | int32 code = 1; 101 | string msg = 2; 102 | int64 mpp_version = 3; 103 | } 104 | -------------------------------------------------------------------------------- /proto/recoverdatapb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package recover_data; 3 | 4 | import "gogoproto/gogo.proto"; 5 | import "rustproto.proto"; 6 | 7 | option (gogoproto.sizer_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.unmarshaler_all) = true; 10 | option (rustproto.lite_runtime_all) = true; 11 | 12 | // request to read region meata from a store 13 | message ReadRegionMetaRequest { 14 | uint64 store_id = 1; 15 | } 16 | 17 | message Error { 18 | string msg = 1; 19 | } 20 | 21 | message RegionMeta { 22 | uint64 region_id = 1; 23 | uint64 peer_id = 2; 24 | uint64 last_log_term = 3; 25 | uint64 last_index = 4; 26 | uint64 commit_index = 5; 27 | uint64 version = 6; 28 | bool tombstone = 7; //reserved, it may be used in late phase for peer check 29 | bytes start_key = 8; 30 | bytes end_key = 9; 31 | } 32 | 33 | // command to store for recover region 34 | message RecoverRegionRequest { 35 | uint64 region_id = 1; 36 | bool as_leader = 2; // force region_id as leader 37 | bool tombstone = 3; // set Peer to tombstoned in late phase 38 | } 39 | 40 | message RecoverRegionResponse { 41 | Error error = 1; 42 | uint64 store_id = 2; 43 | } 44 | 45 | // wait apply to last index 46 | message WaitApplyRequest { 47 | uint64 store_id = 1; 48 | } 49 | 50 | message WaitApplyResponse { 51 | Error error = 1; 52 | } 53 | 54 | // resolve data by resolved_ts 55 | message ResolveKvDataRequest { 56 | uint64 resolved_ts = 1; 57 | } 58 | 59 | message ResolveKvDataResponse { 60 | Error error = 1; 61 | uint64 store_id = 2; 62 | uint64 resolved_key_count = 3; // reserved for summary of restore 63 | // cursor of delete key.commit_ts, reserved for progress of restore 64 | // progress is (current_commit_ts - resolved_ts) / (backup_ts - resolved_ts) x 100% 65 | uint64 current_commit_ts = 4; 66 | } 67 | 68 | // a recovery workflow likes 69 | // 1. BR read ReadRegionMeta to get all region meta 70 | // 2. BR send recover region to tikv, e.g assign leader and wait leader apply to last index 71 | // 3. BR wait all regions in tikv to apply to last index (no write during the recovery) 72 | // 4. BR resolved kv data 73 | service RecoverData { 74 | // read region meta to ready region meta 75 | rpc ReadRegionMeta(ReadRegionMetaRequest) returns (stream RegionMeta) {} 76 | // execute the recovery command 77 | rpc RecoverRegion(stream RecoverRegionRequest) returns (RecoverRegionResponse) {} 78 | // wait all region apply to last index 79 | rpc WaitApply(WaitApplyRequest) returns (WaitApplyResponse) {} 80 | // execute delete data from kv db 81 | rpc ResolveKvData(ResolveKvDataRequest) returns (stream ResolveKvDataResponse) {} 82 | } -------------------------------------------------------------------------------- /proto/replication_modepb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package replication_modepb; 3 | 4 | enum ReplicationMode { 5 | // The standard mode. Replicate logs to majority peer. 6 | MAJORITY = 0; 7 | // DR mode. Replicate logs among 2 DCs. 8 | DR_AUTO_SYNC = 1; 9 | } 10 | 11 | // The replication status sync from PD to TiKV. 12 | message ReplicationStatus { 13 | ReplicationMode mode = 1; 14 | DRAutoSync dr_auto_sync = 2; 15 | } 16 | 17 | enum DRAutoSyncState { 18 | // Raft logs need to sync between different DCs 19 | SYNC = 0; 20 | // Wait for switching to ASYNC. Stop sync raft logs between DCs. 21 | ASYNC_WAIT = 1; 22 | // Raft logs need to sync to majority peers 23 | ASYNC = 2; 24 | // Switching from ASYNC to SYNC mode 25 | SYNC_RECOVER = 3; 26 | } 27 | 28 | // The status of dr-autosync mode. 29 | message DRAutoSync { 30 | // The key of the label that used for distinguish different DC. 31 | string label_key = 1; 32 | DRAutoSyncState state = 2; 33 | // Unique ID of the state, it increases after each state transfer. 34 | uint64 state_id = 3; 35 | // Duration to wait before switching to SYNC by force (in seconds) 36 | int32 wait_sync_timeout_hint = 4; 37 | // Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. 38 | repeated uint64 available_stores = 5; 39 | // Stores should forbid region split. 40 | bool pause_region_split = 6; 41 | } 42 | 43 | enum RegionReplicationState { 44 | // The region's state is unknown 45 | UNKNOWN = 0; 46 | // Logs sync to majority peers 47 | SIMPLE_MAJORITY = 1; 48 | // Logs sync to different DCs 49 | INTEGRITY_OVER_LABEL = 2; 50 | } 51 | 52 | // The replication status sync from TiKV to PD. 53 | message RegionReplicationStatus { 54 | RegionReplicationState state = 1; 55 | // Unique ID of the state, it increases after each state transfer. 56 | uint64 state_id = 2; 57 | } 58 | 59 | message StoreDRAutoSyncStatus { 60 | DRAutoSyncState state = 1; 61 | uint64 state_id = 2; 62 | } 63 | -------------------------------------------------------------------------------- /proto/resource_usage_agent.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package resource_usage_agent; 4 | 5 | import "gogoproto/gogo.proto"; 6 | import "rustproto.proto"; 7 | 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (rustproto.lite_runtime_all) = true; 12 | 13 | option java_package = "org.tikv.kvproto"; 14 | 15 | // ResourceUsageAgent is the service for storing resource usage records. 16 | service ResourceUsageAgent { 17 | // Report the resource usage records. By default, the records with the same 18 | // resource group tag will be batched by minute. 19 | rpc Report(stream ResourceUsageRecord) returns (EmptyResponse) {} 20 | } 21 | 22 | // TiKV implements ResourceMeteringPubSub service for clients to subscribe to resource metering records. 23 | service ResourceMeteringPubSub { 24 | // Clients subscribe to resource metering records through this RPC, and TiKV periodically (e.g. per minute) 25 | // publishes resource metering records to clients via gRPC stream. 26 | rpc Subscribe(ResourceMeteringRequest) returns (stream ResourceUsageRecord) {} 27 | } 28 | 29 | message ResourceMeteringRequest {} 30 | 31 | message EmptyResponse {} 32 | 33 | message ResourceUsageRecord { 34 | oneof record_oneof { 35 | GroupTagRecord record = 1; 36 | } 37 | } 38 | 39 | // GroupTagRecord is a set of resource usage data grouped by resource_group_tag. 40 | message GroupTagRecord { 41 | bytes resource_group_tag = 1; 42 | repeated GroupTagRecordItem items = 2; 43 | } 44 | 45 | message GroupTagRecordItem { 46 | uint64 timestamp_sec = 1; 47 | uint32 cpu_time_ms = 2; 48 | uint32 read_keys = 3; 49 | uint32 write_keys = 4; 50 | } 51 | -------------------------------------------------------------------------------- /proto/tracepb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package tracepb; 3 | 4 | option java_package = "org.tikv.kvproto"; 5 | 6 | service TraceRecordPubSub { 7 | // Subscribe the Trace records generated on this service. The service will periodically (e.g. per minute) 8 | // publishes Trace records to clients via gRPC stream. 9 | rpc Subscribe(TraceRecordRequest) returns (stream TraceRecord) {} 10 | } 11 | 12 | message TraceRecordRequest {} 13 | 14 | message TraceRecord { 15 | oneof record_oneof { 16 | Report report = 1; 17 | NotifyCollect notify_collect = 2; 18 | } 19 | } 20 | 21 | message RemoteParentSpan { 22 | // A unique id to identify the request. It's usually a UUID. 23 | uint64 trace_id = 1; 24 | // The span of remote caller that is awaiting the request. 25 | uint64 span_id = 2; 26 | } 27 | 28 | // The context of the request to be traced. 29 | message TraceContext { 30 | repeated RemoteParentSpan remote_parent_spans = 1; 31 | // Report the trace records only if the duration of handling the request exceeds the threshold. 32 | uint32 duration_threshold_ms = 2; 33 | } 34 | 35 | // Report the spans collected when handling a request on a service. 36 | message Report { 37 | repeated RemoteParentSpan remote_parent_spans = 1; 38 | repeated Span spans = 2; 39 | } 40 | 41 | // Notify the subscriber to persis the spans of the trace. 42 | message NotifyCollect { 43 | uint64 trace_id = 1; 44 | } 45 | 46 | message Span { 47 | // The unique span id within the spans with the same `trace_id`. 48 | // The most significant 32 bits should be random number generated by each service instance. 49 | uint64 span_id = 1; 50 | uint64 parent_id = 2; 51 | uint64 begin_unix_ns = 3; 52 | uint64 duration_ns = 4; 53 | string event = 5; 54 | repeated Property properties = 6; 55 | } 56 | 57 | message Property { 58 | string key = 1; 59 | string value = 2; 60 | } 61 | -------------------------------------------------------------------------------- /proto/tsopb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package tsopb; 3 | 4 | import "pdpb.proto"; 5 | 6 | import "gogoproto/gogo.proto"; 7 | import "rustproto.proto"; 8 | 9 | option (gogoproto.sizer_all) = true; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (rustproto.lite_runtime_all) = true; 13 | 14 | option java_package = "org.tikv.kvproto"; 15 | 16 | service TSO { 17 | rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} 18 | // Find the keyspace group that the keyspace belongs to by keyspace id. 19 | rpc FindGroupByKeyspaceID (FindGroupByKeyspaceIDRequest) returns (FindGroupByKeyspaceIDResponse) {} 20 | // Get the minimum timestamp across all keyspace groups served by the TSO server who receives 21 | // and handle the request. If the TSO server/pod is not serving any keyspace group, return 22 | // an empty timestamp, and the client needs to skip the empty timestamps when collecting 23 | // the min timestamp from all TSO servers/pods. 24 | rpc GetMinTS (GetMinTSRequest) returns (GetMinTSResponse) {} 25 | } 26 | 27 | message RequestHeader { 28 | // cluster_id is the ID of the cluster which be sent to. 29 | uint64 cluster_id = 1; 30 | // sender_id is the ID of the sender server. 31 | uint64 sender_id = 2; 32 | 33 | // keyspace_id is the unique id of the tenant/keyspace. 34 | uint32 keyspace_id = 3; 35 | // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. 36 | uint32 keyspace_group_id = 4; 37 | } 38 | 39 | message ResponseHeader { 40 | // cluster_id is the ID of the cluster which sent the response. 41 | uint64 cluster_id = 1; 42 | Error error = 2; 43 | 44 | // keyspace_id is the unique id of the tenant/keyspace as the response receiver. 45 | uint32 keyspace_id = 3; 46 | // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. 47 | uint32 keyspace_group_id = 4; 48 | } 49 | 50 | enum ErrorType { 51 | OK = 0; 52 | UNKNOWN = 1; 53 | NOT_BOOTSTRAPPED = 2; 54 | ALREADY_BOOTSTRAPPED = 3; 55 | INVALID_VALUE = 4; 56 | CLUSTER_MISMATCHED = 5; 57 | } 58 | 59 | message Error { 60 | ErrorType type = 1; 61 | string message = 2; 62 | } 63 | 64 | message TsoRequest { 65 | RequestHeader header = 1; 66 | 67 | uint32 count = 2; 68 | string dc_location = 3; 69 | } 70 | 71 | message TsoResponse { 72 | ResponseHeader header = 1; 73 | 74 | uint32 count = 2; 75 | pdpb.Timestamp timestamp = 3; 76 | } 77 | 78 | message Participant { 79 | // name is the unique name of the TSO participant. 80 | string name = 1; 81 | // id is the unique id of the TSO participant. 82 | uint64 id = 2; 83 | // listen_urls is the serivce endpoint list in the url format. 84 | // listen_urls[0] is primary service endpoint. 85 | repeated string listen_urls = 3; 86 | } 87 | 88 | message KeyspaceGroupMember { 89 | string address = 1; 90 | bool is_primary = 2; 91 | } 92 | 93 | message SplitState { 94 | uint32 split_source = 1; 95 | } 96 | 97 | message KeyspaceGroup { 98 | uint32 id = 1; 99 | string user_kind = 2; 100 | SplitState split_state = 3; 101 | repeated KeyspaceGroupMember members = 4; 102 | } 103 | 104 | message FindGroupByKeyspaceIDRequest { 105 | RequestHeader header = 1; 106 | uint32 keyspace_id = 2; 107 | } 108 | 109 | message FindGroupByKeyspaceIDResponse { 110 | ResponseHeader header = 1; 111 | KeyspaceGroup keyspace_group = 2; 112 | } 113 | 114 | message GetMinTSRequest { 115 | RequestHeader header = 1; 116 | string dc_location = 2; 117 | } 118 | 119 | message GetMinTSResponse { 120 | ResponseHeader header = 1; 121 | pdpb.Timestamp timestamp = 2; 122 | // the count of keyspace group primaries that the TSO server/pod is serving 123 | uint32 keyspace_groups_serving = 3; 124 | // the total count of keyspace groups 125 | uint32 keyspace_groups_total = 4; 126 | } 127 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.84.1" 3 | components = ["rustfmt", "clippy"] 4 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | reorder_imports = true 3 | 4 | # Those options are disabled for not supported in stable rust. 5 | # We should enable them back once stabilized. 6 | # 7 | # version = "Two" 8 | # imports_granularity = "Item" 9 | # group_imports = "StdExternalCrate" 10 | # where_single_line = true 11 | # trailing_comma = "Vertical" 12 | # overflow_delimited_expr = true 13 | # format_code_in_doc_comments = true 14 | # normalize_comments = true 15 | -------------------------------------------------------------------------------- /src/common/errors.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::result; 4 | 5 | use thiserror::Error; 6 | 7 | use crate::proto::kvrpcpb; 8 | use crate::region::RegionVerId; 9 | use crate::BoundRange; 10 | 11 | /// An error originating from the TiKV client or dependencies. 12 | #[derive(Debug, Error)] 13 | #[allow(clippy::large_enum_variant)] 14 | pub enum Error { 15 | /// Feature is not implemented. 16 | #[error("Unimplemented feature")] 17 | Unimplemented, 18 | /// Duplicate key insertion happens. 19 | #[error("Duplicate key insertion")] 20 | DuplicateKeyInsertion, 21 | /// Failed to resolve a lock 22 | #[error("Failed to resolve lock")] 23 | ResolveLockError(Vec), 24 | /// Will raise this error when using a pessimistic txn only operation on an optimistic txn 25 | #[error("Invalid operation for this type of transaction")] 26 | InvalidTransactionType, 27 | /// It's not allowed to perform operations in a transaction after it has been committed or rolled back. 28 | #[error("Cannot read or write data after any attempt to commit or roll back the transaction")] 29 | OperationAfterCommitError, 30 | /// We tried to use 1pc for a transaction, but it didn't work. Probably should have used 2pc. 31 | #[error("1PC transaction could not be committed.")] 32 | OnePcFailure, 33 | /// An operation requires a primary key, but the transaction was empty. 34 | #[error("transaction has no primary key")] 35 | NoPrimaryKey, 36 | /// For raw client, operation is not supported in atomic/non-atomic mode. 37 | #[error( 38 | "The operation is not supported in current mode, please consider using RawClient with or without atomic mode" 39 | )] 40 | UnsupportedMode, 41 | #[error("There is no current_regions in the EpochNotMatch error")] 42 | NoCurrentRegions, 43 | #[error("The specified entry is not found in the region cache")] 44 | EntryNotFoundInRegionCache, 45 | /// Wraps a `std::io::Error`. 46 | #[error("IO error: {0}")] 47 | Io(#[from] std::io::Error), 48 | /// Wraps a `std::io::Error`. 49 | #[error("tokio channel error: {0}")] 50 | Channel(#[from] tokio::sync::oneshot::error::RecvError), 51 | /// Wraps a `grpcio::Error`. 52 | #[error("gRPC error: {0}")] 53 | Grpc(#[from] tonic::transport::Error), 54 | /// Wraps a `reqwest::Error`. 55 | /// Wraps a `grpcio::Error`. 56 | #[error("gRPC api error: {0}")] 57 | GrpcAPI(#[from] tonic::Status), 58 | /// Wraps a `grpcio::Error`. 59 | #[error("url error: {0}")] 60 | Url(#[from] tonic::codegen::http::uri::InvalidUri), 61 | /// Represents that a futures oneshot channel was cancelled. 62 | #[error("A futures oneshot channel was canceled. {0}")] 63 | Canceled(#[from] futures::channel::oneshot::Canceled), 64 | /// Errors caused by changes of region information 65 | #[error("Region error: {0:?}")] 66 | RegionError(Box), 67 | /// Whether the transaction is committed or not is undetermined 68 | #[error("Whether the transaction is committed or not is undetermined")] 69 | UndeterminedError(Box), 70 | /// Wraps `crate::proto::kvrpcpb::KeyError` 71 | #[error("{0:?}")] 72 | KeyError(Box), 73 | /// Multiple errors generated from the ExtractError plan. 74 | #[error("Multiple errors: {0:?}")] 75 | ExtractedErrors(Vec), 76 | /// Multiple key errors 77 | #[error("Multiple key errors: {0:?}")] 78 | MultipleKeyErrors(Vec), 79 | /// Invalid ColumnFamily 80 | #[error("Unsupported column family {}", _0)] 81 | ColumnFamilyError(String), 82 | /// Can't join tokio tasks 83 | #[error("Failed to join tokio tasks")] 84 | JoinError(#[from] tokio::task::JoinError), 85 | /// No region is found for the given key. 86 | #[error("Region is not found for key: {:?}", key)] 87 | RegionForKeyNotFound { key: Vec }, 88 | #[error("Region is not found for range: {:?}", range)] 89 | RegionForRangeNotFound { range: BoundRange }, 90 | /// No region is found for the given id. note: distinguish it with the RegionNotFound error in errorpb. 91 | #[error("Region {} is not found in the response", region_id)] 92 | RegionNotFoundInResponse { region_id: u64 }, 93 | /// No leader is found for the given id. 94 | #[error("Leader of region {} is not found", region.id)] 95 | LeaderNotFound { region: RegionVerId }, 96 | /// Scan limit exceeds the maximum 97 | #[error("Limit {} exceeds max scan limit {}", limit, max_limit)] 98 | MaxScanLimitExceeded { limit: u32, max_limit: u32 }, 99 | #[error("Invalid Semver string: {0:?}")] 100 | InvalidSemver(#[from] semver::Error), 101 | /// A string error returned by TiKV server 102 | #[error("Kv error. {}", message)] 103 | KvError { message: String }, 104 | #[error("{}", message)] 105 | InternalError { message: String }, 106 | #[error("{0}")] 107 | StringError(String), 108 | #[error("PessimisticLock error: {:?}", inner)] 109 | PessimisticLockError { 110 | inner: Box, 111 | success_keys: Vec>, 112 | }, 113 | #[error("Keyspace not found: {0}")] 114 | KeyspaceNotFound(String), 115 | } 116 | 117 | impl From for Error { 118 | fn from(e: crate::proto::errorpb::Error) -> Error { 119 | Error::RegionError(Box::new(e)) 120 | } 121 | } 122 | 123 | impl From for Error { 124 | fn from(e: crate::proto::kvrpcpb::KeyError) -> Error { 125 | Error::KeyError(Box::new(e)) 126 | } 127 | } 128 | 129 | /// A result holding an [`Error`](enum@Error). 130 | pub type Result = result::Result; 131 | 132 | #[doc(hidden)] 133 | #[macro_export] 134 | macro_rules! internal_err { 135 | ($e:expr) => ({ 136 | $crate::Error::InternalError { 137 | message: format!("[{}:{}]: {}", file!(), line!(), $e) 138 | } 139 | }); 140 | ($f:tt, $($arg:expr),+) => ({ 141 | internal_err!(format!($f, $($arg),+)) 142 | }); 143 | } 144 | -------------------------------------------------------------------------------- /src/common/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | mod errors; 4 | pub mod security; 5 | 6 | pub use self::errors::Error; 7 | pub use self::errors::Result; 8 | -------------------------------------------------------------------------------- /src/common/security.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::fs::File; 4 | use std::io::Read; 5 | use std::path::Path; 6 | use std::path::PathBuf; 7 | use std::time::Duration; 8 | 9 | use log::info; 10 | use regex::Regex; 11 | use tonic::transport::Channel; 12 | use tonic::transport::ClientTlsConfig; 13 | use tonic::transport::Identity; 14 | use tonic::transport::{Certificate, Endpoint}; 15 | 16 | use crate::internal_err; 17 | use crate::Result; 18 | 19 | lazy_static::lazy_static! { 20 | static ref SCHEME_REG: Regex = Regex::new(r"^\s*(https?://)").unwrap(); 21 | } 22 | 23 | fn check_pem_file(tag: &str, path: &Path) -> Result { 24 | File::open(path) 25 | .map_err(|e| internal_err!("failed to open {} to load {}: {:?}", path.display(), tag, e)) 26 | } 27 | 28 | fn load_pem_file(tag: &str, path: &Path) -> Result> { 29 | let mut file = check_pem_file(tag, path)?; 30 | let mut key = vec![]; 31 | file.read_to_end(&mut key) 32 | .map_err(|e| { 33 | internal_err!( 34 | "failed to load {} from path {}: {:?}", 35 | tag, 36 | path.display(), 37 | e 38 | ) 39 | }) 40 | .map(|_| key) 41 | } 42 | 43 | /// Manages the TLS protocol 44 | #[derive(Default)] 45 | pub struct SecurityManager { 46 | /// The PEM encoding of the server’s CA certificates. 47 | ca: Vec, 48 | /// The PEM encoding of the server’s certificate chain. 49 | cert: Vec, 50 | /// The path to the file that contains the PEM encoding of the server’s private key. 51 | key: PathBuf, 52 | } 53 | 54 | impl SecurityManager { 55 | /// Load TLS configuration from files. 56 | pub fn load( 57 | ca_path: impl AsRef, 58 | cert_path: impl AsRef, 59 | key_path: impl Into, 60 | ) -> Result { 61 | let key_path = key_path.into(); 62 | check_pem_file("private key", &key_path)?; 63 | Ok(SecurityManager { 64 | ca: load_pem_file("ca", ca_path.as_ref())?, 65 | cert: load_pem_file("certificate", cert_path.as_ref())?, 66 | key: key_path, 67 | }) 68 | } 69 | 70 | /// Connect to gRPC server using TLS connection. If TLS is not configured, use normal connection. 71 | pub async fn connect( 72 | &self, 73 | // env: Arc, 74 | addr: &str, 75 | factory: Factory, 76 | ) -> Result 77 | where 78 | Factory: FnOnce(Channel) -> Client, 79 | { 80 | info!("connect to rpc server at endpoint: {:?}", addr); 81 | let channel = if !self.ca.is_empty() { 82 | self.tls_channel(addr).await? 83 | } else { 84 | self.default_channel(addr).await? 85 | }; 86 | let ch = channel.connect().await?; 87 | 88 | Ok(factory(ch)) 89 | } 90 | 91 | async fn tls_channel(&self, addr: &str) -> Result { 92 | let addr = "https://".to_string() + &SCHEME_REG.replace(addr, ""); 93 | let builder = self.endpoint(addr.to_string())?; 94 | let tls = ClientTlsConfig::new() 95 | .ca_certificate(Certificate::from_pem(&self.ca)) 96 | .identity(Identity::from_pem( 97 | &self.cert, 98 | load_pem_file("private key", &self.key)?, 99 | )); 100 | let builder = builder.tls_config(tls)?; 101 | Ok(builder) 102 | } 103 | 104 | async fn default_channel(&self, addr: &str) -> Result { 105 | let addr = "http://".to_string() + &SCHEME_REG.replace(addr, ""); 106 | self.endpoint(addr) 107 | } 108 | 109 | fn endpoint(&self, addr: String) -> Result { 110 | let endpoint = Channel::from_shared(addr)? 111 | .tcp_keepalive(Some(Duration::from_secs(10))) 112 | .keep_alive_timeout(Duration::from_secs(3)); 113 | Ok(endpoint) 114 | } 115 | } 116 | 117 | #[cfg(test)] 118 | mod tests { 119 | use std::fs::File; 120 | use std::io::Write; 121 | use std::path::PathBuf; 122 | 123 | use tempfile; 124 | 125 | use super::*; 126 | 127 | #[test] 128 | fn test_security() { 129 | let temp = tempfile::tempdir().unwrap(); 130 | let example_ca = temp.path().join("ca"); 131 | let example_cert = temp.path().join("cert"); 132 | let example_pem = temp.path().join("key"); 133 | for (id, f) in [&example_ca, &example_cert, &example_pem] 134 | .iter() 135 | .enumerate() 136 | { 137 | File::create(f).unwrap().write_all(&[id as u8]).unwrap(); 138 | } 139 | let cert_path: PathBuf = format!("{}", example_cert.display()).into(); 140 | let key_path: PathBuf = format!("{}", example_pem.display()).into(); 141 | let ca_path: PathBuf = format!("{}", example_ca.display()).into(); 142 | let mgr = SecurityManager::load(ca_path, cert_path, &key_path).unwrap(); 143 | assert_eq!(mgr.ca, vec![0]); 144 | assert_eq!(mgr.cert, vec![1]); 145 | let key = load_pem_file("private key", &key_path).unwrap(); 146 | assert_eq!(key, vec![2]); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/compat.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | //! This module contains utility types and functions for making the transition 4 | //! from futures 0.1 to 1.0 easier. 5 | 6 | use std::pin::Pin; 7 | 8 | use futures::prelude::*; 9 | use futures::ready; 10 | use futures::task::Context; 11 | use futures::task::Poll; 12 | 13 | /// A future implementing a tail-recursive loop. 14 | /// 15 | /// Created by the `loop_fn` function. 16 | #[derive(Debug)] 17 | #[must_use = "futures do nothing unless polled"] 18 | pub struct LoopFn { 19 | future: A, 20 | func: F, 21 | errored: bool, 22 | } 23 | 24 | pub fn stream_fn(initial_state: S, mut func: F) -> LoopFn 25 | where 26 | F: FnMut(S) -> A, 27 | A: Future, E>>, 28 | { 29 | LoopFn { 30 | future: func(initial_state), 31 | func, 32 | errored: false, 33 | } 34 | } 35 | 36 | impl Stream for LoopFn 37 | where 38 | F: FnMut(S) -> A, 39 | A: Future, E>>, 40 | { 41 | type Item = Result; 42 | fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 43 | if self.errored { 44 | return Poll::Ready(None); 45 | } 46 | unsafe { 47 | let this = Pin::get_unchecked_mut(self); 48 | match ready!(Pin::new_unchecked(&mut this.future).poll(cx)) { 49 | Err(e) => { 50 | this.errored = true; 51 | Poll::Ready(Some(Err(e))) 52 | } 53 | Ok(None) => Poll::Ready(None), 54 | Ok(Some((s, t))) => { 55 | this.future = (this.func)(s); 56 | Poll::Ready(Some(Ok(t))) 57 | } 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::path::PathBuf; 4 | use std::time::Duration; 5 | 6 | use serde_derive::Deserialize; 7 | use serde_derive::Serialize; 8 | 9 | /// The configuration for either a [`RawClient`](crate::RawClient) or a 10 | /// [`TransactionClient`](crate::TransactionClient). 11 | /// 12 | /// See also [`TransactionOptions`](crate::TransactionOptions) which provides more ways to configure 13 | /// requests. 14 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 15 | #[serde(default)] 16 | #[serde(rename_all = "kebab-case")] 17 | pub struct Config { 18 | pub ca_path: Option, 19 | pub cert_path: Option, 20 | pub key_path: Option, 21 | pub timeout: Duration, 22 | pub keyspace: Option, 23 | } 24 | 25 | const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(2); 26 | 27 | impl Default for Config { 28 | fn default() -> Self { 29 | Config { 30 | ca_path: None, 31 | cert_path: None, 32 | key_path: None, 33 | timeout: DEFAULT_REQUEST_TIMEOUT, 34 | keyspace: None, 35 | } 36 | } 37 | } 38 | 39 | impl Config { 40 | /// Set the certificate authority, certificate, and key locations for clients. 41 | /// 42 | /// By default, this client will use an insecure connection over instead of one protected by 43 | /// Transport Layer Security (TLS). Your deployment may have chosen to rely on security measures 44 | /// such as a private network, or a VPN layer to provide secure transmission. 45 | /// 46 | /// To use a TLS secured connection, use the `with_security` function to set the required 47 | /// parameters. 48 | /// 49 | /// TiKV does not currently offer encrypted storage (or encryption-at-rest). 50 | /// 51 | /// # Examples 52 | /// ```rust 53 | /// # use tikv_client::Config; 54 | /// let config = Config::default().with_security("root.ca", "internal.cert", "internal.key"); 55 | /// ``` 56 | #[must_use] 57 | pub fn with_security( 58 | mut self, 59 | ca_path: impl Into, 60 | cert_path: impl Into, 61 | key_path: impl Into, 62 | ) -> Self { 63 | self.ca_path = Some(ca_path.into()); 64 | self.cert_path = Some(cert_path.into()); 65 | self.key_path = Some(key_path.into()); 66 | self 67 | } 68 | 69 | /// Set the timeout for clients. 70 | /// 71 | /// The timeout is used for all requests when using or connecting to a TiKV cluster (including 72 | /// PD nodes). If the request does not complete within timeout, the request is cancelled and 73 | /// an error returned to the user. 74 | /// 75 | /// The default timeout is two seconds. 76 | /// 77 | /// # Examples 78 | /// ```rust 79 | /// # use tikv_client::Config; 80 | /// # use std::time::Duration; 81 | /// let config = Config::default().with_timeout(Duration::from_secs(10)); 82 | /// ``` 83 | #[must_use] 84 | pub fn with_timeout(mut self, timeout: Duration) -> Self { 85 | self.timeout = timeout; 86 | self 87 | } 88 | 89 | /// Set to use default keyspace. 90 | /// 91 | /// Server should enable `storage.api-version = 2` to use this feature. 92 | #[must_use] 93 | pub fn with_default_keyspace(self) -> Self { 94 | self.with_keyspace("DEFAULT") 95 | } 96 | 97 | /// Set the use keyspace for the client. 98 | /// 99 | /// Server should enable `storage.api-version = 2` to use this feature. 100 | #[must_use] 101 | pub fn with_keyspace(mut self, keyspace: &str) -> Self { 102 | self.keyspace = Some(keyspace.to_owned()); 103 | self 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/generated/disk_usage.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] 3 | #[repr(i32)] 4 | pub enum DiskUsage { 5 | Normal = 0, 6 | AlmostFull = 1, 7 | AlreadyFull = 2, 8 | } 9 | impl DiskUsage { 10 | /// String value of the enum field names used in the ProtoBuf definition. 11 | /// 12 | /// The values are not transformed in any way and thus are considered stable 13 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 14 | pub fn as_str_name(&self) -> &'static str { 15 | match self { 16 | DiskUsage::Normal => "Normal", 17 | DiskUsage::AlmostFull => "AlmostFull", 18 | DiskUsage::AlreadyFull => "AlreadyFull", 19 | } 20 | } 21 | /// Creates an enum from field names used in the ProtoBuf definition. 22 | pub fn from_str_name(value: &str) -> ::core::option::Option { 23 | match value { 24 | "Normal" => Some(Self::Normal), 25 | "AlmostFull" => Some(Self::AlmostFull), 26 | "AlreadyFull" => Some(Self::AlreadyFull), 27 | _ => None, 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/generated/mod.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | pub mod autoid { 3 | include!("autoid.rs"); 4 | } 5 | pub mod backup { 6 | include!("backup.rs"); 7 | } 8 | pub mod cdcpb { 9 | include!("cdcpb.rs"); 10 | } 11 | pub mod configpb { 12 | include!("configpb.rs"); 13 | } 14 | pub mod coprocessor { 15 | include!("coprocessor.rs"); 16 | } 17 | pub mod deadlock { 18 | include!("deadlock.rs"); 19 | } 20 | pub mod debugpb { 21 | include!("debugpb.rs"); 22 | } 23 | pub mod diagnosticspb { 24 | include!("diagnosticspb.rs"); 25 | } 26 | pub mod disaggregated { 27 | include!("disaggregated.rs"); 28 | } 29 | pub mod disk_usage { 30 | include!("disk_usage.rs"); 31 | } 32 | pub mod encryptionpb { 33 | include!("encryptionpb.rs"); 34 | } 35 | pub mod enginepb { 36 | include!("enginepb.rs"); 37 | } 38 | pub mod eraftpb { 39 | include!("eraftpb.rs"); 40 | } 41 | pub mod errorpb { 42 | include!("errorpb.rs"); 43 | } 44 | pub mod gcpb { 45 | include!("gcpb.rs"); 46 | } 47 | pub mod google { 48 | pub mod api { 49 | include!("google.api.rs"); 50 | } 51 | } 52 | pub mod import_kvpb { 53 | include!("import_kvpb.rs"); 54 | } 55 | pub mod import_sstpb { 56 | include!("import_sstpb.rs"); 57 | } 58 | pub mod keyspacepb { 59 | include!("keyspacepb.rs"); 60 | } 61 | pub mod kvrpcpb { 62 | include!("kvrpcpb.rs"); 63 | } 64 | pub mod logbackup { 65 | include!("logbackup.rs"); 66 | } 67 | pub mod meta_storagepb { 68 | include!("meta_storagepb.rs"); 69 | } 70 | pub mod metapb { 71 | include!("metapb.rs"); 72 | } 73 | pub mod mpp { 74 | include!("mpp.rs"); 75 | } 76 | pub mod pdpb { 77 | include!("pdpb.rs"); 78 | } 79 | pub mod raft_cmdpb { 80 | include!("raft_cmdpb.rs"); 81 | } 82 | pub mod raft_serverpb { 83 | include!("raft_serverpb.rs"); 84 | } 85 | pub mod recover_data { 86 | include!("recover_data.rs"); 87 | } 88 | pub mod replication_modepb { 89 | include!("replication_modepb.rs"); 90 | } 91 | pub mod resource_manager { 92 | include!("resource_manager.rs"); 93 | } 94 | pub mod resource_usage_agent { 95 | include!("resource_usage_agent.rs"); 96 | } 97 | pub mod schedulingpb { 98 | include!("schedulingpb.rs"); 99 | } 100 | pub mod tikvpb { 101 | include!("tikvpb.rs"); 102 | } 103 | pub mod tracepb { 104 | include!("tracepb.rs"); 105 | } 106 | pub mod tsopb { 107 | include!("tsopb.rs"); 108 | } 109 | -------------------------------------------------------------------------------- /src/generated/replication_modepb.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | /// The replication status sync from PD to TiKV. 3 | #[allow(clippy::derive_partial_eq_without_eq)] 4 | #[derive(Clone, PartialEq, ::prost::Message)] 5 | pub struct ReplicationStatus { 6 | #[prost(enumeration = "ReplicationMode", tag = "1")] 7 | pub mode: i32, 8 | #[prost(message, optional, tag = "2")] 9 | pub dr_auto_sync: ::core::option::Option, 10 | } 11 | /// The status of dr-autosync mode. 12 | #[allow(clippy::derive_partial_eq_without_eq)] 13 | #[derive(Clone, PartialEq, ::prost::Message)] 14 | pub struct DrAutoSync { 15 | /// The key of the label that used for distinguish different DC. 16 | #[prost(string, tag = "1")] 17 | pub label_key: ::prost::alloc::string::String, 18 | #[prost(enumeration = "DrAutoSyncState", tag = "2")] 19 | pub state: i32, 20 | /// Unique ID of the state, it increases after each state transfer. 21 | #[prost(uint64, tag = "3")] 22 | pub state_id: u64, 23 | /// Duration to wait before switching to SYNC by force (in seconds) 24 | #[prost(int32, tag = "4")] 25 | pub wait_sync_timeout_hint: i32, 26 | /// Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. 27 | #[prost(uint64, repeated, tag = "5")] 28 | pub available_stores: ::prost::alloc::vec::Vec, 29 | /// Stores should forbid region split. 30 | #[prost(bool, tag = "6")] 31 | pub pause_region_split: bool, 32 | } 33 | /// The replication status sync from TiKV to PD. 34 | #[allow(clippy::derive_partial_eq_without_eq)] 35 | #[derive(Clone, PartialEq, ::prost::Message)] 36 | pub struct RegionReplicationStatus { 37 | #[prost(enumeration = "RegionReplicationState", tag = "1")] 38 | pub state: i32, 39 | /// Unique ID of the state, it increases after each state transfer. 40 | #[prost(uint64, tag = "2")] 41 | pub state_id: u64, 42 | } 43 | #[allow(clippy::derive_partial_eq_without_eq)] 44 | #[derive(Clone, PartialEq, ::prost::Message)] 45 | pub struct StoreDrAutoSyncStatus { 46 | #[prost(enumeration = "DrAutoSyncState", tag = "1")] 47 | pub state: i32, 48 | #[prost(uint64, tag = "2")] 49 | pub state_id: u64, 50 | } 51 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] 52 | #[repr(i32)] 53 | pub enum ReplicationMode { 54 | /// The standard mode. Replicate logs to majority peer. 55 | Majority = 0, 56 | /// DR mode. Replicate logs among 2 DCs. 57 | DrAutoSync = 1, 58 | } 59 | impl ReplicationMode { 60 | /// String value of the enum field names used in the ProtoBuf definition. 61 | /// 62 | /// The values are not transformed in any way and thus are considered stable 63 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 64 | pub fn as_str_name(&self) -> &'static str { 65 | match self { 66 | ReplicationMode::Majority => "MAJORITY", 67 | ReplicationMode::DrAutoSync => "DR_AUTO_SYNC", 68 | } 69 | } 70 | /// Creates an enum from field names used in the ProtoBuf definition. 71 | pub fn from_str_name(value: &str) -> ::core::option::Option { 72 | match value { 73 | "MAJORITY" => Some(Self::Majority), 74 | "DR_AUTO_SYNC" => Some(Self::DrAutoSync), 75 | _ => None, 76 | } 77 | } 78 | } 79 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] 80 | #[repr(i32)] 81 | pub enum DrAutoSyncState { 82 | /// Raft logs need to sync between different DCs 83 | Sync = 0, 84 | /// Wait for switching to ASYNC. Stop sync raft logs between DCs. 85 | AsyncWait = 1, 86 | /// Raft logs need to sync to majority peers 87 | Async = 2, 88 | /// Switching from ASYNC to SYNC mode 89 | SyncRecover = 3, 90 | } 91 | impl DrAutoSyncState { 92 | /// String value of the enum field names used in the ProtoBuf definition. 93 | /// 94 | /// The values are not transformed in any way and thus are considered stable 95 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 96 | pub fn as_str_name(&self) -> &'static str { 97 | match self { 98 | DrAutoSyncState::Sync => "SYNC", 99 | DrAutoSyncState::AsyncWait => "ASYNC_WAIT", 100 | DrAutoSyncState::Async => "ASYNC", 101 | DrAutoSyncState::SyncRecover => "SYNC_RECOVER", 102 | } 103 | } 104 | /// Creates an enum from field names used in the ProtoBuf definition. 105 | pub fn from_str_name(value: &str) -> ::core::option::Option { 106 | match value { 107 | "SYNC" => Some(Self::Sync), 108 | "ASYNC_WAIT" => Some(Self::AsyncWait), 109 | "ASYNC" => Some(Self::Async), 110 | "SYNC_RECOVER" => Some(Self::SyncRecover), 111 | _ => None, 112 | } 113 | } 114 | } 115 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] 116 | #[repr(i32)] 117 | pub enum RegionReplicationState { 118 | /// The region's state is unknown 119 | Unknown = 0, 120 | /// Logs sync to majority peers 121 | SimpleMajority = 1, 122 | /// Logs sync to different DCs 123 | IntegrityOverLabel = 2, 124 | } 125 | impl RegionReplicationState { 126 | /// String value of the enum field names used in the ProtoBuf definition. 127 | /// 128 | /// The values are not transformed in any way and thus are considered stable 129 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 130 | pub fn as_str_name(&self) -> &'static str { 131 | match self { 132 | RegionReplicationState::Unknown => "UNKNOWN", 133 | RegionReplicationState::SimpleMajority => "SIMPLE_MAJORITY", 134 | RegionReplicationState::IntegrityOverLabel => "INTEGRITY_OVER_LABEL", 135 | } 136 | } 137 | /// Creates an enum from field names used in the ProtoBuf definition. 138 | pub fn from_str_name(value: &str) -> ::core::option::Option { 139 | match value { 140 | "UNKNOWN" => Some(Self::Unknown), 141 | "SIMPLE_MAJORITY" => Some(Self::SimpleMajority), 142 | "INTEGRITY_OVER_LABEL" => Some(Self::IntegrityOverLabel), 143 | _ => None, 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/generated/span.rs: -------------------------------------------------------------------------------- 1 | #[allow(clippy::derive_partial_eq_without_eq)] 2 | #[derive(Clone, PartialEq, ::prost::Message)] 3 | pub struct SpanSet { 4 | #[prost(uint64, tag = "1")] 5 | pub start_time_ns: u64, 6 | #[prost(uint64, tag = "2")] 7 | pub cycles_per_sec: u64, 8 | #[prost(message, repeated, tag = "3")] 9 | pub spans: ::prost::alloc::vec::Vec, 10 | #[prost(uint64, tag = "4")] 11 | pub create_time_ns: u64, 12 | } 13 | #[allow(clippy::derive_partial_eq_without_eq)] 14 | #[derive(Clone, PartialEq, ::prost::Message)] 15 | pub struct Root {} 16 | #[allow(clippy::derive_partial_eq_without_eq)] 17 | #[derive(Clone, PartialEq, ::prost::Message)] 18 | pub struct Parent { 19 | #[prost(uint64, tag = "1")] 20 | pub id: u64, 21 | } 22 | #[allow(clippy::derive_partial_eq_without_eq)] 23 | #[derive(Clone, PartialEq, ::prost::Message)] 24 | pub struct Continue { 25 | #[prost(uint64, tag = "1")] 26 | pub id: u64, 27 | } 28 | #[allow(clippy::derive_partial_eq_without_eq)] 29 | #[derive(Clone, PartialEq, ::prost::Message)] 30 | pub struct Link { 31 | #[prost(oneof = "link::Link", tags = "1, 2, 3")] 32 | pub link: ::core::option::Option, 33 | } 34 | /// Nested message and enum types in `Link`. 35 | pub mod link { 36 | #[allow(clippy::derive_partial_eq_without_eq)] 37 | #[derive(Clone, PartialEq, ::prost::Oneof)] 38 | pub enum Link { 39 | #[prost(message, tag = "1")] 40 | Root(super::Root), 41 | #[prost(message, tag = "2")] 42 | Parent(super::Parent), 43 | #[prost(message, tag = "3")] 44 | Continue(super::Continue), 45 | } 46 | } 47 | #[allow(clippy::derive_partial_eq_without_eq)] 48 | #[derive(Clone, PartialEq, ::prost::Message)] 49 | pub struct Span { 50 | #[prost(uint64, tag = "1")] 51 | pub id: u64, 52 | #[prost(message, optional, tag = "2")] 53 | pub link: ::core::option::Option, 54 | #[prost(uint64, tag = "3")] 55 | pub begin_cycles: u64, 56 | #[prost(uint64, tag = "4")] 57 | pub end_cycles: u64, 58 | #[prost(uint32, tag = "5")] 59 | pub event: u32, 60 | } 61 | -------------------------------------------------------------------------------- /src/kv/key.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::fmt; 4 | use std::ops::Bound; 5 | 6 | #[allow(unused_imports)] 7 | #[cfg(test)] 8 | use proptest::arbitrary::any_with; 9 | #[allow(unused_imports)] 10 | #[cfg(test)] 11 | use proptest::collection::size_range; 12 | #[cfg(test)] 13 | use proptest_derive::Arbitrary; 14 | 15 | use super::HexRepr; 16 | use crate::kv::codec::BytesEncoder; 17 | use crate::kv::codec::{self}; 18 | use crate::proto::kvrpcpb; 19 | use crate::proto::kvrpcpb::KvPair; 20 | 21 | const _PROPTEST_KEY_MAX: usize = 1024 * 2; // 2 KB 22 | 23 | /// The key part of a key/value pair. 24 | /// 25 | /// In TiKV, keys are an ordered sequence of bytes. This has an advantage over choosing `String` as 26 | /// valid `UTF-8` is not required. This means that the user is permitted to store any data they wish, 27 | /// as long as it can be represented by bytes. (Which is to say, pretty much anything!) 28 | /// 29 | /// This type wraps around an owned value, so it should be treated it like `String` or `Vec`. 30 | /// 31 | /// # Examples 32 | /// ```rust 33 | /// use tikv_client::Key; 34 | /// 35 | /// let static_str: &'static str = "TiKV"; 36 | /// let from_static_str = Key::from(static_str.to_owned()); 37 | /// 38 | /// let string: String = String::from(static_str); 39 | /// let from_string = Key::from(string); 40 | /// assert_eq!(from_static_str, from_string); 41 | /// 42 | /// let vec: Vec = static_str.as_bytes().to_vec(); 43 | /// let from_vec = Key::from(vec); 44 | /// assert_eq!(from_static_str, from_vec); 45 | /// 46 | /// let bytes = static_str.as_bytes().to_vec(); 47 | /// let from_bytes = Key::from(bytes); 48 | /// assert_eq!(from_static_str, from_bytes); 49 | /// ``` 50 | /// 51 | /// While `.into()` is usually sufficient for obtaining the buffer itself, sometimes type inference 52 | /// isn't able to determine the correct type. Notably in the `assert_eq!()` and `==` cases. In 53 | /// these cases using the fully-qualified-syntax is useful: 54 | /// 55 | /// # Examples 56 | /// ```rust 57 | /// use tikv_client::Key; 58 | /// 59 | /// let buf = "TiKV".as_bytes().to_owned(); 60 | /// let key = Key::from(buf.clone()); 61 | /// assert_eq!(Into::>::into(key), buf); 62 | /// ``` 63 | /// 64 | /// Many functions which accept a `Key` accept an `Into`, which means all of the above types 65 | /// can be passed directly to those functions. 66 | #[derive(Default, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] 67 | #[cfg_attr(test, derive(Arbitrary))] 68 | #[repr(transparent)] 69 | pub struct Key( 70 | #[cfg_attr( 71 | test, 72 | proptest(strategy = "any_with::>((size_range(_PROPTEST_KEY_MAX), ()))") 73 | )] 74 | pub(crate) Vec, 75 | ); 76 | 77 | impl AsRef for kvrpcpb::Mutation { 78 | fn as_ref(&self) -> &Key { 79 | self.key.as_ref() 80 | } 81 | } 82 | 83 | pub struct KvPairTTL(pub KvPair, pub u64); 84 | 85 | impl AsRef for KvPairTTL { 86 | fn as_ref(&self) -> &Key { 87 | self.0.key.as_ref() 88 | } 89 | } 90 | 91 | impl From for (KvPair, u64) { 92 | fn from(value: KvPairTTL) -> Self { 93 | (value.0, value.1) 94 | } 95 | } 96 | 97 | impl Key { 98 | /// The empty key. 99 | pub const EMPTY: Self = Key(Vec::new()); 100 | 101 | /// Return whether the key is empty. 102 | #[inline] 103 | pub fn is_empty(&self) -> bool { 104 | self.0.is_empty() 105 | } 106 | 107 | /// Return whether the last byte of key is 0. 108 | #[inline] 109 | pub(super) fn zero_terminated(&self) -> bool { 110 | self.0.last().map(|i| *i == 0).unwrap_or(false) 111 | } 112 | 113 | /// Push a zero to the end of the key. 114 | /// 115 | /// Extending a zero makes the new key the smallest key that is greater than than the original one. 116 | #[inline] 117 | pub(crate) fn next_key(mut self) -> Self { 118 | self.0.push(0); 119 | self 120 | } 121 | 122 | /// Convert the key to a lower bound. The key is treated as inclusive. 123 | #[inline] 124 | pub(super) fn into_lower_bound(mut self) -> Bound { 125 | if self.zero_terminated() { 126 | self.0.pop().unwrap(); 127 | Bound::Excluded(self) 128 | } else { 129 | Bound::Included(self) 130 | } 131 | } 132 | 133 | /// Convert the key to an upper bound. The key is treated as exclusive. 134 | #[inline] 135 | pub(super) fn into_upper_bound(mut self) -> Bound { 136 | if self.zero_terminated() { 137 | self.0.pop().unwrap(); 138 | Bound::Included(self) 139 | } else { 140 | Bound::Excluded(self) 141 | } 142 | } 143 | 144 | /// Return the MVCC-encoded representation of the key. 145 | #[inline] 146 | #[must_use] 147 | pub fn to_encoded(&self) -> Key { 148 | let len = codec::max_encoded_bytes_size(self.0.len()); 149 | let mut encoded = Vec::with_capacity(len); 150 | encoded.encode_bytes(&self.0, false).unwrap(); 151 | Key(encoded) 152 | } 153 | 154 | pub fn len(&self) -> usize { 155 | self.0.len() 156 | } 157 | } 158 | 159 | impl From> for Key { 160 | fn from(v: Vec) -> Self { 161 | Key(v) 162 | } 163 | } 164 | 165 | impl From for Key { 166 | fn from(v: String) -> Key { 167 | Key(v.into_bytes()) 168 | } 169 | } 170 | 171 | impl From for Vec { 172 | fn from(key: Key) -> Self { 173 | key.0 174 | } 175 | } 176 | 177 | impl<'a> From<&'a Key> for &'a [u8] { 178 | fn from(key: &'a Key) -> Self { 179 | &key.0 180 | } 181 | } 182 | 183 | impl<'a> From<&'a Vec> for &'a Key { 184 | fn from(key: &'a Vec) -> Self { 185 | unsafe { &*(key as *const Vec as *const Key) } 186 | } 187 | } 188 | impl AsRef for Key { 189 | fn as_ref(&self) -> &Key { 190 | self 191 | } 192 | } 193 | 194 | impl AsRef for Vec { 195 | fn as_ref(&self) -> &Key { 196 | unsafe { &*(self as *const Vec as *const Key) } 197 | } 198 | } 199 | 200 | impl fmt::Debug for Key { 201 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 202 | write!(f, "Key({})", HexRepr(&self.0)) 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /src/kv/kvpair.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::fmt; 4 | use std::str; 5 | 6 | #[cfg(test)] 7 | use proptest_derive::Arbitrary; 8 | 9 | use super::HexRepr; 10 | use super::Key; 11 | use super::Value; 12 | use crate::proto::kvrpcpb; 13 | 14 | /// A key/value pair. 15 | /// 16 | /// # Examples 17 | /// ```rust 18 | /// # use tikv_client::{Key, Value, KvPair}; 19 | /// let key = "key".to_owned(); 20 | /// let value = "value".to_owned(); 21 | /// let constructed = KvPair::new(key.clone(), value.clone()); 22 | /// let from_tuple = KvPair::from((key, value)); 23 | /// assert_eq!(constructed, from_tuple); 24 | /// ``` 25 | /// 26 | /// Many functions which accept a `KvPair` accept an `Into`, which means all of the above 27 | /// types (Like a `(Key, Value)`) can be passed directly to those functions. 28 | #[derive(Default, Clone, Eq, PartialEq, Hash)] 29 | #[cfg_attr(test, derive(Arbitrary))] 30 | pub struct KvPair(pub Key, pub Value); 31 | 32 | impl KvPair { 33 | /// Create a new `KvPair`. 34 | #[inline] 35 | pub fn new(key: impl Into, value: impl Into) -> Self { 36 | KvPair(key.into(), value.into()) 37 | } 38 | 39 | /// Immutably borrow the `Key` part of the `KvPair`. 40 | #[inline] 41 | pub fn key(&self) -> &Key { 42 | &self.0 43 | } 44 | 45 | /// Immutably borrow the `Value` part of the `KvPair`. 46 | #[inline] 47 | pub fn value(&self) -> &Value { 48 | &self.1 49 | } 50 | 51 | /// Consume `self` and return the `Key` part. 52 | #[inline] 53 | pub fn into_key(self) -> Key { 54 | self.0 55 | } 56 | 57 | /// Consume `self` and return the `Value` part. 58 | #[inline] 59 | pub fn into_value(self) -> Value { 60 | self.1 61 | } 62 | 63 | /// Mutably borrow the `Key` part of the `KvPair`. 64 | #[inline] 65 | pub fn key_mut(&mut self) -> &mut Key { 66 | &mut self.0 67 | } 68 | 69 | /// Mutably borrow the `Value` part of the `KvPair`. 70 | #[inline] 71 | pub fn value_mut(&mut self) -> &mut Value { 72 | &mut self.1 73 | } 74 | 75 | /// Set the `Key` part of the `KvPair`. 76 | #[inline] 77 | pub fn set_key(&mut self, k: impl Into) { 78 | self.0 = k.into(); 79 | } 80 | 81 | /// Set the `Value` part of the `KvPair`. 82 | #[inline] 83 | pub fn set_value(&mut self, v: impl Into) { 84 | self.1 = v.into(); 85 | } 86 | } 87 | 88 | impl From<(K, V)> for KvPair 89 | where 90 | K: Into, 91 | V: Into, 92 | { 93 | fn from((k, v): (K, V)) -> Self { 94 | KvPair(k.into(), v.into()) 95 | } 96 | } 97 | 98 | impl From for (Key, Value) { 99 | fn from(pair: KvPair) -> Self { 100 | (pair.0, pair.1) 101 | } 102 | } 103 | 104 | impl From for Key { 105 | fn from(pair: KvPair) -> Self { 106 | pair.0 107 | } 108 | } 109 | 110 | impl From for KvPair { 111 | fn from(pair: kvrpcpb::KvPair) -> Self { 112 | KvPair(Key::from(pair.key), pair.value) 113 | } 114 | } 115 | 116 | impl From for kvrpcpb::KvPair { 117 | fn from(pair: KvPair) -> Self { 118 | let mut result = kvrpcpb::KvPair::default(); 119 | let (key, value) = pair.into(); 120 | result.key = key.into(); 121 | result.value = value; 122 | result 123 | } 124 | } 125 | 126 | impl AsRef for KvPair { 127 | fn as_ref(&self) -> &Key { 128 | &self.0 129 | } 130 | } 131 | 132 | impl AsRef for KvPair { 133 | fn as_ref(&self) -> &Value { 134 | &self.1 135 | } 136 | } 137 | 138 | impl fmt::Debug for KvPair { 139 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 140 | let KvPair(key, value) = self; 141 | match str::from_utf8(value) { 142 | Ok(s) => write!(f, "KvPair({}, {:?})", HexRepr(&key.0), s), 143 | Err(_) => write!(f, "KvPair({}, {})", HexRepr(&key.0), HexRepr(value)), 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/kv/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | use std::fmt; 3 | 4 | mod bound_range; 5 | pub mod codec; 6 | mod key; 7 | mod kvpair; 8 | mod value; 9 | 10 | pub use bound_range::BoundRange; 11 | pub use bound_range::IntoOwnedRange; 12 | pub use key::Key; 13 | pub use key::KvPairTTL; 14 | pub use kvpair::KvPair; 15 | pub use value::Value; 16 | 17 | struct HexRepr<'a>(pub &'a [u8]); 18 | 19 | impl fmt::Display for HexRepr<'_> { 20 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 21 | for byte in self.0 { 22 | write!(f, "{byte:02X}")?; 23 | } 24 | Ok(()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/kv/value.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | const _PROPTEST_VALUE_MAX: usize = 1024 * 16; // 16 KB 4 | 5 | /// The value part of a key/value pair. An alias for `Vec`. 6 | /// 7 | /// In TiKV, a value is an ordered sequence of bytes. This has an advantage over choosing `String` 8 | /// as valid `UTF-8` is not required. This means that the user is permitted to store any data they wish, 9 | /// as long as it can be represented by bytes. (Which is to say, pretty much anything!) 10 | /// 11 | /// Since `Value` is just an alias for `Vec`, conversions to and from it are easy. 12 | /// 13 | /// Many functions which accept a `Value` accept an `Into`. 14 | pub type Value = Vec; 15 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | //! This crate provides an easy-to-use client for [TiKV](https://github.com/tikv/tikv), a 4 | //! distributed, transactional key-value database written in Rust. 5 | //! 6 | //! This crate lets you connect to a TiKV cluster and use either a transactional or raw (simple 7 | //! get/put style without transactional consistency guarantees) API to access and update your data. 8 | //! 9 | //! The TiKV Rust client supports several levels of abstraction. The most convenient way to use the 10 | //! client is via [`RawClient`] and [`TransactionClient`]. This gives a very high-level API which 11 | //! mostly abstracts over the distributed nature of the store and has sensible defaults for all 12 | //! protocols. This interface can be configured, primarily when creating the client or transaction 13 | //! objects via the [`Config`] and [`TransactionOptions`] structs. Using some options, you can take 14 | //! over parts of the protocols (such as retrying failed messages) yourself. 15 | //! 16 | //! The lowest level of abstraction is to create and send gRPC messages directly to TiKV (and PD) 17 | //! nodes. The `tikv-client-store` and `tikv-client-pd` crates make this easier than using the 18 | //! protobuf definitions and a gRPC library directly, but give you the same level of control. 19 | //! 20 | //! In between these levels of abstraction, you can send and receive individual messages to the TiKV 21 | //! cluster, but take advantage of library code for common operations such as resolving data to 22 | //! regions and thus nodes in the cluster, or retrying failed messages. This can be useful for 23 | //! testing a TiKV cluster or for some advanced use cases. See the [`request`] module for 24 | //! this API, and [`raw::lowering`] and [`transaction::lowering`] for 25 | //! convenience methods for creating request objects. 26 | //! 27 | //! ## Choosing an API 28 | //! 29 | //! This crate offers both [raw](RawClient) and 30 | //! [transactional](Transaction) APIs. You should choose just one for your system. 31 | //! 32 | //! The consequence of supporting transactions is increased overhead of coordination with the 33 | //! placement driver and TiKV, and additional code complexity. 34 | //! 35 | //! *While it is possible to use both APIs at the same time, doing so is unsafe and unsupported.* 36 | //! 37 | //! ### Transactional 38 | //! 39 | //! The [transactional](Transaction) API supports **transactions** via multi-version 40 | //! concurrency control (MVCC). 41 | //! 42 | //! Best when you mostly do complex sets of actions, actions which may require a rollback, 43 | //! operations affecting multiple keys or values, or operations that depend on strong consistency. 44 | //! 45 | //! 46 | //! ### Raw 47 | //! 48 | //! The [raw](RawClient) API has reduced coordination overhead, but lacks any 49 | //! transactional abilities. 50 | //! 51 | //! Best when you mostly do single value changes, and have very limited cross-value 52 | //! requirements. You will not be able to use transactions with this API. 53 | //! 54 | //! ## Usage 55 | //! 56 | //! The general flow of using the client crate is to create either a raw or transaction client 57 | //! object (which can be configured) then send commands using the client object, or use it to create 58 | //! transactions objects. In the latter case, the transaction is built up using various commands and 59 | //! then committed (or rolled back). 60 | //! 61 | //! ### Examples 62 | //! 63 | //! Raw mode: 64 | //! 65 | //! ```rust,no_run 66 | //! # use tikv_client::{RawClient, Result}; 67 | //! # use futures::prelude::*; 68 | //! # fn main() -> Result<()> { 69 | //! # futures::executor::block_on(async { 70 | //! let client = RawClient::new(vec!["127.0.0.1:2379"]).await?; 71 | //! client.put("key".to_owned(), "value".to_owned()).await?; 72 | //! let value = client.get("key".to_owned()).await?; 73 | //! # Ok(()) 74 | //! # })} 75 | //! ``` 76 | //! 77 | //! Transactional mode: 78 | //! 79 | //! ```rust,no_run 80 | //! # use tikv_client::{TransactionClient, Result}; 81 | //! # use futures::prelude::*; 82 | //! # fn main() -> Result<()> { 83 | //! # futures::executor::block_on(async { 84 | //! let txn_client = TransactionClient::new(vec!["127.0.0.1:2379"]).await?; 85 | //! let mut txn = txn_client.begin_optimistic().await?; 86 | //! txn.put("key".to_owned(), "value".to_owned()).await?; 87 | //! let value = txn.get("key".to_owned()).await?; 88 | //! txn.commit().await?; 89 | //! # Ok(()) 90 | //! # })} 91 | //! ``` 92 | 93 | #![allow(clippy::field_reassign_with_default)] 94 | 95 | pub mod backoff; 96 | #[doc(hidden)] 97 | pub mod raw; 98 | pub mod request; 99 | #[doc(hidden)] 100 | pub mod transaction; 101 | 102 | mod common; 103 | mod compat; 104 | mod config; 105 | mod kv; 106 | mod pd; 107 | mod proto; 108 | mod region; 109 | mod region_cache; 110 | mod stats; 111 | mod store; 112 | mod timestamp; 113 | mod util; 114 | 115 | #[cfg(test)] 116 | mod mock; 117 | #[cfg(test)] 118 | mod proptests; 119 | 120 | #[doc(inline)] 121 | pub use common::security::SecurityManager; 122 | #[doc(inline)] 123 | pub use common::Error; 124 | #[doc(inline)] 125 | pub use common::Result; 126 | #[doc(inline)] 127 | pub use config::Config; 128 | 129 | #[doc(inline)] 130 | pub use crate::backoff::Backoff; 131 | #[doc(inline)] 132 | pub use crate::kv::BoundRange; 133 | #[doc(inline)] 134 | pub use crate::kv::IntoOwnedRange; 135 | #[doc(inline)] 136 | pub use crate::kv::Key; 137 | #[doc(inline)] 138 | pub use crate::kv::KvPair; 139 | #[doc(inline)] 140 | pub use crate::kv::Value; 141 | #[doc(inline)] 142 | pub use crate::raw::lowering as raw_lowering; 143 | #[doc(inline)] 144 | pub use crate::raw::Client as RawClient; 145 | #[doc(inline)] 146 | pub use crate::raw::ColumnFamily; 147 | #[doc(inline)] 148 | pub use crate::request::RetryOptions; 149 | #[doc(inline)] 150 | pub use crate::timestamp::Timestamp; 151 | #[doc(inline)] 152 | pub use crate::timestamp::TimestampExt; 153 | #[doc(inline)] 154 | pub use crate::transaction::lowering as transaction_lowering; 155 | #[doc(inline)] 156 | pub use crate::transaction::CheckLevel; 157 | #[doc(inline)] 158 | pub use crate::transaction::Client as TransactionClient; 159 | #[doc(inline)] 160 | pub use crate::transaction::Snapshot; 161 | #[doc(inline)] 162 | pub use crate::transaction::Transaction; 163 | #[doc(inline)] 164 | pub use crate::transaction::TransactionOptions; 165 | -------------------------------------------------------------------------------- /src/pd/mod.rs: -------------------------------------------------------------------------------- 1 | mod client; 2 | mod cluster; 3 | mod retry; 4 | mod timestamp; 5 | 6 | pub use self::client::PdClient; 7 | pub use self::client::PdRpcClient; 8 | pub use self::cluster::Cluster; 9 | pub use self::cluster::Connection; 10 | pub use self::retry::RetryClient; 11 | pub use self::retry::RetryClientTrait; 12 | -------------------------------------------------------------------------------- /src/proptests/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | // Note: This module exists and includes some integration tests because the `/tests/` 4 | // directory tests don't have access to `cfg(tests)` functions and we don't want to force 5 | // users to depend on proptest or manually enable features to test. 6 | 7 | // Temporarily disabled 8 | // 9 | // use proptest::strategy::Strategy; 10 | // use std::env::var; 11 | // 12 | // mod raw; 13 | // pub(crate) const ENV_PD_ADDRS: &str = "PD_ADDRS"; 14 | // pub(crate) const PROPTEST_BATCH_SIZE_MAX: usize = 16; 15 | // 16 | // pub fn arb_batch( 17 | // single_strategy: impl Strategy, 18 | // max_batch_size: impl Into>, 19 | // ) -> impl Strategy> { 20 | // let max_batch_size = max_batch_size.into().unwrap_or(PROPTEST_BATCH_SIZE_MAX); 21 | // proptest::collection::vec(single_strategy, 0..max_batch_size) 22 | // } 23 | // 24 | // pub fn pd_addrs() -> Vec { 25 | // var(ENV_PD_ADDRS) 26 | // .expect(&format!("Expected {}:", ENV_PD_ADDRS)) 27 | // .split(",") 28 | // .map(From::from) 29 | // .collect() 30 | // } 31 | -------------------------------------------------------------------------------- /src/proptests/raw.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | use super::{arb_batch, pd_addrs}; 3 | use crate::{raw::Client, Config, KvPair, Value}; 4 | use futures::executor::block_on; 5 | use proptest::{arbitrary::any, proptest}; 6 | use std::collections::HashSet; 7 | 8 | proptest! { 9 | /// Test single point (put, get, delete) operations on keys. 10 | #[test] 11 | #[cfg_attr(not(feature = "integration-tests"), ignore)] 12 | fn point( 13 | pair in any::(), 14 | ) { 15 | let client = block_on(Client::new(Config::new(pd_addrs()))).unwrap(); 16 | 17 | block_on( 18 | client.put(pair.key().clone(), pair.value().clone()) 19 | ).unwrap(); 20 | 21 | let out_value = block_on( 22 | client.get(pair.key().clone()) 23 | ).unwrap(); 24 | 25 | assert_eq!(Some(Value::from(pair.value().clone())), out_value); 26 | 27 | block_on( 28 | client.delete(pair.key().clone()) 29 | ).unwrap(); 30 | } 31 | } 32 | 33 | proptest! { 34 | /// Test batch (put, get, delete) operations on keys. 35 | #[test] 36 | #[cfg_attr(not(feature = "integration-tests"), ignore)] 37 | fn batch( 38 | kvs in arb_batch(any::(), None), 39 | ) { 40 | let client = block_on(Client::new(Config::new(pd_addrs()))).unwrap(); 41 | let keys = kvs.iter().map(|kv| kv.key()).cloned().collect::>(); 42 | let key_set = keys.iter().collect::>(); 43 | 44 | // skip if there are duplicated keys 45 | if keys.len() == key_set.len() { 46 | block_on( 47 | client.batch_put(kvs.clone()) 48 | ).unwrap(); 49 | 50 | let out_value = block_on( 51 | client.batch_get(keys.clone()) 52 | ).unwrap(); 53 | assert_eq!(kvs, out_value); 54 | 55 | block_on( 56 | client.batch_delete(keys.clone()) 57 | ).unwrap(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/proto.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | #![allow(clippy::large_enum_variant)] 4 | #![allow(clippy::enum_variant_names)] 5 | 6 | pub use protos::*; 7 | 8 | #[allow(clippy::doc_lazy_continuation)] 9 | mod protos { 10 | include!("generated/mod.rs"); 11 | } 12 | -------------------------------------------------------------------------------- /src/raw/lowering.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | //! This module provides constructor functions for requests which take arguments as high-level 4 | //! types (i.e., the types from the client crate) and converts these to the types used in the 5 | //! generated protobuf code, then calls the low-level ctor functions in the requests module. 6 | 7 | use std::iter::Iterator; 8 | use std::ops::Range; 9 | use std::sync::Arc; 10 | 11 | use crate::proto::kvrpcpb; 12 | use crate::proto::metapb; 13 | use crate::raw::requests; 14 | use crate::BoundRange; 15 | use crate::ColumnFamily; 16 | use crate::Key; 17 | use crate::KvPair; 18 | use crate::Value; 19 | 20 | pub fn new_raw_get_request(key: Key, cf: Option) -> kvrpcpb::RawGetRequest { 21 | requests::new_raw_get_request(key.into(), cf) 22 | } 23 | 24 | pub fn new_raw_batch_get_request( 25 | keys: impl Iterator, 26 | cf: Option, 27 | ) -> kvrpcpb::RawBatchGetRequest { 28 | requests::new_raw_batch_get_request(keys.map(Into::into).collect(), cf) 29 | } 30 | 31 | pub fn new_raw_get_key_ttl_request( 32 | key: Key, 33 | cf: Option, 34 | ) -> kvrpcpb::RawGetKeyTtlRequest { 35 | requests::new_raw_get_key_ttl_request(key.into(), cf) 36 | } 37 | 38 | pub fn new_raw_put_request( 39 | key: Key, 40 | value: Value, 41 | cf: Option, 42 | ttl: u64, 43 | atomic: bool, 44 | ) -> kvrpcpb::RawPutRequest { 45 | requests::new_raw_put_request(key.into(), value, ttl, cf, atomic) 46 | } 47 | 48 | pub fn new_raw_batch_put_request( 49 | pairs: impl Iterator, 50 | ttls: impl Iterator, 51 | cf: Option, 52 | atomic: bool, 53 | ) -> kvrpcpb::RawBatchPutRequest { 54 | let pairs = pairs.map(Into::into).collect::>(); 55 | let ttls = ttls.take(pairs.len()).collect::>(); 56 | assert_eq!(pairs.len(), ttls.len()); 57 | requests::new_raw_batch_put_request(pairs, ttls, cf, atomic) 58 | } 59 | 60 | pub fn new_raw_delete_request( 61 | key: Key, 62 | cf: Option, 63 | atomic: bool, 64 | ) -> kvrpcpb::RawDeleteRequest { 65 | requests::new_raw_delete_request(key.into(), cf, atomic) 66 | } 67 | 68 | pub fn new_raw_batch_delete_request( 69 | keys: impl Iterator, 70 | cf: Option, 71 | ) -> kvrpcpb::RawBatchDeleteRequest { 72 | requests::new_raw_batch_delete_request(keys.map(Into::into).collect(), cf) 73 | } 74 | 75 | pub fn new_raw_delete_range_request( 76 | range: BoundRange, 77 | cf: Option, 78 | ) -> kvrpcpb::RawDeleteRangeRequest { 79 | let (start_key, end_key) = range.into_keys(); 80 | requests::new_raw_delete_range_request(start_key.into(), end_key.unwrap_or_default().into(), cf) 81 | } 82 | 83 | pub fn new_raw_scan_request( 84 | range: BoundRange, 85 | limit: u32, 86 | key_only: bool, 87 | reverse: bool, 88 | cf: Option, 89 | ) -> kvrpcpb::RawScanRequest { 90 | let (start_key, end_key) = range.into_keys(); 91 | requests::new_raw_scan_request( 92 | start_key.into(), 93 | end_key.unwrap_or_default().into(), 94 | limit, 95 | key_only, 96 | reverse, 97 | cf, 98 | ) 99 | } 100 | 101 | pub fn new_raw_batch_scan_request( 102 | ranges: impl Iterator, 103 | each_limit: u32, 104 | key_only: bool, 105 | cf: Option, 106 | ) -> kvrpcpb::RawBatchScanRequest { 107 | requests::new_raw_batch_scan_request(ranges.map(Into::into).collect(), each_limit, key_only, cf) 108 | } 109 | 110 | pub fn new_cas_request( 111 | key: Key, 112 | value: Value, 113 | previous_value: Option, 114 | cf: Option, 115 | ) -> kvrpcpb::RawCasRequest { 116 | requests::new_cas_request(key.into(), value, previous_value, cf) 117 | } 118 | 119 | pub fn new_raw_coprocessor_request( 120 | copr_name: String, 121 | copr_version_req: String, 122 | ranges: impl Iterator, 123 | request_builder: impl Fn(metapb::Region, Vec>) -> Vec + Send + Sync + 'static, 124 | ) -> requests::RawCoprocessorRequest { 125 | requests::new_raw_coprocessor_request( 126 | copr_name, 127 | copr_version_req, 128 | ranges.map(Into::into).collect(), 129 | Arc::new(move |region, ranges| { 130 | request_builder( 131 | region, 132 | ranges 133 | .into_iter() 134 | .map(|range| range.start_key.into()..range.end_key.into()) 135 | .collect(), 136 | ) 137 | }), 138 | ) 139 | } 140 | -------------------------------------------------------------------------------- /src/raw/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | //! Raw related functionality. 4 | //! 5 | //! Using the [`raw::Client`](client::Client) you can utilize TiKV's raw interface. 6 | //! 7 | //! This interface offers optimal performance as it does not require coordination with a timestamp 8 | //! oracle, while the transactional interface does. 9 | //! 10 | //! **Warning:** It is not advisable to use both raw and transactional functionality in the same keyspace. 11 | 12 | use std::convert::TryFrom; 13 | use std::fmt; 14 | 15 | pub use self::client::Client; 16 | use crate::Error; 17 | 18 | mod client; 19 | pub mod lowering; 20 | mod requests; 21 | 22 | /// A [`ColumnFamily`](ColumnFamily) is an optional parameter for [`raw::Client`](Client) requests. 23 | /// 24 | /// TiKV uses RocksDB's `ColumnFamily` support. You can learn more about RocksDB's `ColumnFamily`s [on their wiki](https://github.com/facebook/rocksdb/wiki/Column-Families). 25 | /// 26 | /// By default in TiKV data is stored in three different `ColumnFamily` values, configurable in the TiKV server's configuration: 27 | /// 28 | /// * Default: Where real user data is stored. Set by `[rocksdb.defaultcf]`. 29 | /// * Write: Where MVCC and index related data are stored. Set by `[rocksdb.writecf]`. 30 | /// * Lock: Where lock information is stored. Set by `[rocksdb.lockcf]`. 31 | /// 32 | /// Not providing a call a `ColumnFamily` means it will use the default value of `default`. 33 | /// 34 | /// The best (and only) way to create a [`ColumnFamily`](ColumnFamily) is via the `From` implementation: 35 | /// 36 | /// # Examples 37 | /// ```rust 38 | /// # use tikv_client::ColumnFamily; 39 | /// # use std::convert::TryFrom; 40 | /// 41 | /// let cf = ColumnFamily::try_from("write").unwrap(); 42 | /// let cf = ColumnFamily::try_from(String::from("write")).unwrap(); 43 | /// ``` 44 | /// 45 | /// **But, you should not need to worry about all this:** Many functions which accept a 46 | /// `ColumnFamily` accept an `Into`, which means all of the above types can be passed 47 | /// directly to those functions. 48 | #[derive(Clone, Eq, PartialEq, Hash, Debug)] 49 | pub enum ColumnFamily { 50 | Default, 51 | Lock, 52 | Write, 53 | } 54 | 55 | impl TryFrom<&str> for ColumnFamily { 56 | type Error = Error; 57 | 58 | fn try_from(value: &str) -> Result { 59 | match value { 60 | "default" => Ok(ColumnFamily::Default), 61 | "lock" => Ok(ColumnFamily::Lock), 62 | "write" => Ok(ColumnFamily::Write), 63 | s => Err(Error::ColumnFamilyError(s.to_owned())), 64 | } 65 | } 66 | } 67 | 68 | impl TryFrom for ColumnFamily { 69 | type Error = Error; 70 | 71 | fn try_from(value: String) -> Result { 72 | TryFrom::try_from(&*value) 73 | } 74 | } 75 | 76 | impl fmt::Display for ColumnFamily { 77 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 78 | match self { 79 | ColumnFamily::Default => f.write_str("default"), 80 | ColumnFamily::Lock => f.write_str("lock"), 81 | ColumnFamily::Write => f.write_str("write"), 82 | } 83 | } 84 | } 85 | 86 | trait RawRpcRequest: Default { 87 | fn set_cf(&mut self, cf: String); 88 | 89 | fn maybe_set_cf(&mut self, cf: Option) { 90 | if let Some(cf) = cf { 91 | self.set_cf(cf.to_string()); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/region.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use derive_new::new; 4 | 5 | use crate::proto::metapb; 6 | use crate::Error; 7 | use crate::Key; 8 | use crate::Result; 9 | 10 | /// The ID of a region 11 | pub type RegionId = u64; 12 | /// The ID of a store 13 | pub type StoreId = u64; 14 | 15 | /// The ID and version information of a region. 16 | #[derive(Eq, PartialEq, Hash, Clone, Default, Debug)] 17 | pub struct RegionVerId { 18 | /// The ID of the region 19 | pub id: RegionId, 20 | /// Conf change version, auto increment when add or remove peer 21 | pub conf_ver: u64, 22 | /// Region version, auto increment when split or merge 23 | pub ver: u64, 24 | } 25 | 26 | /// Information about a TiKV region and its leader. 27 | /// 28 | /// In TiKV all data is partitioned by range. Each partition is called a region. 29 | #[derive(new, Clone, Default, Debug, PartialEq)] 30 | pub struct RegionWithLeader { 31 | pub region: metapb::Region, 32 | pub leader: Option, 33 | } 34 | 35 | impl Eq for RegionWithLeader {} 36 | 37 | impl RegionWithLeader { 38 | pub fn contains(&self, key: &Key) -> bool { 39 | let key: &[u8] = key.into(); 40 | let start_key = &self.region.start_key; 41 | let end_key = &self.region.end_key; 42 | key >= start_key.as_slice() && (key < end_key.as_slice() || end_key.is_empty()) 43 | } 44 | 45 | pub fn start_key(&self) -> Key { 46 | self.region.start_key.to_vec().into() 47 | } 48 | 49 | pub fn end_key(&self) -> Key { 50 | self.region.end_key.to_vec().into() 51 | } 52 | 53 | pub fn range(&self) -> (Key, Key) { 54 | (self.start_key(), self.end_key()) 55 | } 56 | 57 | pub fn ver_id(&self) -> RegionVerId { 58 | let region = &self.region; 59 | let epoch = region.region_epoch.as_ref().unwrap(); 60 | RegionVerId { 61 | id: region.id, 62 | conf_ver: epoch.conf_ver, 63 | ver: epoch.version, 64 | } 65 | } 66 | 67 | pub fn id(&self) -> RegionId { 68 | self.region.id 69 | } 70 | 71 | pub fn get_store_id(&self) -> Result { 72 | self.leader 73 | .as_ref() 74 | .cloned() 75 | .ok_or_else(|| Error::LeaderNotFound { 76 | region: self.ver_id(), 77 | }) 78 | .map(|s| s.store_id) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::time::Duration; 4 | use std::time::Instant; 5 | 6 | use prometheus::register_histogram; 7 | use prometheus::register_histogram_vec; 8 | use prometheus::register_int_counter_vec; 9 | use prometheus::Histogram; 10 | use prometheus::HistogramVec; 11 | use prometheus::IntCounterVec; 12 | 13 | use crate::Result; 14 | 15 | pub struct RequestStats { 16 | start: Instant, 17 | cmd: &'static str, 18 | duration: &'static HistogramVec, 19 | failed_duration: &'static HistogramVec, 20 | failed_counter: &'static IntCounterVec, 21 | } 22 | 23 | impl RequestStats { 24 | pub fn new( 25 | cmd: &'static str, 26 | duration: &'static HistogramVec, 27 | counter: &'static IntCounterVec, 28 | failed_duration: &'static HistogramVec, 29 | failed_counter: &'static IntCounterVec, 30 | ) -> Self { 31 | counter.with_label_values(&[cmd]).inc(); 32 | RequestStats { 33 | start: Instant::now(), 34 | cmd, 35 | duration, 36 | failed_duration, 37 | failed_counter, 38 | } 39 | } 40 | 41 | pub fn done(&self, r: Result) -> Result { 42 | if r.is_ok() { 43 | self.duration 44 | .with_label_values(&[self.cmd]) 45 | .observe(duration_to_sec(self.start.elapsed())); 46 | } else { 47 | self.failed_duration 48 | .with_label_values(&[self.cmd]) 49 | .observe(duration_to_sec(self.start.elapsed())); 50 | self.failed_counter.with_label_values(&[self.cmd]).inc(); 51 | } 52 | r 53 | } 54 | } 55 | 56 | pub fn tikv_stats(cmd: &'static str) -> RequestStats { 57 | RequestStats::new( 58 | cmd, 59 | &TIKV_REQUEST_DURATION_HISTOGRAM_VEC, 60 | &TIKV_REQUEST_COUNTER_VEC, 61 | &TIKV_FAILED_REQUEST_DURATION_HISTOGRAM_VEC, 62 | &TIKV_FAILED_REQUEST_COUNTER_VEC, 63 | ) 64 | } 65 | 66 | pub fn pd_stats(cmd: &'static str) -> RequestStats { 67 | RequestStats::new( 68 | cmd, 69 | &PD_REQUEST_DURATION_HISTOGRAM_VEC, 70 | &PD_REQUEST_COUNTER_VEC, 71 | &PD_FAILED_REQUEST_DURATION_HISTOGRAM_VEC, 72 | &PD_FAILED_REQUEST_COUNTER_VEC, 73 | ) 74 | } 75 | 76 | #[allow(dead_code)] 77 | pub fn observe_tso_batch(batch_size: usize) { 78 | PD_TSO_BATCH_SIZE_HISTOGRAM.observe(batch_size as f64); 79 | } 80 | 81 | lazy_static::lazy_static! { 82 | static ref TIKV_REQUEST_DURATION_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( 83 | "tikv_request_duration_seconds", 84 | "Bucketed histogram of TiKV requests duration", 85 | &["type"] 86 | ) 87 | .unwrap(); 88 | static ref TIKV_REQUEST_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( 89 | "tikv_request_total", 90 | "Total number of requests sent to TiKV", 91 | &["type"] 92 | ) 93 | .unwrap(); 94 | static ref TIKV_FAILED_REQUEST_DURATION_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( 95 | "tikv_failed_request_duration_seconds", 96 | "Bucketed histogram of failed TiKV requests duration", 97 | &["type"] 98 | ) 99 | .unwrap(); 100 | static ref TIKV_FAILED_REQUEST_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( 101 | "tikv_failed_request_total", 102 | "Total number of failed requests sent to TiKV", 103 | &["type"] 104 | ) 105 | .unwrap(); 106 | static ref PD_REQUEST_DURATION_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( 107 | "pd_request_duration_seconds", 108 | "Bucketed histogram of PD requests duration", 109 | &["type"] 110 | ) 111 | .unwrap(); 112 | static ref PD_REQUEST_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( 113 | "pd_request_total", 114 | "Total number of requests sent to PD", 115 | &["type"] 116 | ) 117 | .unwrap(); 118 | static ref PD_FAILED_REQUEST_DURATION_HISTOGRAM_VEC: HistogramVec = register_histogram_vec!( 119 | "pd_failed_request_duration_seconds", 120 | "Bucketed histogram of failed PD requests duration", 121 | &["type"] 122 | ) 123 | .unwrap(); 124 | static ref PD_FAILED_REQUEST_COUNTER_VEC: IntCounterVec = register_int_counter_vec!( 125 | "pd_failed_request_total", 126 | "Total number of failed requests sent to PD", 127 | &["type"] 128 | ) 129 | .unwrap(); 130 | static ref PD_TSO_BATCH_SIZE_HISTOGRAM: Histogram = register_histogram!( 131 | "pd_tso_batch_size", 132 | "Bucketed histogram of TSO request batch size" 133 | ) 134 | .unwrap(); 135 | } 136 | 137 | /// Convert Duration to seconds. 138 | #[inline] 139 | fn duration_to_sec(d: Duration) -> f64 { 140 | let nanos = f64::from(d.subsec_nanos()); 141 | // In most cases, we can't have so large Duration, so here just panic if overflow now. 142 | d.as_secs() as f64 + (nanos / 1_000_000_000.0) 143 | } 144 | -------------------------------------------------------------------------------- /src/store/client.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::any::Any; 4 | use std::sync::Arc; 5 | use std::time::Duration; 6 | 7 | use async_trait::async_trait; 8 | use derive_new::new; 9 | use tonic::transport::Channel; 10 | 11 | use super::Request; 12 | use crate::proto::tikvpb::tikv_client::TikvClient; 13 | use crate::Result; 14 | use crate::SecurityManager; 15 | 16 | /// A trait for connecting to TiKV stores. 17 | #[async_trait] 18 | pub trait KvConnect: Sized + Send + Sync + 'static { 19 | type KvClient: KvClient + Clone + Send + Sync + 'static; 20 | 21 | async fn connect(&self, address: &str) -> Result; 22 | } 23 | 24 | #[derive(new, Clone)] 25 | pub struct TikvConnect { 26 | security_mgr: Arc, 27 | timeout: Duration, 28 | } 29 | 30 | #[async_trait] 31 | impl KvConnect for TikvConnect { 32 | type KvClient = KvRpcClient; 33 | 34 | async fn connect(&self, address: &str) -> Result { 35 | self.security_mgr 36 | .connect(address, TikvClient::new) 37 | .await 38 | .map(|c| KvRpcClient::new(c, self.timeout)) 39 | } 40 | } 41 | 42 | #[async_trait] 43 | pub trait KvClient { 44 | async fn dispatch(&self, req: &dyn Request) -> Result>; 45 | } 46 | 47 | /// This client handles requests for a single TiKV node. It converts the data 48 | /// types and abstractions of the client program into the grpc data types. 49 | #[derive(new, Clone)] 50 | pub struct KvRpcClient { 51 | rpc_client: TikvClient, 52 | timeout: Duration, 53 | } 54 | 55 | #[async_trait] 56 | impl KvClient for KvRpcClient { 57 | async fn dispatch(&self, request: &dyn Request) -> Result> { 58 | request.dispatch(&self.rpc_client, self.timeout).await 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/store/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | mod client; 4 | mod errors; 5 | mod request; 6 | 7 | use std::cmp::max; 8 | use std::cmp::min; 9 | use std::sync::Arc; 10 | 11 | use derive_new::new; 12 | use futures::prelude::*; 13 | use futures::stream::BoxStream; 14 | 15 | pub use self::client::KvClient; 16 | pub use self::client::KvConnect; 17 | pub use self::client::TikvConnect; 18 | pub use self::errors::HasKeyErrors; 19 | pub use self::errors::HasRegionError; 20 | pub use self::errors::HasRegionErrors; 21 | pub use self::request::Request; 22 | use crate::pd::PdClient; 23 | use crate::proto::kvrpcpb; 24 | use crate::region::RegionWithLeader; 25 | use crate::BoundRange; 26 | use crate::Key; 27 | use crate::Result; 28 | 29 | #[derive(new, Clone)] 30 | pub struct RegionStore { 31 | pub region_with_leader: RegionWithLeader, 32 | pub client: Arc, 33 | } 34 | 35 | #[derive(new, Clone)] 36 | pub struct Store { 37 | pub client: Arc, 38 | } 39 | 40 | /// Maps keys to a stream of stores. `key_data` must be sorted in increasing order 41 | pub fn region_stream_for_keys( 42 | key_data: impl Iterator + Send + Sync + 'static, 43 | pd_client: Arc, 44 | ) -> BoxStream<'static, Result<(Vec, RegionWithLeader)>> 45 | where 46 | PdC: PdClient, 47 | K: AsRef + Into + Send + Sync + 'static, 48 | KOut: Send + Sync + 'static, 49 | { 50 | pd_client.clone().group_keys_by_region(key_data) 51 | } 52 | 53 | #[allow(clippy::type_complexity)] 54 | pub fn region_stream_for_range( 55 | range: (Vec, Vec), 56 | pd_client: Arc, 57 | ) -> BoxStream<'static, Result<((Vec, Vec), RegionWithLeader)>> { 58 | let bnd_range = if range.1.is_empty() { 59 | BoundRange::range_from(range.0.clone().into()) 60 | } else { 61 | BoundRange::from(range.clone()) 62 | }; 63 | pd_client 64 | .regions_for_range(bnd_range) 65 | .map_ok(move |region| { 66 | let region_range = region.range(); 67 | let result_range = range_intersection( 68 | region_range, 69 | (range.0.clone().into(), range.1.clone().into()), 70 | ); 71 | ((result_range.0.into(), result_range.1.into()), region) 72 | }) 73 | .boxed() 74 | } 75 | 76 | /// The range used for request should be the intersection of `region_range` and `range`. 77 | fn range_intersection(region_range: (Key, Key), range: (Key, Key)) -> (Key, Key) { 78 | let (lower, upper) = region_range; 79 | let up = if upper.is_empty() { 80 | range.1 81 | } else if range.1.is_empty() { 82 | upper 83 | } else { 84 | min(upper, range.1) 85 | }; 86 | (max(lower, range.0), up) 87 | } 88 | 89 | pub fn region_stream_for_ranges( 90 | ranges: Vec, 91 | pd_client: Arc, 92 | ) -> BoxStream<'static, Result<(Vec, RegionWithLeader)>> { 93 | pd_client.clone().group_ranges_by_region(ranges) 94 | } 95 | -------------------------------------------------------------------------------- /src/store/request.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use std::any::Any; 4 | use std::time::Duration; 5 | 6 | use async_trait::async_trait; 7 | use tonic::transport::Channel; 8 | use tonic::IntoRequest; 9 | 10 | use crate::proto::kvrpcpb; 11 | use crate::proto::tikvpb::tikv_client::TikvClient; 12 | use crate::store::RegionWithLeader; 13 | use crate::Error; 14 | use crate::Result; 15 | 16 | #[async_trait] 17 | pub trait Request: Any + Sync + Send + 'static { 18 | async fn dispatch( 19 | &self, 20 | client: &TikvClient, 21 | timeout: Duration, 22 | ) -> Result>; 23 | fn label(&self) -> &'static str; 24 | fn as_any(&self) -> &dyn Any; 25 | fn set_leader(&mut self, leader: &RegionWithLeader) -> Result<()>; 26 | fn set_api_version(&mut self, api_version: kvrpcpb::ApiVersion); 27 | } 28 | 29 | macro_rules! impl_request { 30 | ($name: ident, $fun: ident, $label: literal) => { 31 | #[async_trait] 32 | impl Request for kvrpcpb::$name { 33 | async fn dispatch( 34 | &self, 35 | client: &TikvClient, 36 | timeout: Duration, 37 | ) -> Result> { 38 | let mut req = self.clone().into_request(); 39 | req.set_timeout(timeout); 40 | client 41 | .clone() 42 | .$fun(req) 43 | .await 44 | .map(|r| Box::new(r.into_inner()) as Box) 45 | .map_err(Error::GrpcAPI) 46 | } 47 | 48 | fn label(&self) -> &'static str { 49 | $label 50 | } 51 | 52 | fn as_any(&self) -> &dyn Any { 53 | self 54 | } 55 | 56 | fn set_leader(&mut self, leader: &RegionWithLeader) -> Result<()> { 57 | let ctx = self.context.get_or_insert(kvrpcpb::Context::default()); 58 | let leader_peer = leader.leader.as_ref().ok_or(Error::LeaderNotFound { 59 | region: leader.ver_id(), 60 | })?; 61 | ctx.region_id = leader.region.id; 62 | ctx.region_epoch = leader.region.region_epoch.clone(); 63 | ctx.peer = Some(leader_peer.clone()); 64 | Ok(()) 65 | } 66 | 67 | fn set_api_version(&mut self, api_version: kvrpcpb::ApiVersion) { 68 | let ctx = self.context.get_or_insert(kvrpcpb::Context::default()); 69 | ctx.api_version = api_version.into(); 70 | } 71 | } 72 | }; 73 | } 74 | 75 | impl_request!(RawGetRequest, raw_get, "raw_get"); 76 | impl_request!(RawBatchGetRequest, raw_batch_get, "raw_batch_get"); 77 | impl_request!(RawGetKeyTtlRequest, raw_get_key_ttl, "raw_get_key_ttl"); 78 | impl_request!(RawPutRequest, raw_put, "raw_put"); 79 | impl_request!(RawBatchPutRequest, raw_batch_put, "raw_batch_put"); 80 | impl_request!(RawDeleteRequest, raw_delete, "raw_delete"); 81 | impl_request!(RawBatchDeleteRequest, raw_batch_delete, "raw_batch_delete"); 82 | impl_request!(RawScanRequest, raw_scan, "raw_scan"); 83 | impl_request!(RawBatchScanRequest, raw_batch_scan, "raw_batch_scan"); 84 | impl_request!(RawDeleteRangeRequest, raw_delete_range, "raw_delete_range"); 85 | impl_request!(RawCasRequest, raw_compare_and_swap, "raw_compare_and_swap"); 86 | impl_request!(RawCoprocessorRequest, raw_coprocessor, "raw_coprocessor"); 87 | 88 | impl_request!(GetRequest, kv_get, "kv_get"); 89 | impl_request!(ScanRequest, kv_scan, "kv_scan"); 90 | impl_request!(PrewriteRequest, kv_prewrite, "kv_prewrite"); 91 | impl_request!(CommitRequest, kv_commit, "kv_commit"); 92 | impl_request!(CleanupRequest, kv_cleanup, "kv_cleanup"); 93 | impl_request!(BatchGetRequest, kv_batch_get, "kv_batch_get"); 94 | impl_request!(BatchRollbackRequest, kv_batch_rollback, "kv_batch_rollback"); 95 | impl_request!( 96 | PessimisticRollbackRequest, 97 | kv_pessimistic_rollback, 98 | "kv_pessimistic_rollback" 99 | ); 100 | impl_request!(ResolveLockRequest, kv_resolve_lock, "kv_resolve_lock"); 101 | impl_request!(ScanLockRequest, kv_scan_lock, "kv_scan_lock"); 102 | impl_request!( 103 | PessimisticLockRequest, 104 | kv_pessimistic_lock, 105 | "kv_pessimistic_lock" 106 | ); 107 | impl_request!(TxnHeartBeatRequest, kv_txn_heart_beat, "kv_txn_heart_beat"); 108 | impl_request!( 109 | CheckTxnStatusRequest, 110 | kv_check_txn_status, 111 | "kv_check_txn_status" 112 | ); 113 | impl_request!( 114 | CheckSecondaryLocksRequest, 115 | kv_check_secondary_locks, 116 | "kv_check_secondary_locks_request" 117 | ); 118 | impl_request!(GcRequest, kv_gc, "kv_gc"); 119 | impl_request!(DeleteRangeRequest, kv_delete_range, "kv_delete_range"); 120 | impl_request!( 121 | UnsafeDestroyRangeRequest, 122 | unsafe_destroy_range, 123 | "unsafe_destroy_range" 124 | ); 125 | -------------------------------------------------------------------------------- /src/timestamp.rs: -------------------------------------------------------------------------------- 1 | //! A timestamp returned from the timestamp oracle. 2 | //! 3 | //! The version used in transactions can be converted from a timestamp. 4 | //! The lower 18 (PHYSICAL_SHIFT_BITS) bits are the logical part of the timestamp. 5 | //! The higher bits of the version are the physical part of the timestamp. 6 | 7 | use std::convert::TryInto; 8 | 9 | pub use crate::proto::pdpb::Timestamp; 10 | 11 | const PHYSICAL_SHIFT_BITS: i64 = 18; 12 | const LOGICAL_MASK: i64 = (1 << PHYSICAL_SHIFT_BITS) - 1; 13 | 14 | /// A helper trait to convert a Timestamp to and from an u64. 15 | /// 16 | /// Currently the only implmentation of this trait is [`Timestamp`](Timestamp) in TiKV. 17 | /// It contains a physical part (first 46 bits) and a logical part (last 18 bits). 18 | pub trait TimestampExt: Sized { 19 | /// Convert the timestamp to u64. 20 | fn version(&self) -> u64; 21 | /// Convert u64 to a timestamp. 22 | fn from_version(version: u64) -> Self; 23 | /// Convert u64 to an optional timestamp, where `0` represents no timestamp. 24 | fn try_from_version(version: u64) -> Option; 25 | } 26 | 27 | impl TimestampExt for Timestamp { 28 | fn version(&self) -> u64 { 29 | ((self.physical << PHYSICAL_SHIFT_BITS) + self.logical) 30 | .try_into() 31 | .expect("Overflow converting timestamp to version") 32 | } 33 | 34 | fn from_version(version: u64) -> Self { 35 | let version = version as i64; 36 | Self { 37 | physical: version >> PHYSICAL_SHIFT_BITS, 38 | logical: version & LOGICAL_MASK, 39 | // Now we only support global transactions: suffix_bits: 0, 40 | ..Default::default() 41 | } 42 | } 43 | 44 | fn try_from_version(version: u64) -> Option { 45 | if version == 0 { 46 | None 47 | } else { 48 | Some(Self::from_version(version)) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/transaction/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | //! Transactional related functionality. 4 | //! 5 | //! Using the [`TransactionClient`](client::Client) you can utilize TiKV's transactional interface. 6 | //! 7 | //! This interface offers SQL-like transactions on top of the raw interface. 8 | //! 9 | //! **Warning:** It is not advisable to use both raw and transactional functionality in the same keyspace. 10 | 11 | pub use client::Client; 12 | pub(crate) use lock::resolve_locks; 13 | pub(crate) use lock::HasLocks; 14 | pub use snapshot::Snapshot; 15 | pub use transaction::CheckLevel; 16 | #[doc(hidden)] 17 | pub use transaction::HeartbeatOption; 18 | pub use transaction::Mutation; 19 | pub use transaction::Transaction; 20 | pub use transaction::TransactionOptions; 21 | 22 | mod buffer; 23 | mod client; 24 | mod lock; 25 | pub mod lowering; 26 | mod requests; 27 | pub use lock::LockResolver; 28 | pub use lock::ResolveLocksContext; 29 | pub use lock::ResolveLocksOptions; 30 | mod snapshot; 31 | #[allow(clippy::module_inception)] 32 | mod transaction; 33 | -------------------------------------------------------------------------------- /src/transaction/snapshot.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | use derive_new::new; 4 | use log::{debug, trace}; 5 | 6 | use crate::BoundRange; 7 | use crate::Key; 8 | use crate::KvPair; 9 | use crate::Result; 10 | use crate::Transaction; 11 | use crate::Value; 12 | 13 | /// A read-only transaction which reads at the given timestamp. 14 | /// 15 | /// It behaves as if the snapshot was taken at the given timestamp, 16 | /// i.e. it can read operations happened before the timestamp, 17 | /// but ignores operations after the timestamp. 18 | /// 19 | /// See the [Transaction](struct@crate::Transaction) docs for more information on the methods. 20 | #[derive(new)] 21 | pub struct Snapshot { 22 | transaction: Transaction, 23 | } 24 | 25 | impl Snapshot { 26 | /// Get the value associated with the given key. 27 | pub async fn get(&mut self, key: impl Into) -> Result> { 28 | trace!("invoking get request on snapshot"); 29 | self.transaction.get(key).await 30 | } 31 | 32 | /// Check whether the key exists. 33 | pub async fn key_exists(&mut self, key: impl Into) -> Result { 34 | debug!("invoking key_exists request on snapshot"); 35 | self.transaction.key_exists(key).await 36 | } 37 | 38 | /// Get the values associated with the given keys. 39 | pub async fn batch_get( 40 | &mut self, 41 | keys: impl IntoIterator>, 42 | ) -> Result> { 43 | debug!("invoking batch_get request on snapshot"); 44 | self.transaction.batch_get(keys).await 45 | } 46 | 47 | /// Scan a range, return at most `limit` key-value pairs that lying in the range. 48 | pub async fn scan( 49 | &mut self, 50 | range: impl Into, 51 | limit: u32, 52 | ) -> Result> { 53 | debug!("invoking scan request on snapshot"); 54 | self.transaction.scan(range, limit).await 55 | } 56 | 57 | /// Scan a range, return at most `limit` keys that lying in the range. 58 | pub async fn scan_keys( 59 | &mut self, 60 | range: impl Into, 61 | limit: u32, 62 | ) -> Result> { 63 | debug!("invoking scan_keys request on snapshot"); 64 | self.transaction.scan_keys(range, limit).await 65 | } 66 | 67 | /// Similar to scan, but in the reverse direction. 68 | pub async fn scan_reverse( 69 | &mut self, 70 | range: impl Into, 71 | limit: u32, 72 | ) -> Result> { 73 | debug!("invoking scan_reverse request on snapshot"); 74 | self.transaction.scan_reverse(range, limit).await 75 | } 76 | 77 | /// Similar to scan_keys, but in the reverse direction. 78 | pub async fn scan_keys_reverse( 79 | &mut self, 80 | range: impl Into, 81 | limit: u32, 82 | ) -> Result> { 83 | debug!("invoking scan_keys_reverse request on snapshot"); 84 | self.transaction.scan_keys_reverse(range, limit).await 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/util/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | /// Extension trait to iterators to add `flat_map_ok`. 4 | pub trait FlatMapOkIterExt: Iterator + Sized { 5 | /// Flattens an iterator of iterators into a single iterator. The outer iterator returns a Result; 6 | /// if there is any error, that error is preserved in the output. Any Ok values are fed into the 7 | /// function `f` which must produce an iterator (or value that can be converted into an iterator), 8 | /// that iterator is iterated into the output with each value wrapped in an `Ok`. 9 | fn flat_map_ok(self, f: F) -> FlatMapOk 10 | where 11 | Self: Iterator>, 12 | U: IntoIterator, 13 | F: FnMut(I) -> U, 14 | Ti: Iterator, 15 | { 16 | FlatMapOk { 17 | iter: self.fuse(), 18 | frontiter: None, 19 | f, 20 | } 21 | } 22 | } 23 | 24 | impl FlatMapOkIterExt for I {} 25 | 26 | /// Iterator for `flat_map_ok`. 27 | pub struct FlatMapOk { 28 | iter: std::iter::Fuse, 29 | frontiter: Option>, 30 | f: F, 31 | } 32 | 33 | // FIXME: implement other iterator methods like size_hint, etc. 34 | impl< 35 | T: IntoIterator, 36 | U: Iterator>, 37 | F: FnMut(I) -> T, 38 | Ti: Iterator, 39 | I, 40 | E, 41 | > Iterator for FlatMapOk 42 | { 43 | type Item = std::result::Result; 44 | 45 | fn next(&mut self) -> Option { 46 | loop { 47 | match &mut self.frontiter { 48 | Some(Ok(inner)) => match inner.next() { 49 | None => self.frontiter = None, 50 | Some(elt) => return Some(Ok(elt)), 51 | }, 52 | Some(Err(_)) => { 53 | let e = self.frontiter.take().unwrap().err().unwrap(); 54 | return Some(Err(e)); 55 | } 56 | None => {} 57 | } 58 | match self.iter.next() { 59 | None => return None, 60 | Some(Ok(inner)) => self.frontiter = Some(Ok((self.f)(inner).into_iter())), 61 | Some(Err(e)) => self.frontiter = Some(Err(e)), 62 | } 63 | } 64 | } 65 | } 66 | 67 | #[cfg(test)] 68 | mod test { 69 | use super::*; 70 | 71 | #[test] 72 | fn test_iter_and_collect() { 73 | let result: Result, ()> = Vec::new() 74 | .into_iter() 75 | .flat_map_ok(|i| Some(i).into_iter()) 76 | .collect(); 77 | assert_eq!(result.unwrap(), Vec::::new()); 78 | 79 | let result: Result, ()> = vec![Result::::Ok(0), Ok(1), Ok(2)] 80 | .into_iter() 81 | .flat_map_ok(|i| Some(i).into_iter()) 82 | .collect(); 83 | assert_eq!(result.unwrap(), vec![0, 1, 2]); 84 | 85 | let result: Result, ()> = vec![Result::::Ok(0), Err(()), Ok(2)] 86 | .into_iter() 87 | .flat_map_ok(|i| Some(i).into_iter()) 88 | .collect(); 89 | assert_eq!(result, Err(())); 90 | 91 | let result: Vec> = vec![Result::::Ok(0), Err(()), Ok(2)] 92 | .into_iter() 93 | .flat_map_ok(|i| vec![i, i, i].into_iter()) 94 | .collect(); 95 | assert_eq!( 96 | result, 97 | vec![Ok(0), Ok(0), Ok(0), Err(()), Ok(2), Ok(2), Ok(2)] 98 | ); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/util/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. 2 | 3 | pub mod iter; 4 | -------------------------------------------------------------------------------- /taplo.toml: -------------------------------------------------------------------------------- 1 | ## https://taplo.tamasfe.dev/configuration/file.html 2 | 3 | include = ["**/Cargo.toml"] 4 | 5 | [formatting] 6 | # Align consecutive entries vertically. 7 | align_entries = false 8 | # Append trailing commas for multi-line arrays. 9 | array_trailing_comma = true 10 | # Expand arrays to multiple lines that exceed the maximum column width. 11 | array_auto_expand = true 12 | # Collapse arrays that don't exceed the maximum column width and don't contain comments. 13 | array_auto_collapse = false 14 | # Omit white space padding from single-line arrays 15 | compact_arrays = true 16 | # Omit white space padding from the start and end of inline tables. 17 | compact_inline_tables = false 18 | # Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account. 19 | # Note that this is not set in stone, and works on a best-effort basis. 20 | column_width = 120 21 | # Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented. 22 | indent_tables = false 23 | # The substring that is used for indentation, should be tabs or spaces (but technically can be anything). 24 | indent_string = ' ' 25 | # Add trailing newline at the end of the file if not present. 26 | trailing_newline = true 27 | # Alphabetically reorder keys that are not separated by empty lines. 28 | reorder_keys = false 29 | # Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped. 30 | allowed_blank_lines = 1 31 | # Use CRLF for line endings. 32 | crlf = false 33 | 34 | [[rule]] 35 | keys = ["dependencies", "dev-dependencies", "build-dependencies"] 36 | formatting = { reorder_keys = true } 37 | -------------------------------------------------------------------------------- /tests/common/ctl.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0. 2 | //! The module provides some utility functions to control and get information 3 | //! from PD, using its HTTP API. 4 | 5 | use tikv_client::Error; 6 | 7 | use super::pd_addrs; 8 | use crate::common::Result; 9 | 10 | pub async fn get_region_count() -> Result { 11 | let res = reqwest::get(format!("http://{}/pd/api/v1/regions", pd_addrs()[0])) 12 | .await 13 | .map_err(|e| Error::StringError(e.to_string()))?; 14 | 15 | let body = res 16 | .text() 17 | .await 18 | .map_err(|e| Error::StringError(e.to_string()))?; 19 | let value: serde_json::Value = serde_json::from_str(body.as_ref()).unwrap(); 20 | value["count"] 21 | .as_u64() 22 | .ok_or_else(|| Error::StringError("pd region count does not return an integer".to_owned())) 23 | } 24 | --------------------------------------------------------------------------------