├── .editorconfig ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .gitmodules ├── .travis.yml ├── CHANGES.md ├── Cargo.toml ├── LICENSE.txt ├── README.md ├── cityhash-rs ├── Cargo.toml ├── benches │ └── bench.rs └── src │ └── lib.rs ├── cityhash ├── .gitignore ├── Cargo.toml ├── build.rs └── src │ ├── cc │ ├── BUCK │ ├── CMakeLists.txt │ ├── COPYING │ ├── city.cc │ ├── city.h │ ├── citycrc.h │ └── config.h │ └── lib.rs ├── clickhouse-driver ├── .gitignore ├── Cargo.toml ├── examples │ ├── array.rs │ ├── bulk-insert.rs │ ├── insert-select.rs │ └── select.rs ├── src │ ├── client.rs │ ├── code.rs │ ├── compression │ │ └── mod.rs │ ├── errors.rs │ ├── lib.rs │ ├── pool │ │ ├── builder.rs │ │ ├── disconnect.rs │ │ ├── mod.rs │ │ ├── options.rs │ │ ├── recycler.rs │ │ └── util.rs │ ├── prelude.rs │ ├── protocol │ │ ├── block.rs │ │ ├── code.rs │ │ ├── column.rs │ │ ├── command.rs │ │ ├── decoder.rs │ │ ├── encoder.rs │ │ ├── insert.rs │ │ ├── mod.rs │ │ ├── packet.rs │ │ ├── query.rs │ │ ├── simd.rs │ │ └── value.rs │ ├── sync │ │ ├── mod.rs │ │ └── waker_set.rs │ └── types │ │ ├── decimal.rs │ │ ├── mod.rs │ │ └── parser.rs └── tests │ ├── common │ └── mod.rs │ ├── insert.rs │ ├── pool.rs │ └── query.rs ├── derive ├── Cargo.toml └── src │ └── lib.rs ├── docker-compose.yml ├── lz4a ├── Cargo.toml ├── benches │ └── bench.rs ├── build.rs └── src │ └── lib.rs └── tests ├── bench ├── .gitignore ├── BENCHMARK.md ├── clickhouse-rs │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── go │ ├── go.mod │ ├── go.sum │ └── main.go ├── java │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── github │ │ │ └── ddulesov │ │ │ └── App.java │ │ └── test │ │ └── java │ │ └── com │ │ └── github │ │ └── ddulesov │ │ └── AppTest.java └── py │ └── bench.py ├── create.sql ├── createx.sql ├── main.tsv └── mainx.tsv /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | end_of_line = lf 3 | insert_final_newline = true 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 4 -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | services: 17 | clickhouse: 18 | # Docker Hub image 19 | image: yandex/clickhouse-server 20 | ports: 21 | - 8123:8123 22 | - 9000:9000 23 | steps: 24 | - uses: actions/checkout@v2 25 | with: 26 | submodules: true 27 | - name: CreateClickhouseDB 28 | run: | 29 | curl --data-binary @tests/create.sql -u 'default:' 'http://localhost:8123/default' 30 | curl --data-binary @tests/createx.sql -u 'default:' 'http://localhost:8123/default' 31 | curl --data-binary @tests/main.tsv -u 'default:' 'http://localhost:8123/default?query=INSERT%20INTO%20main(i64,u64,i32,u32,i16,u16,i8,u8,f32,f64,title,lcs,mon,d,t,dt64,uuid,e8,e16,ip4,ip6,n,dm1,dm2)%20FORMAT%20TabSeparated' 32 | curl --data-binary @tests/mainx.tsv -u 'default:' 'http://localhost:8123/default?query=INSERT%20INTO%20mainx(i64,a8,a16,a32,a64,a8d,aip4,aip6,ad,adt,adc,lcs)%20FORMAT%20TabSeparated' 33 | - name: Build 34 | run: cargo build --verbose 35 | - name: Run Clippy 36 | run: cargo clippy --all-targets 37 | - name: Run Tests 38 | run: cargo test --all 39 | - name: Build Examples 40 | run: cargo build --examples 41 | - name: Int128 Build 42 | run: | 43 | cd clickhouse-driver 44 | cargo build --no-default-features --features 'cityhash int128' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | .idea 4 | scripts -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lz4a/liblz4"] 2 | path = lz4a/liblz4 3 | url = https://github.com/lz4/lz4.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: rust 3 | branches: 4 | except: 5 | - dev 6 | - patch 7 | addons: 8 | apt: 9 | packages: 10 | - curl 11 | rust: 12 | - stable 13 | - beta 14 | - nightly 15 | matrix: 16 | allow_failures: 17 | - rust: nightly 18 | fast_finish: true 19 | services: 20 | - docker 21 | before_install: 22 | - docker-compose up -d 23 | - rustup component add clippy 24 | # - cargo install -f cargo-travis 25 | - export PATH=$HOME/.cargo/bin:$PATH; 26 | - curl --data-binary @tests/create.sql -u 'default:' 'http://localhost:8123/default' 27 | - curl --data-binary @tests/createx.sql -u 'default:' 'http://localhost:8123/default' 28 | - curl --data-binary @tests/main.tsv -u 'default:' 'http://localhost:8123/default?query=INSERT%20INTO%20main(i64,u64,i32,u32,i16,u16,i8,u8,f32,f64,title,lcs,mon,d,t,dt64,uuid,e8,e16,ip4,ip6,n,dm1,dm2)%20FORMAT%20TabSeparated' 29 | - curl --data-binary @tests/mainx.tsv -u 'default:' 'http://localhost:8123/default?query=INSERT%20INTO%20mainx(i64,a8,a16,a32,a64,a8d,aip4,aip6,ad,adt,adc,lcs)%20FORMAT%20TabSeparated' 30 | script: 31 | - cargo build 32 | - cargo clippy --all-targets 33 | - cargo test --all --verbose 34 | 35 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 2 | ## CHANGELOG 3 | 4 | ### unreleased 5 | * Add `LowCardinality(String)` data type 6 | 7 | 8 | ## TODO 9 | * [ ] Add string description to `ConversionError` 10 | * [ ] Draw up `Array(X)` column type 11 | * [ ] Optimize out `LZ4ReadAdapter` 12 | * [ ] Make `LowCardinality` writable 13 | * [ ] Add more query/insert tests 14 | * [ ] Add tls support 15 | * [ ] Add benchmarks 16 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "clickhouse-driver","cityhash","lz4a","derive","cityhash-rs" 5 | ] 6 | 7 | [profile.release] 8 | opt-level = 3 9 | overflow-checks = false 10 | 11 | [profile.bench] 12 | opt-level = 3 13 | overflow-checks = false 14 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) "dmitry dulesov " 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 15 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ClickHouse Rust Client # 2 | [![Build Status](https://travis-ci.org/ddulesov/clickhouse_driver.svg?branch=master)](https://travis-ci.org/ddulesov/clickhouse_driver) 3 | ![Rust](https://github.com/ddulesov/clickhouse_driver/workflows/Rust/badge.svg?branch=master) 4 | 5 | Asynchronous pure rust tokio-based Clickhouse client library 6 | 7 | development status: **alpha** 8 | 9 | Tested on Linux x86-64 (ubuntu 20.04 LTS), Windows 10. 10 | 11 | ### Why ### 12 | 13 | * To create small and robust driver for Clickhouse, fast open-source column oriented database 14 | * To learn rust concurrency and zero-cost abstraction 15 | 16 | ### Supported features ### 17 | 18 | * Asynchronous tokio-based engine 19 | * Native Clickhouse protocol 20 | * LZ4 compression 21 | * Persistent connection pool 22 | * Simple row to object mapper 23 | * Date | DateTime | DateTime64- read/write 24 | * (U)Int(8|16|32|64) - read/write 25 | * Float32 | Float64 - read/write 26 | * UUID - read/write 27 | * String | FixedString- read/write 28 | * Ipv4 | Ipv6 - read/write 29 | * Nullable(*) - read/write 30 | * Decimal - read/write 31 | * Enum8, Enum16 - read/write 32 | 33 | ### Use cases ### 34 | 35 | * Make query using SQL syntax supported by Clickhouse Server 36 | * Execute arbitrary DDL commands 37 | * Query Server status 38 | * Insert into Clickhouse Server big (possibly continues ) data stream 39 | * Load-balancing using round-robin method 40 | 41 | ### Quick start ### 42 | Require rust 1.42. 43 | 44 | The package has not published in crates.io. 45 | Download source from [home git](https://github.com/ddulesov/clickhouse_driver) 46 | ```bash 47 | git module update --init --recursive 48 | ``` 49 | 50 | Building requires rust 1.41 stable or nightly, 51 | tokio-0.2.x. 52 | 53 | - Add next lines into the `dependencies` section of your `Cargo.toml`: 54 | 55 | ```toml 56 | clickhouse-driver = { version="0.1.0-alpha.3", path="../path_to_package/clickhouse-driver"} 57 | clickhouse-driver-lz4 = { version="0.1.0", path="../path_to_package/lz4a"} 58 | clickhouse-driver-cthrs = { version="0.1.0", path="../path_to_package/cityhash-rs"} 59 | 60 | ``` 61 | - Add usage in main.rs 62 | ```rust 63 | extern crate clickhouse_driver; 64 | use clickhouse_driver::prelude::*; 65 | ``` 66 | 67 | to connect to server provide connection url 68 | ``` 69 | tcp://username:password@localhost/database?paramname=paramvalue&... 70 | ``` 71 | for example 72 | ``` 73 | tcp://user:default@localhost/log?ping_timout=200ms&execute_timeout=5s&query_timeout=20s&pool_max=4&compression=lz4 74 | ``` 75 | ### Supported URL parameters 76 | * `compression` - accepts 'lz4' or 'none'. 77 | lz4 - fast and efficient compression method. 78 | It can significantly reduce transmitted data size and time if used for 79 | big data chunks. For small data it's better to choose none compression; 80 | 81 | * `connection_timeout` - timeout for establishing connection. 82 | Default is 500ms; 83 | 84 | * `execute_timeout` - timeout for waiting result of **execute** method call 85 | If the execute used for alter huge table it can take 86 | long time to complete. In this case set this parameter to appropriate 87 | value. In other cases leave the default value (180 sec); 88 | 89 | * `query_timout` - timeout for waiting response from the server with 90 | next block of data in **query** call. 91 | Note. Large data query may take long time. This timeout requires that only 92 | one chunk of data will receive until the end of timeout. 93 | Default value is 180sec; 94 | 95 | * `insert_timeout` - timeout for waiting result of `insert` call 96 | insert method call returns error if the server does not receive 97 | message until the end of insert_timeout. 98 | As insert data processing is asynchronous it doesn't include server block processing time. 99 | Default value is 180 sec; 100 | 101 | * `ping_timout` - wait before ping response. 102 | The host will be considered unavailable if the server 103 | does not return pong response until the end of ping_timeout; 104 | 105 | * `retry_timeout` - the number of seconds to wait before send next ping 106 | if the server does not return; 107 | 108 | * `ping_before_query` - 1 (default) or 0. This option if set 109 | requires the driver to check Clickhouse server responsibility 110 | after returning connection from pool; 111 | 112 | * `pool_min` - minimal connection pool size. 113 | the number of idle connections that can be kept in the pool; 114 | Default value is 2; 115 | 116 | * `pool_max` - maximum number of established connections that the pool 117 | can issued. If the task require new connection while the pool reaches the maximum 118 | and there is not idle connection then this task will be put in waiting queue. 119 | Default value is 10; 120 | 121 | * `readonly` - 0 (default) |1|2. 122 | 0 - all commands allowed. 123 | 2- select queries and change settings, 124 | 1 - only select queries ; 125 | 126 | * `keepalive` - keepalive TCP option; 127 | 128 | * `host` - alternative host(s) 129 | 130 | All timeout parameters accept integer number - the number of seconds. 131 | To specify timeout in milliseconds add `ms` at the end. 132 | Examples: 133 | - `200ms` ( 200 mseconds ) 134 | - `20` ( 20 seconds ) 135 | - `10s` ( 10 seconds ) 136 | 137 | ### Example 138 | ```rust 139 | 140 | struct Blob { 141 | id: u64, 142 | url: String, 143 | date: ServerDate, 144 | client: Uuid, 145 | ip: Ipv4Addr, 146 | value: Decimal32, 147 | } 148 | 149 | impl Deserialize for Blob { 150 | fn deserialize(row: Row) -> errors::Result { 151 | let err = || errors::ConversionError::UnsupportedConversion; 152 | 153 | let id: u64 = row.value(0)?.ok_or_else(err)?; 154 | let url: &str = row.value(1)?.ok_or_else(err)?; 155 | let date: ServerDate = row.value(2)?.ok_or_else(err)?; 156 | let client: Uuid = row.value(3)?.ok_or_else(err)?; 157 | let ip = row.value(4)?.ok_or_else(err)?; 158 | let value: Decimal32 = row.value(5)?.ok_or_else(err)?; 159 | 160 | Ok(Blob { 161 | id, 162 | date, 163 | client, 164 | value, 165 | url: url.to_string(), 166 | ip, 167 | }) 168 | } 169 | } 170 | 171 | #[tokio::main] 172 | async fn main() -> Result<(), io::Error> { 173 | 174 | // CREATE TABLE IF NOT EXISTS blob ( 175 | // id UInt64, 176 | // url String, 177 | // date DateTime, 178 | // client UUID, 179 | // ip IPv4 180 | // ) ENGINE=MergeTree PARTITION BY id ORDER BY date 181 | 182 | let database_url = 183 | env::var("DATABASE_URL").unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 184 | 185 | let pool = Pool::create(database_url.as_str())?; 186 | { 187 | let mut conn = pool.connection().await?; 188 | conn.ping().await?; 189 | 190 | let mut result = conn 191 | .query("SELECT id, url, date, client, ip FROM blob WHERE id=150 ORDER BY date LIMIT 30000") 192 | .await?; 193 | 194 | while let Some(block) = result.next().await? { 195 | for blob in block.iter::() { 196 | ... 197 | } 198 | } 199 | } 200 | 201 | Ok(()) 202 | } 203 | ``` 204 | 205 | * For more examples see clickhouse-driver/examples/ directory* 206 | 207 | ### Known issues and limitations ### 208 | 209 | * Doesn't support multidimensional Array, 210 | * Array data types readonly 211 | * LowCardinality - readonly and just String base type 212 | * Insert method support only limited data types 213 | `insert` requires that inserted data exactly matches table column type 214 | - Int8(16|32|64) - i8(16|32|64) 215 | - UInt8(16|32|64) - u8(16|32|64) 216 | - Float32 - f32 217 | - Float64 - f64 218 | - Date - chrono::Date 219 | - DateTime - chrono::DateTime 220 | - UUID - Uuid 221 | - IPv4 - AddrIpv4 222 | - IPv6 - AddrIpv6 223 | - String - &str,String, or &[u8] 224 | - Enum8|16 - &str or String. Also, i16 value of enum index can be retrieved. 225 | 226 | ### Roadmap 227 | 228 | * `Array` column data type - read/write 229 | * `Tuple` - no plans to support 230 | * `AggregateFunction` - no plans to support 231 | * `LowCardinality` - add write support, extend it to `Date`, `DateTime` types 232 | * `Serde` - Row serializer/deserializer interface in addition to ad-hoc one 233 | * `TLS` 234 | * C-API ? 235 | * `async_std` runtime 236 | 237 | 238 | -------------------------------------------------------------------------------- /cityhash-rs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-driver-cthrs" 3 | description = "rust CityHash for ClickHouse Asynchronous Driver." 4 | version = "0.1.1" 5 | authors = ["dmitry dulesov "] 6 | edition = "2018" 7 | license = "MIT" 8 | repository = "https://github.com/ddulesov/clickhouse_driver" 9 | 10 | [dependencies] 11 | 12 | [dev-dependencies] 13 | clickhouse-driver-cth = { version="0.1.0", path="../cityhash"} 14 | rand = "0.7.3" 15 | 16 | [features] 17 | default = [] 18 | simd = [] 19 | 20 | -------------------------------------------------------------------------------- /cityhash-rs/benches/bench.rs: -------------------------------------------------------------------------------- 1 | #![cfg(rustc_nightly)] 2 | #![feature(test)] 3 | #![feature(core_intrinsics)] 4 | 5 | extern crate test; 6 | 7 | #[cfg(target_arch = "x86_64")] 8 | use core::arch::x86_64::_mm_crc32_u64; 9 | //use core::intrinsics::size_of; 10 | use test::Bencher; 11 | 12 | use rand::prelude::*; 13 | 14 | use clickhouse_cityhash::{_CityHash128, c_char}; 15 | use clickhouse_cityhash_rs::*; 16 | 17 | #[inline] 18 | fn city_hash_ref(source: &[u8]) -> Pair { 19 | let h = unsafe { _CityHash128(source.as_ptr() as *const c_char, source.len()) }; 20 | Pair(h.0, h.1) 21 | } 22 | 23 | #[target_feature(enable = "sse4.2")] 24 | unsafe fn crc32_64(mut src: &[u8]) -> u64 { 25 | let mut seed: u64 = 0; 26 | while src.len() >= 8 { 27 | let a = fetch64(src); 28 | seed = _mm_crc32_u64(seed, a); 29 | src = &src[8..]; 30 | } 31 | 32 | if !src.is_empty() { 33 | let mut buf = [0u8; 8]; 34 | let s = std::cmp::min(8, src.len()); 35 | 36 | buf[0..s].copy_from_slice(&src[0..s]); 37 | let a = u64::from_le_bytes(buf); 38 | seed = _mm_crc32_u64(seed, a); 39 | } 40 | 41 | seed 42 | } 43 | 44 | fn data(len: usize) -> Vec { 45 | let mut b: Vec = Vec::with_capacity(len as usize); 46 | unsafe { 47 | b.set_len(len as usize); 48 | } 49 | 50 | rand::thread_rng().fill(&mut b[..]); 51 | b 52 | } 53 | 54 | #[bench] 55 | fn bench_1k_rust(b: &mut Bencher) { 56 | let input = data(1024); 57 | let mut out = city_hash_128(&input[..]); 58 | b.iter(|| { 59 | out = city_hash_128(&input[..]); 60 | }); 61 | } 62 | 63 | #[bench] 64 | fn bench_1k_cpp(b: &mut Bencher) { 65 | let input = data(1024); 66 | let mut out = city_hash_ref(&input[..]); 67 | b.iter(|| { 68 | out = city_hash_ref(&input[..]); 69 | }); 70 | } 71 | 72 | #[bench] 73 | fn bench_64k_rust(b: &mut Bencher) { 74 | let input = data(64 * 1024); 75 | let mut out = city_hash_128(&input[..]); 76 | b.iter(|| { 77 | out = city_hash_128(&input[..]); 78 | }); 79 | } 80 | 81 | #[bench] 82 | fn bench_64k_cpp(b: &mut Bencher) { 83 | let input = data(64 * 1024); 84 | let mut out = city_hash_ref(&input[..]); 85 | b.iter(|| { 86 | out = city_hash_ref(&input[..]); 87 | }); 88 | } 89 | 90 | #[bench] 91 | fn bench_1m_rust(b: &mut Bencher) { 92 | let input = data(1024 * 1024); 93 | let mut out = city_hash_128(&input[..]); 94 | b.iter(|| { 95 | out = city_hash_128(&input[..]); 96 | }); 97 | } 98 | 99 | #[bench] 100 | fn bench_1m_cpp(b: &mut Bencher) { 101 | let input = data(1024 * 1024); 102 | let mut out = city_hash_ref(&input[..]); 103 | b.iter(|| { 104 | out = city_hash_ref(&input[..]); 105 | }); 106 | } 107 | 108 | #[bench] 109 | #[cfg(feature = "simd")] 110 | fn bench_64k_rust_city_crc(b: &mut Bencher) { 111 | let input = data(64 * 1024); 112 | let mut out = unsafe { city_crc(&input[..], 0) }; 113 | b.iter(|| { 114 | out = unsafe { city_crc(&input[..], 0) }; 115 | }); 116 | } 117 | 118 | #[bench] 119 | fn bench_64k_rust_crc32_64(b: &mut Bencher) { 120 | let input = data(64 * 1024); 121 | let mut out = unsafe { crc32_64(&input[..]) }; 122 | b.iter(|| { 123 | out = unsafe { crc32_64(&input[..]) }; 124 | }); 125 | } 126 | -------------------------------------------------------------------------------- /cityhash-rs/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![allow(arithmetic_overflow)] 3 | #![allow(clippy::many_single_char_names)] 4 | #[cfg(test)] 5 | extern crate clickhouse_driver_cth; 6 | #[cfg(test)] 7 | extern crate std; 8 | 9 | use core::mem::size_of; 10 | use core::ptr::read_unaligned; 11 | 12 | #[derive(Debug, PartialEq)] 13 | pub struct Pair(pub u64, pub u64); 14 | 15 | /// Compare Hash with first 16 byte of Clickhouse Packet Header 16 | /// @note it's work only with little endian system only 17 | impl PartialEq<&[u8]> for Pair { 18 | fn eq(&self, other: &&[u8]) -> bool { 19 | other.len() == 16 && (self.0 == fetch64(*other)) && (self.1 == fetch64(&other[8..])) 20 | } 21 | } 22 | 23 | const K0: u64 = 0xc3a5c85c97cb3127; 24 | const K1: u64 = 0xb492b66fbe98f273; 25 | const K2: u64 = 0x9ae16a3b2f90404f; 26 | const K3: u64 = 0xc949d7c7509e6557; 27 | 28 | // #[inline(always)] 29 | // fn rotate(val: u64, shift: u64) -> u64 { 30 | // if shift == 0 { 31 | // val 32 | // } else { 33 | // (val >> shift) | (val << (64 - shift)) 34 | // } 35 | // } 36 | 37 | /// The same as rotate but `shift` must not be eq 0 38 | #[inline(always)] 39 | fn rotate_least(val: u64, shift: u64) -> u64 { 40 | (val >> shift) | (val << (64 - shift)) 41 | } 42 | 43 | #[inline(always)] 44 | fn shift_mix(val: u64) -> u64 { 45 | val ^ (val >> 47) 46 | } 47 | 48 | #[cfg(target_endian = "little")] 49 | #[allow(clippy::cast_ptr_alignment)] 50 | #[inline] 51 | pub fn fetch64(src: &[u8]) -> u64 { 52 | debug_assert!(src.len() >= size_of::()); 53 | let ptr = src.as_ptr() as *const u64; 54 | unsafe { read_unaligned(ptr) } 55 | } 56 | 57 | #[cfg(target_endian = "little")] 58 | #[allow(clippy::cast_ptr_alignment)] 59 | #[inline] 60 | fn fetch32(src: &[u8]) -> u32 { 61 | debug_assert!(src.len() >= size_of::()); 62 | let ptr = src.as_ptr() as *const u32; 63 | unsafe { read_unaligned(ptr) } 64 | } 65 | 66 | #[cfg(not(target_endian = "little"))] 67 | #[allow(clippy::cast_ptr_alignment)] 68 | #[inline] 69 | fn fetch64(src: &[u8]) -> u64 { 70 | debug_assert!(src.len() >= mem::size_of::()); 71 | let ptr = src.as_ptr() as *const u64; 72 | let src = unsafe { read_unaligned(ptr) }; 73 | src.swap_bytes() 74 | } 75 | 76 | #[cfg(not(target_endian = "little"))] 77 | #[allow(clippy::cast_ptr_alignment)] 78 | #[inline] 79 | fn fetch32(src: &[u8]) -> u32 { 80 | debug_assert!(src.len() >= size_of::()); 81 | let ptr = src.as_ptr() as *const u32; 82 | let src = unsafe { read_unaligned(ptr) }; 83 | src.swap_bytes() 84 | } 85 | 86 | fn hash_len0to16(src: &[u8]) -> u64 { 87 | if src.len() > 8 { 88 | let a: u64 = fetch64(src); 89 | let b: u64 = fetch64(&src[src.len() - 8..]); 90 | b ^ hash_len16( 91 | a, 92 | rotate_least(b.wrapping_add(src.len() as u64), src.len() as u64), 93 | ) 94 | } else if src.len() >= 4 { 95 | let a = fetch32(src) as u64; 96 | hash_len16( 97 | (a << 3).wrapping_add(src.len() as u64), 98 | fetch32(&src[src.len() - 4..]) as u64, 99 | ) 100 | } else if !src.is_empty() { 101 | let a: u8 = src[0]; 102 | let b: u8 = src[src.len() >> 1]; 103 | let c: u8 = src[src.len() - 1]; 104 | let y: u64 = (a as u64).wrapping_add((b as u64) << 8); 105 | let z: u64 = (src.len() as u64).wrapping_add((c as u64) << 2); 106 | shift_mix(y.wrapping_mul(K2) ^ z.wrapping_mul(K3)).wrapping_mul(K2) 107 | } else { 108 | K2 109 | } 110 | } 111 | 112 | fn citymurmur(mut src: &[u8], seed: Pair) -> Pair { 113 | let mut a: u64 = seed.0; 114 | let mut b: u64 = seed.1; 115 | let mut c: u64; 116 | let mut d: u64; 117 | 118 | if src.len() <= 16 { 119 | a = shift_mix(a.wrapping_mul(K1)).wrapping_mul(K1); 120 | c = b.wrapping_mul(K1).wrapping_add(hash_len0to16(src)); 121 | d = if src.len() >= 8 { fetch64(src) } else { c }; 122 | d = shift_mix(a.wrapping_add(d)); 123 | } else { 124 | c = hash_len16(fetch64(&src[src.len() - 8..]).wrapping_add(K1), a); 125 | d = hash_len16( 126 | b.wrapping_add(src.len() as u64), 127 | c.wrapping_add(fetch64(&src[src.len() - 16..])), 128 | ); 129 | a = a.wrapping_add(d); 130 | loop { 131 | a ^= shift_mix(fetch64(src).wrapping_mul(K1)).wrapping_mul(K1); 132 | a = a.wrapping_mul(K1); 133 | b ^= a; 134 | c ^= shift_mix(fetch64(&src[8..]).wrapping_mul(K1)).wrapping_mul(K1); 135 | c = c.wrapping_mul(K1); 136 | d ^= c; 137 | src = &src[16..]; 138 | if src.len() <= 16 { 139 | break; 140 | } 141 | } 142 | } 143 | 144 | a = hash_len16(a, c); 145 | b = hash_len16(d, b); 146 | Pair(a ^ b, hash_len16(b, a)) 147 | } 148 | 149 | const KMUL: u64 = 0x9ddfea08eb382d69; 150 | 151 | #[inline(always)] 152 | fn hash_128to64(x: Pair) -> u64 { 153 | let mut a: u64 = (x.0 ^ x.1).wrapping_mul(KMUL); 154 | a = shift_mix(a); 155 | let mut b: u64 = (x.1 ^ a).wrapping_mul(KMUL); 156 | b = shift_mix(b); 157 | b.wrapping_mul(KMUL) 158 | } 159 | 160 | #[inline] 161 | fn hash_len16(u: u64, v: u64) -> u64 { 162 | hash_128to64(Pair(u, v)) 163 | } 164 | 165 | #[inline(always)] 166 | #[allow(non_snake_case)] 167 | fn _weakHashLen32WithSeeds(w: u64, x: u64, y: u64, z: u64, mut a: u64, mut b: u64) -> Pair { 168 | a = a.wrapping_add(w); 169 | b = rotate_least(b.wrapping_add(a).wrapping_add(z), 21); 170 | let c = a; 171 | a = a.wrapping_add(x).wrapping_add(y); 172 | b = b.wrapping_add(rotate_least(a, 44)); 173 | Pair(a.wrapping_add(z), b.wrapping_add(c)) 174 | } 175 | 176 | fn weak_hash_len32with_seeds(src: &[u8], a: u64, b: u64) -> Pair { 177 | _weakHashLen32WithSeeds( 178 | fetch64(src), 179 | fetch64(&src[8..]), 180 | fetch64(&src[16..]), 181 | fetch64(&src[24..]), 182 | a, 183 | b, 184 | ) 185 | } 186 | 187 | fn cityhash128withseed(mut src: &[u8], seed: Pair) -> Pair { 188 | // We expect len >= 128 to be the common case. Keep 56 bytes of state: 189 | // v, w, x, y, and z. 190 | let mut x: u64 = seed.0; 191 | let mut y: u64 = seed.1; 192 | 193 | let mut z: u64 = K1.wrapping_mul(src.len() as u64); 194 | let t: u64 = K1 195 | .wrapping_mul(rotate_least(y ^ K1, 49)) 196 | .wrapping_add(fetch64(src)); 197 | 198 | let mut v = Pair( 199 | t, 200 | K1.wrapping_mul(rotate_least(t, 42)) 201 | .wrapping_add(fetch64(&src[8..])), 202 | ); 203 | let mut w = Pair( 204 | K1.wrapping_mul(rotate_least(y.wrapping_add(z), 35)) 205 | .wrapping_add(x), 206 | K1.wrapping_mul(rotate_least(x.wrapping_add(fetch64(&src[88..])), 53)), 207 | ); 208 | 209 | // This is the same inner loop as CityHash64(), manually unrolled. 210 | loop { 211 | x = K1.wrapping_mul(rotate_least( 212 | x.wrapping_add(y) 213 | .wrapping_add(v.0) 214 | .wrapping_add(fetch64(&src[16..])), 215 | 37, 216 | )); 217 | y = K1.wrapping_mul(rotate_least( 218 | y.wrapping_add(v.1).wrapping_add(fetch64(&src[48..])), 219 | 42, 220 | )); 221 | x ^= w.1; 222 | y ^= v.0; 223 | z = rotate_least(z ^ w.0, 33); 224 | v = weak_hash_len32with_seeds(src, K1.wrapping_mul(v.1), x.wrapping_add(w.0)); 225 | w = weak_hash_len32with_seeds(&src[32..], z.wrapping_add(w.1), y); 226 | core::mem::swap(&mut z, &mut x); 227 | 228 | //next 64 byte block 229 | src = &src[64..]; 230 | 231 | x = K1.wrapping_mul(rotate_least( 232 | x.wrapping_add(y) 233 | .wrapping_add(v.0) 234 | .wrapping_add(fetch64(&src[16..])), 235 | 37, 236 | )); 237 | y = K1.wrapping_mul(rotate_least( 238 | y.wrapping_add(v.1).wrapping_add(fetch64(&src[48..])), 239 | 42, 240 | )); 241 | x ^= w.1; 242 | y ^= v.0; 243 | z = rotate_least(z ^ w.0, 33); 244 | v = weak_hash_len32with_seeds(src, K1.wrapping_mul(v.1), x.wrapping_add(w.0)); 245 | w = weak_hash_len32with_seeds(&src[32..], z.wrapping_add(w.1), y); 246 | core::mem::swap(&mut z, &mut x); 247 | // next 64 bytes 248 | if src.len() < (128 + 64) { 249 | break; 250 | } 251 | src = &src[64..]; 252 | } 253 | y = y.wrapping_add(K0.wrapping_mul(rotate_least(w.0, 37)).wrapping_add(z)); 254 | x = x.wrapping_add(K0.wrapping_mul(rotate_least(v.0.wrapping_add(z), 49))); 255 | // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s. 256 | while src.len() > 64 { 257 | y = K0 258 | .wrapping_mul(rotate_least(y.wrapping_sub(x), 42)) 259 | .wrapping_add(v.1); 260 | w.0 = w.0.wrapping_add(fetch64(&src[src.len() - 16..])); 261 | x = K0.wrapping_mul(rotate_least(x, 49)).wrapping_add(w.0); 262 | w.0 = w.0.wrapping_add(v.0); 263 | v = weak_hash_len32with_seeds(&src[src.len() - 32..], v.0, v.1); 264 | src = &src[0..src.len() - 32]; 265 | } 266 | // At this point our 48 bytes of state should contain more than 267 | // enough information for a strong 128-bit hash. We use two 268 | // different 48-byte-to-8-byte hashes to get a 16-byte final result. 269 | x = hash_len16(x, v.0); 270 | y = hash_len16(y, w.0); 271 | 272 | Pair( 273 | hash_len16(x.wrapping_add(v.1), w.1).wrapping_add(y), 274 | hash_len16(x.wrapping_add(w.1), y.wrapping_add(v.1)), 275 | ) 276 | } 277 | 278 | #[inline] 279 | pub fn city_hash_128(src: &[u8]) -> Pair { 280 | if src.len() >= 144 { 281 | cityhash128withseed(&src[16..], Pair(fetch64(&src[..]) ^ K3, fetch64(&src[8..]))) 282 | } else if src.len() >= 16 { 283 | citymurmur(&src[16..], Pair(fetch64(&src[..]) ^ K3, fetch64(&src[8..]))) 284 | } else if src.len() >= 8 { 285 | citymurmur( 286 | &[], 287 | Pair( 288 | fetch64(&src[..]) ^ (K0.wrapping_mul(src.len() as u64)), 289 | fetch64(&src[src.len() - 8..]) ^ K1, 290 | ), 291 | ) 292 | } else { 293 | citymurmur(src, Pair(K0, K1)) 294 | } 295 | } 296 | 297 | #[cfg(test)] 298 | mod tests { 299 | use super::*; 300 | use clickhouse_driver_cth::{_CityHash128, _CityMurmur, c_char, Hash128}; 301 | use std::vec::Vec; 302 | 303 | fn city_hash_ref(source: &[u8]) -> Pair { 304 | let h = unsafe { _CityHash128(source.as_ptr() as *const c_char, source.len()) }; 305 | Pair(h.0, h.1) 306 | } 307 | 308 | fn city_murmur_ref(source: &[u8], seed: Pair) -> Pair { 309 | let h = unsafe { 310 | _CityMurmur( 311 | source.as_ptr() as *const c_char, 312 | source.len(), 313 | Hash128(seed.0, seed.1), 314 | ) 315 | }; 316 | Pair(h.0, h.1) 317 | } 318 | 319 | // fn hash_len_0to16_ref(source: &[u8]) -> u64 { 320 | // unsafe { _HashLen0to16(source.as_ptr() as *const c_char, source.len()) } 321 | // } 322 | 323 | // #[test] 324 | // fn test_hash_len_0to16() { 325 | // let src = b""; 326 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 327 | // 328 | // let src = b"4444"; 329 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 330 | // 331 | // let src = b"999999999"; 332 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 333 | // 334 | // let src = b"1234567890123456"; 335 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 336 | // 337 | // let src = b"12"; 338 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 339 | // 340 | // let src = b"abcdef"; 341 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 342 | // 343 | // let src = b"0123456789abcdef"; 344 | // assert_eq!(hash_len_0to16_ref(src), hash_len0to16(src)); 345 | // } 346 | 347 | #[test] 348 | fn test_cityhash_unaligned_load() { 349 | let src = [1u8, 0, 0, 1, 0, 0, 0, 0xFF, 0x1F, 0x1F, 0, 0]; 350 | assert_eq!(fetch64(&src[..]), 0xFF00000001000001_u64); 351 | } 352 | 353 | // #[test] 354 | // fn test_cityhash_hash16() { 355 | // let a: u64 = unsafe { testHashLen16(0xFF, 0xFF) }; 356 | // let b: u64 = hash_len16(0xFF, 0xFF); 357 | // assert_eq!(a, b); 358 | // 359 | // let a: u64 = unsafe { testHashLen16(0x1F, 0xFF) }; 360 | // let b: u64 = hash_len16(0x1F, 0xFF); 361 | // assert_eq!(a, b); 362 | // 363 | // let a: u64 = unsafe { testHashLen16(0x00, 0x7A) }; 364 | // let b: u64 = hash_len16(0x00, 0x7A); 365 | // assert_eq!(a, b); 366 | // 367 | // let a: u64 = unsafe { testHashLen16(0x00FF12A6, 0x7AF8375E) }; 368 | // let b: u64 = hash_len16(0x00FF12A6, 0x7AF8375E); 369 | // assert_eq!(a, b); 370 | // } 371 | 372 | #[test] 373 | fn test_citymurmur() { 374 | assert_eq!( 375 | city_murmur_ref(b"", Pair(K0, K1)), 376 | citymurmur(b"", Pair(K0, K1)) 377 | ); 378 | assert_eq!( 379 | city_murmur_ref(b"0123456789", Pair(K0, K1)), 380 | citymurmur(b"0123456789", Pair(K0, K1)) 381 | ); 382 | assert_eq!( 383 | city_murmur_ref(b"0123456789abcdef", Pair(K0, K1)), 384 | citymurmur(b"0123456789abcdef", Pair(K0, K1)) 385 | ); 386 | let src = b"0123456789012345678901234567890123456789012345678901234567891234"; 387 | assert_eq!( 388 | city_murmur_ref(src, Pair(K0, K1)), 389 | citymurmur(src, Pair(K0, K1)) 390 | ); 391 | } 392 | 393 | #[test] 394 | fn test_hash_128() { 395 | const MAX_SIZE: u32 = 1024 * 10; 396 | const ITER_COUNT: u8 = 5; 397 | use rand::Rng; 398 | for s in 8..MAX_SIZE { 399 | let mut b: Vec = Vec::with_capacity(s as usize); 400 | unsafe { 401 | b.set_len(s as usize); 402 | } 403 | for _ in 0..ITER_COUNT { 404 | rand::thread_rng().fill(&mut b[..]); 405 | assert_eq!(city_hash_ref(b.as_ref()), city_hash_128(b.as_ref())); 406 | } 407 | } 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /cityhash/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /cityhash/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-driver-cth" 3 | description = "CityHash binding for ClickHouse Asynchronous Driver." 4 | version = "0.1.0" 5 | authors = ["dmitry dulesov "] 6 | edition = "2018" 7 | build = "build.rs" 8 | license = "MIT" 9 | repository = "https://github.com/ddulesov/clickhouse_driver" 10 | 11 | [dependencies] 12 | libc = "0.2" 13 | 14 | [build-dependencies] 15 | cc = "1.0.54" 16 | 17 | [features] 18 | default = [] 19 | int128 = [] 20 | -------------------------------------------------------------------------------- /cityhash/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cc; 2 | 3 | fn main() { 4 | let mut compiler = cc::Build::new(); 5 | compiler.file("src/cc/city.cc").cpp(true).opt_level(3); 6 | compiler.compile("libcityhash"); 7 | } 8 | -------------------------------------------------------------------------------- /cityhash/src/cc/BUCK: -------------------------------------------------------------------------------- 1 | cxx_library( 2 | name = 'cityhash', 3 | header_namespace = 'cityhash', 4 | exported_headers = subdir_glob([ 5 | ('', '*.h'), 6 | ]), 7 | srcs = glob([ 8 | '*.cc', 9 | ]), 10 | visibility = [ 11 | '//...', 12 | ], 13 | ) 14 | -------------------------------------------------------------------------------- /cityhash/src/cc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ADD_LIBRARY (cityhash-lib STATIC 2 | city.cc 3 | ) 4 | 5 | set_property(TARGET cityhash-lib PROPERTY POSITION_INDEPENDENT_CODE ON) 6 | -------------------------------------------------------------------------------- /cityhash/src/cc/COPYING: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /cityhash/src/cc/city.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file provides a few functions for hashing strings. On x86-64 24 | // hardware in 2011, CityHash64() is faster than other high-quality 25 | // hash functions, such as Murmur. This is largely due to higher 26 | // instruction-level parallelism. CityHash64() and CityHash128() also perform 27 | // well on hash-quality tests. 28 | // 29 | // CityHash128() is optimized for relatively long strings and returns 30 | // a 128-bit hash. For strings more than about 2000 bytes it can be 31 | // faster than CityHash64(). 32 | // 33 | // Functions in the CityHash family are not suitable for cryptography. 34 | // 35 | // WARNING: This code has not been tested on big-endian platforms! 36 | // It is known to work well on little-endian platforms that have a small penalty 37 | // for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. 38 | // 39 | // By the way, for some hash functions, given strings a and b, the hash 40 | // of a+b is easily derived from the hashes of a and b. This property 41 | // doesn't hold for any hash functions in this file. 42 | 43 | #ifndef CITY_HASH_H_ 44 | #define CITY_HASH_H_ 45 | 46 | #include // for size_t. 47 | #include 48 | #include 49 | 50 | typedef uint8_t uint8; 51 | typedef uint32_t uint32; 52 | typedef uint64_t uint64; 53 | typedef std::pair uint128; 54 | 55 | 56 | inline uint64 Uint128Low64(const uint128& x) { return x.first; } 57 | inline uint64 Uint128High64(const uint128& x) { return x.second; } 58 | 59 | // Hash function for a byte array. 60 | // static uint64 CityHash64(const char *buf, size_t len); 61 | 62 | // Hash function for a byte array. For convenience, a 64-bit seed is also 63 | // hashed into the result. 64 | // static uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed); 65 | 66 | // Hash function for a byte array. For convenience, two seeds are also 67 | // hashed into the result. 68 | // static uint64 CityHash64WithSeeds(const char *buf, size_t len, uint64 seed0, uint64 seed1); 69 | 70 | // Hash function for a byte array. 71 | // static uint128 CityHash128(const char *s, size_t len); 72 | 73 | // Hash function for a byte array. For convenience, a 128-bit seed is also 74 | // hashed into the result. 75 | static uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed); 76 | 77 | // Hash 128 input bits down to 64 bits of output. 78 | // This is intended to be a reasonably good hash function. 79 | static inline uint64 Hash128to64(const uint128& x) { 80 | // Murmur-inspired hashing. 81 | const uint64 kMul = 0x9ddfea08eb382d69ULL; 82 | uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul; 83 | a ^= (a >> 47); 84 | uint64 b = (Uint128High64(x) ^ a) * kMul; 85 | b ^= (b >> 47); 86 | b *= kMul; 87 | return b; 88 | } 89 | 90 | 91 | #endif // CITY_HASH_H_ 92 | -------------------------------------------------------------------------------- /cityhash/src/cc/citycrc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include "city.h" 32 | 33 | 34 | // Hash function for a byte array. 35 | static uint128 CityHashCrc128(const char *s, size_t len); 36 | 37 | // Hash function for a byte array. For convenience, a 128-bit seed is also 38 | // hashed into the result. 39 | static uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 40 | 41 | // Hash function for a byte array. Sets result[0] ... result[3]. 42 | static void CityHashCrc256(const char *s, size_t len, uint64 *result); 43 | 44 | #endif // CITY_HASH_CRC_H_ 45 | -------------------------------------------------------------------------------- /cityhash/src/cc/config.h: -------------------------------------------------------------------------------- 1 | /* config.h. Generated from config.h.in by configure. */ 2 | /* config.h.in. Generated from configure.ac by autoheader. */ 3 | 4 | /* Define if building universal (internal helper macro) */ 5 | /* #undef AC_APPLE_UNIVERSAL_BUILD */ 6 | 7 | /* Define to 1 if the compiler supports __builtin_expect. */ 8 | #if _MSC_VER 9 | # define HAVE_BUILTIN_EXPECT 0 10 | #else 11 | # define HAVE_BUILTIN_EXPECT 1 12 | #endif 13 | 14 | /* Define to 1 if you have the header file. */ 15 | #define HAVE_DLFCN_H 1 16 | 17 | /* Define to 1 if you have the header file. */ 18 | #define HAVE_INTTYPES_H 1 19 | 20 | /* Define to 1 if you have the header file. */ 21 | #define HAVE_MEMORY_H 1 22 | 23 | /* Define to 1 if you have the header file. */ 24 | #define HAVE_STDINT_H 1 25 | 26 | /* Define to 1 if you have the header file. */ 27 | #define HAVE_STDLIB_H 1 28 | 29 | /* Define to 1 if you have the header file. */ 30 | #define HAVE_STRINGS_H 1 31 | 32 | /* Define to 1 if you have the header file. */ 33 | #define HAVE_STRING_H 1 34 | 35 | /* Define to 1 if you have the header file. */ 36 | #define HAVE_SYS_STAT_H 1 37 | 38 | /* Define to 1 if you have the header file. */ 39 | #define HAVE_SYS_TYPES_H 1 40 | 41 | /* Define to 1 if you have the header file. */ 42 | #define HAVE_UNISTD_H 1 43 | 44 | /* Define to the sub-directory in which libtool stores uninstalled libraries. 45 | */ 46 | #define LT_OBJDIR ".libs/" 47 | 48 | /* Define to the address where bug reports for this package should be sent. */ 49 | #define PACKAGE_BUGREPORT "cityhash-discuss@googlegroups.com" 50 | 51 | /* Define to the full name of this package. */ 52 | #define PACKAGE_NAME "CityHash" 53 | 54 | /* Define to the full name and version of this package. */ 55 | #define PACKAGE_STRING "CityHash 1.0.2" 56 | 57 | /* Define to the one symbol short name of this package. */ 58 | #define PACKAGE_TARNAME "cityhash" 59 | 60 | /* Define to the home page for this package. */ 61 | #define PACKAGE_URL "" 62 | 63 | /* Define to the version of this package. */ 64 | #define PACKAGE_VERSION "1.0.2" 65 | 66 | /* Define to 1 if you have the ANSI C header files. */ 67 | #define STDC_HEADERS 1 68 | 69 | /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most 70 | significant byte first (like Motorola and SPARC, unlike Intel). */ 71 | #if defined AC_APPLE_UNIVERSAL_BUILD 72 | # if defined __BIG_ENDIAN__ 73 | # define WORDS_BIGENDIAN 1 74 | # endif 75 | #else 76 | # ifndef WORDS_BIGENDIAN 77 | /* # undef WORDS_BIGENDIAN */ 78 | # endif 79 | #endif 80 | 81 | /* Define for Solaris 2.5.1 so the uint32_t typedef from , 82 | , or is not used. If the typedef were allowed, the 83 | #define below would cause a syntax error. */ 84 | /* #undef _UINT32_T */ 85 | 86 | /* Define for Solaris 2.5.1 so the uint64_t typedef from , 87 | , or is not used. If the typedef were allowed, the 88 | #define below would cause a syntax error. */ 89 | /* #undef _UINT64_T */ 90 | 91 | /* Define for Solaris 2.5.1 so the uint8_t typedef from , 92 | , or is not used. If the typedef were allowed, the 93 | #define below would cause a syntax error. */ 94 | /* #undef _UINT8_T */ 95 | 96 | /* Define to `__inline__' or `__inline' if that's what the C compiler 97 | calls it, or to nothing if 'inline' is not supported under any name. */ 98 | #ifndef __cplusplus 99 | /* #undef inline */ 100 | #endif 101 | 102 | /* Define to `unsigned int' if does not define. */ 103 | /* #undef size_t */ 104 | 105 | /* Define to `int' if does not define. */ 106 | /* #undef ssize_t */ 107 | 108 | /* Define to the type of an unsigned integer type of width exactly 32 bits if 109 | such a type exists and the standard includes do not define it. */ 110 | /* #undef uint32_t */ 111 | 112 | /* Define to the type of an unsigned integer type of width exactly 64 bits if 113 | such a type exists and the standard includes do not define it. */ 114 | /* #undef uint64_t */ 115 | 116 | /* Define to the type of an unsigned integer type of width exactly 8 bits if 117 | such a type exists and the standard includes do not define it. */ 118 | /* #undef uint8_t */ 119 | 120 | #ifdef _MSC_VER 121 | #include 122 | typedef SSIZE_T ssize_t; 123 | #else 124 | #include 125 | #endif 126 | -------------------------------------------------------------------------------- /cityhash/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | extern crate libc; 3 | #[cfg(test)] 4 | #[macro_use] 5 | extern crate std; 6 | 7 | use core::mem::size_of; 8 | use core::ops::Deref; 9 | use core::ptr::read_unaligned; 10 | pub use libc::c_char; 11 | 12 | #[repr(C)] 13 | #[derive(Debug, PartialEq)] 14 | pub struct Hash128(pub u64, pub u64); 15 | 16 | impl Hash128 { 17 | #[cfg(int128)] 18 | #[inline(always)] 19 | fn to_u128(self) -> u128 { 20 | (self.0 as u128) << 64 | self.1 as u128 21 | } 22 | } 23 | 24 | impl Deref for Hash128 { 25 | type Target = [u8; 16]; 26 | #[inline] 27 | fn deref(&self) -> &Self::Target { 28 | unsafe { &*(self as *const Hash128 as *const [u8; 16]) } 29 | } 30 | } 31 | 32 | #[allow(clippy::cast_ptr_alignment)] 33 | #[inline] 34 | pub fn fetch64(src: &[u8]) -> u64 { 35 | debug_assert!(src.len() >= size_of::()); 36 | let ptr = src.as_ptr() as *const u64; 37 | unsafe { read_unaligned(ptr) } 38 | } 39 | 40 | #[allow(clippy::cast_ptr_alignment)] 41 | #[inline] 42 | pub fn fetch128(src: &[u8]) -> u128 { 43 | debug_assert!(src.len() >= size_of::()); 44 | let ptr = src.as_ptr() as *const u128; 45 | unsafe { read_unaligned(ptr) } 46 | } 47 | 48 | #[cfg(not(int128))] 49 | impl PartialEq<&[u8]> for Hash128 { 50 | fn eq(&self, other: &&[u8]) -> bool { 51 | other.len() == 16 && (self.0 == fetch64(*other)) && (self.1 == fetch64(&other[8..])) 52 | } 53 | } 54 | 55 | #[cfg(int128)] 56 | impl PartialEq<&[u8]> for Pair { 57 | fn eq(&self, other: &&[u8]) -> bool { 58 | (other.len() == 16) && (self.to_u128() == fetch128(other)) 59 | } 60 | } 61 | 62 | extern "C" { 63 | pub fn _CityHash128(s: *const c_char, len: usize) -> Hash128; 64 | pub fn _CityHash64(s: *const c_char, len: usize) -> u64; 65 | pub fn _CityMurmur(s: *const c_char, len: usize, seed: Hash128) -> Hash128; 66 | } 67 | 68 | #[cfg(test)] 69 | extern "C" { 70 | // pub fn _HashLen16(u: u64, v: u64) -> u64; 71 | // pub fn _Fetch64(s: *const c_char) -> u64; 72 | // pub fn _HashLen0to16(s: *const c_char, len: usize) -> u64; 73 | } 74 | 75 | pub fn city_hash_128(source: &[u8]) -> Hash128 { 76 | unsafe { _CityHash128(source.as_ptr() as *const c_char, source.len()) } 77 | } 78 | 79 | pub fn city_hash_64(source: &[u8]) -> u64 { 80 | unsafe { _CityHash64(source.as_ptr() as *const c_char, source.len()) } 81 | } 82 | 83 | #[cfg(test)] 84 | mod test { 85 | use crate::*; 86 | 87 | #[test] 88 | fn test_city_hash_128() { 89 | assert_eq!( 90 | city_hash_128(b"abc"), 91 | [ 92 | 0xfe, 0x48, 0x77, 0x57, 0x95, 0xf1, 0x0f, 0x90, 0x7e, 0x0d, 0xb2, 0x55, 0x63, 0x17, 93 | 0xa9, 0x13 94 | ] 95 | .as_ref() 96 | ); 97 | assert_ne!( 98 | city_hash_128(b"abc"), 99 | [ 100 | 0x00, 0x48, 0x77, 0x57, 0x95, 0xf1, 0x0f, 0x90, 0x7e, 0x0d, 0xb2, 0x55, 0x63, 0x17, 101 | 0xa9, 0x13 102 | ] 103 | .as_ref() 104 | ); 105 | assert_eq!( 106 | city_hash_128(b"01234567890abc"), 107 | [ 108 | 0x36, 0x20, 0xe9, 0x1b, 0x54, 0x23, 0x04, 0xbe, 0x2d, 0xc7, 0x32, 0x8d, 0x93, 0xd2, 109 | 0x3b, 0x89 110 | ] 111 | .as_ref() 112 | ); 113 | assert_eq!( 114 | city_hash_128(b"01234567890123456789012345678901234567890123456789012345678901234"), 115 | [ 116 | 0x24, 0xd7, 0xd5, 0xdc, 0x8e, 0xb6, 0x85, 0xb2, 0xb1, 0xd9, 0x78, 0x15, 0xa2, 0x2a, 117 | 0xb0, 0x3d 118 | ] 119 | .as_ref() 120 | ); 121 | assert_eq!( 122 | city_hash_128(b""), 123 | [ 124 | 0x2b, 0x9a, 0xc0, 0x64, 0xfc, 0x9d, 0xf0, 0x3d, 0x29, 0x1e, 0xe5, 0x92, 0xc3, 0x40, 125 | 0xb5, 0x3c 126 | ] 127 | .as_ref() 128 | ); 129 | 130 | assert_ne!( 131 | city_hash_128(b"abc"), 132 | [ 133 | 0xfe, 0x48, 0x77, 0x57, 0x95, 0xf1, 0x0f, 0x90, 0x7e, 0x0d, 0xb2, 0x55, 0x63, 0x17, 134 | 0xa9, 0x11 135 | ] 136 | .as_ref() 137 | ); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /clickhouse-driver/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /clickhouse-driver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-driver" 3 | description = "Asynchronous tokio-based Yandex ClickHouse driver." 4 | version = "0.1.0-alpha.3" 5 | license = "MIT" 6 | authors = ["dmitry dulesov "] 7 | homepage = "https://github.com/ddulesov/clickhouse_driver" 8 | edition = "2018" 9 | keywords = ["tokio", "database", "clickhouse"] 10 | categories = ["database"] 11 | repository = "https://github.com/ddulesov/clickhouse_driver" 12 | 13 | [dependencies] 14 | clickhouse-driver-cth = { version="0.1.0", path="../cityhash", optional=true} 15 | clickhouse-driver-cthrs = { version="0.1.1", path="../cityhash-rs", optional=true} 16 | clickhouse-driver-lz4 = { version="0.1.0", path="../lz4a" } 17 | 18 | thiserror = { version="1.0" } 19 | log = { version="0.4.8" } 20 | url = { version="^2" } 21 | tokio = { version = "0.2", features = ["rt-core", "sync", "tcp", "time", "dns", "stream", "test-util", "io-util","macros"] } 22 | tokio-native-tls = { version = "0.1.0", optional= true } 23 | pin-project-lite = { version="^0.1" } 24 | futures = { version="0.3" } 25 | hostname = { version="^0.3" } 26 | chrono = { version="^0.4" } 27 | chrono-tz = { version="^0.5" } 28 | crossbeam = { version="^0.7" } 29 | slab = { version="0.4.2" } 30 | parking_lot = {version="^0.11"} 31 | uuid = { version="^0.8", features=["v4"] } 32 | lazy_static = { version="^1.4" } 33 | rand= { version="0.7" } 34 | byteorder={ version="^1.3"} 35 | bytes={ version="^0.5" } 36 | 37 | [features] 38 | default = ["cityhash_rs"] 39 | tls = ["tokio-native-tls"] 40 | cityhash_rs = ["clickhouse-driver-cthrs"] 41 | cityhash = ["clickhouse-driver-cth"] 42 | recycle = [] 43 | int128 = [] 44 | extra = [] 45 | 46 | [[example]] 47 | name = "insert-select" 48 | required-features = ["cityhash_rs"] 49 | 50 | [[example]] 51 | name = "bulk-insert" 52 | required-features = ["cityhash_rs"] 53 | 54 | [[example]] 55 | name = "select" 56 | required-features = ["cityhash_rs"] 57 | 58 | [[example]] 59 | name = "array" 60 | required-features = ["cityhash_rs"] 61 | 62 | [[test]] 63 | name = "pool" 64 | 65 | [[test]] 66 | name = "query" 67 | 68 | [[test]] 69 | name = "insert" 70 | -------------------------------------------------------------------------------- /clickhouse-driver/examples/array.rs: -------------------------------------------------------------------------------- 1 | extern crate chrono; 2 | extern crate clickhouse_driver; 3 | extern crate tokio; 4 | extern crate uuid; 5 | 6 | use clickhouse_driver::prelude::*; 7 | use std::net::Ipv4Addr; 8 | use std::{env, io, time}; 9 | 10 | macro_rules! get { 11 | ($row: ident, $i: expr, $err: ident) => { 12 | $row.value($i)?.ok_or_else($err)?; 13 | }; 14 | ($row: ident, $i: expr, $err: ident, opt) => { 15 | $row.value($i)?; 16 | }; 17 | } 18 | // CREATE table mainx( 19 | // i64 UInt64, 20 | // lcs LowCardinality(String), 21 | // aip4 Array(IPv4) 22 | // ) ENGINE=Memory 23 | #[derive(Debug)] 24 | struct ClickhouseRow { 25 | id: i64, 26 | name: Option, 27 | ip: Vec, 28 | } 29 | 30 | impl Deserialize for ClickhouseRow { 31 | fn deserialize(row: Row) -> errors::Result { 32 | let err = || errors::ConversionError::UnsupportedConversion; 33 | 34 | let id: i64 = get!(row, 0, err); 35 | let name: Option = get!(row, 1, err, opt).map(|s: &str| s.to_owned()); 36 | let ip: Vec = get!(row, 2, err); 37 | 38 | Ok(ClickhouseRow { id, name, ip }) 39 | } 40 | } 41 | 42 | #[tokio::main] 43 | async fn main() -> Result<(), io::Error> { 44 | let database_url = 45 | env::var("DATABASE_URL").unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 46 | 47 | let pool = Pool::create(database_url.as_str())?; 48 | { 49 | let mut start = time::Instant::now(); 50 | let mut conn = pool.connection().await?; 51 | eprintln!( 52 | "connection establish in {} msec", 53 | start.elapsed().as_millis() 54 | ); 55 | start = time::Instant::now(); 56 | 57 | let mut result = conn.query("SELECT i64, lcs, aip4 FROM mainx").await?; 58 | 59 | let mut c = 0; 60 | while let Some(block) = result.next().await? { 61 | for item in block.iter::() { 62 | println!("{:?}", item); 63 | c += 1; 64 | } 65 | } 66 | eprintln!("fetch {} rows in {} msec", c, start.elapsed().as_millis()); 67 | } 68 | 69 | Ok(()) 70 | } 71 | -------------------------------------------------------------------------------- /clickhouse-driver/examples/bulk-insert.rs: -------------------------------------------------------------------------------- 1 | extern crate chrono; 2 | extern crate clickhouse_driver; 3 | extern crate tokio; 4 | 5 | use clickhouse_driver::prelude::*; 6 | use std::{env, io, time}; 7 | 8 | static NAMES: [&str; 5] = ["one", "two", "three", "four", "five"]; 9 | /// Block size 10 | const BSIZE: u64 = 10000; 11 | /// The number of blocks 12 | const CIRCLE: u64 = 1000; 13 | 14 | fn next_block(i: u64) -> Block<'static> { 15 | let now = chrono::offset::Utc::now(); 16 | 17 | let dt: Vec<_> = (0..BSIZE) 18 | .map(|idx| now + chrono::Duration::seconds(idx as i64)) 19 | .collect(); 20 | let name: Vec<_> = (0..BSIZE) 21 | .map(|idx| NAMES[idx as usize % NAMES.len()]) 22 | .collect(); 23 | 24 | Block::new("perf_rust2") 25 | .add("id", vec![i as u32; BSIZE as usize]) 26 | .add("name", name) 27 | .add("dt", dt) 28 | } 29 | 30 | #[tokio::main] 31 | async fn main() -> Result<(), io::Error> { 32 | let ddl = r" 33 | CREATE TABLE IF NOT EXISTS perf_rust2 ( 34 | id UInt32, 35 | name String, 36 | dt DateTime 37 | ) Engine=MergeTree PARTITION BY name ORDER BY dt"; 38 | 39 | let database_url = 40 | env::var("DATABASE_URL").unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 41 | 42 | let pool = Pool::create(database_url.as_str())?; 43 | 44 | let mut conn = pool.connection().await?; 45 | 46 | conn.execute("DROP TABLE IF EXISTS perf_rust2").await?; 47 | conn.execute(ddl).await?; 48 | 49 | let start = time::Instant::now(); 50 | let block = next_block(0); 51 | let mut insert = conn.insert(&block).await?; 52 | 53 | for i in 1u64..CIRCLE { 54 | let block = next_block(i); 55 | insert.next(&block).await?; 56 | } 57 | // Commit at the end 58 | insert.commit().await?; 59 | // Stop inserting pipeline before next query be called 60 | // Here it's useless 61 | // drop(insert); 62 | 63 | eprintln!("elapsed {} msec", start.elapsed().as_millis()); 64 | eprintln!("{} rows have been inserted in \n'{}'", BSIZE * CIRCLE, ddl); 65 | 66 | Ok(()) 67 | } 68 | -------------------------------------------------------------------------------- /clickhouse-driver/examples/insert-select.rs: -------------------------------------------------------------------------------- 1 | extern crate chrono; 2 | extern crate clickhouse_driver; 3 | extern crate tokio; 4 | extern crate uuid; 5 | 6 | use std::net::Ipv4Addr; 7 | use std::{env, io, time}; 8 | 9 | use uuid::Uuid; 10 | 11 | use clickhouse_driver::prelude::types::Decimal32; 12 | use clickhouse_driver::prelude::*; 13 | 14 | type ServerDate = chrono::DateTime; 15 | 16 | #[derive(Debug)] 17 | struct Blob { 18 | id: u64, 19 | url: String, 20 | date: ServerDate, 21 | client: Uuid, 22 | ip: Ipv4Addr, 23 | value: Decimal32, 24 | } 25 | 26 | impl Deserialize for Blob { 27 | fn deserialize(row: Row) -> errors::Result { 28 | let err = || errors::ConversionError::UnsupportedConversion; 29 | 30 | let id: u64 = row.value(0)?.ok_or_else(err)?; 31 | let url: &str = row.value(1)?.ok_or_else(err)?; 32 | let date: ServerDate = row.value(2)?.ok_or_else(err)?; 33 | let client: Uuid = row.value(3)?.ok_or_else(err)?; 34 | let ip = row.value(4)?.ok_or_else(err)?; 35 | let value: Decimal32 = row.value(5)?.ok_or_else(err)?; 36 | 37 | Ok(Blob { 38 | id, 39 | date, 40 | client, 41 | value, 42 | url: url.to_string(), 43 | ip, 44 | }) 45 | } 46 | } 47 | const C: u64 = 10000; 48 | 49 | #[tokio::main] 50 | async fn main() -> Result<(), io::Error> { 51 | let ddl = " 52 | CREATE TABLE IF NOT EXISTS blob ( 53 | id UInt64, 54 | url String, 55 | date DateTime, 56 | client UUID, 57 | ip IPv4, 58 | value Decimal32(2) 59 | ) ENGINE=MergeTree PARTITION BY id ORDER BY date"; 60 | 61 | let uuid = Uuid::new_v4(); 62 | let ip: Ipv4Addr = "127.0.0.1".parse().unwrap(); 63 | let value = Decimal32::from(4000_i32, 2); 64 | let now = chrono::offset::Utc::now(); 65 | //let today = chrono::offset::Utc::today(); 66 | 67 | let id = vec![0u64, 159, 146, 150]; 68 | let url = vec![ 69 | "https://www.rust-lang.org/", 70 | "https://tokio.rs/", 71 | "https://github.com/ddulesov/", 72 | "https://internals.rust-lang.org/", 73 | ]; 74 | let date = vec![now; 4]; 75 | let client = vec![uuid; 4]; 76 | let ip = vec![ip; 4]; 77 | let value = vec![value; 4]; 78 | 79 | let block = { 80 | Block::new("blob") 81 | .add("id", id.clone()) 82 | .add("url", url.clone()) 83 | .add("date", date.clone()) 84 | .add("client", client.clone()) 85 | .add("ip", ip.clone()) 86 | .add("value", value.clone()) 87 | }; 88 | 89 | let database_url = 90 | env::var("DATABASE_URL").unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 91 | 92 | let pool = Pool::create(database_url.as_str())?; 93 | { 94 | let mut start = time::Instant::now(); 95 | let mut conn = pool.connection().await?; 96 | eprintln!("connection establish {} msec", start.elapsed().as_millis()); 97 | start = time::Instant::now(); 98 | conn.execute("DROP TABLE IF EXISTS blob").await?; 99 | conn.execute(ddl).await?; 100 | eprintln!("drop and create table {} msec", start.elapsed().as_millis()); 101 | start = time::Instant::now(); 102 | let mut insert = conn.insert(&block).await?; 103 | eprintln!("first block insert {} msec", start.elapsed().as_millis()); 104 | eprintln!("INSERT..."); 105 | start = time::Instant::now(); 106 | for _ in 1u64..C { 107 | // we can use the same block repeatedly 108 | // let block = { 109 | // Block::new("") 110 | // .add("id", id.clone()) 111 | // .add("url", url.clone()) 112 | // .add("date", date.clone()) 113 | // .add("client", client.clone()) 114 | // .add("ip", ip.clone()) 115 | // .add("value", value.clone()) 116 | // }; 117 | insert.next(&block).await?; 118 | } 119 | 120 | insert.commit().await?; 121 | eprintln!( 122 | "{} block insert {} msec", 123 | C - 1, 124 | start.elapsed().as_millis() 125 | ); 126 | // Stop inserting pipeline before next query be called 127 | drop(insert); 128 | 129 | eprintln!("SELECT..."); 130 | start = time::Instant::now(); 131 | let mut result = conn 132 | .query("SELECT id, url, date, client, ip, value FROM blob LIMIT 30000") 133 | .await?; 134 | 135 | while let Some(block) = result.next().await? { 136 | eprintln!("fetch block {} msec", start.elapsed().as_millis()); 137 | for (i, row) in block.iter::().enumerate() { 138 | if i % 1000 == 0 { 139 | println!("{:5} {:?}", i, row); 140 | } 141 | } 142 | start = time::Instant::now(); 143 | } 144 | } 145 | 146 | Ok(()) 147 | } 148 | -------------------------------------------------------------------------------- /clickhouse-driver/examples/select.rs: -------------------------------------------------------------------------------- 1 | extern crate chrono; 2 | extern crate clickhouse_driver; 3 | extern crate tokio; 4 | extern crate uuid; 5 | 6 | use clickhouse_driver::prelude::*; 7 | use std::{env, io, time}; 8 | 9 | type ServerDate = chrono::DateTime; 10 | 11 | #[derive(Debug)] 12 | struct Perf { 13 | id: u32, 14 | name: String, 15 | date: ServerDate, 16 | } 17 | 18 | macro_rules! get { 19 | ($row: ident, $i: expr, $err: ident) => { 20 | $row.value($i)?.ok_or_else($err)?; 21 | }; 22 | } 23 | 24 | impl Deserialize for Perf { 25 | fn deserialize(row: Row) -> errors::Result { 26 | let err = || errors::ConversionError::UnsupportedConversion; 27 | 28 | let id: u32 = get!(row, 0, err); 29 | let name: &str = get!(row, 1, err); 30 | let date: ServerDate = get!(row, 2, err); 31 | 32 | Ok(Perf { 33 | id, 34 | name: name.to_string(), 35 | date, 36 | }) 37 | } 38 | } 39 | 40 | #[tokio::main] 41 | async fn main() -> Result<(), io::Error> { 42 | let database_url = 43 | env::var("DATABASE_URL").unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 44 | 45 | let pool = Pool::create(database_url.as_str())?; 46 | { 47 | let mut start = time::Instant::now(); 48 | let mut conn = pool.connection().await?; 49 | eprintln!( 50 | "connection establish in {} msec", 51 | start.elapsed().as_millis() 52 | ); 53 | start = time::Instant::now(); 54 | 55 | let mut result = conn.query("SELECT id, name, dt FROM perf").await?; 56 | 57 | let mut sum: u64 = 0; 58 | while let Some(block) = result.next().await? { 59 | for item in block.iter::() { 60 | sum += item.id as u64; 61 | } 62 | } 63 | eprintln!("fetch blocks in {} msec", start.elapsed().as_millis()); 64 | eprintln!("{}", sum); 65 | } 66 | 67 | Ok(()) 68 | } 69 | -------------------------------------------------------------------------------- /clickhouse-driver/src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::str::Utf8Error; 2 | use std::{borrow::Cow, io, result}; 3 | 4 | use thiserror::Error; 5 | use url::ParseError; 6 | 7 | use crate::types::SqlType; 8 | 9 | #[derive(Debug, Error, Clone, Eq, PartialEq)] 10 | #[error("{} {} {}", name, code, message)] 11 | pub struct Exception { 12 | pub name: String, 13 | pub code: u32, 14 | pub message: String, 15 | #[cfg(test)] 16 | pub(crate) trace: String, 17 | } 18 | 19 | /// This type represents Clickhouse server error. 20 | #[derive(Debug, Error, Clone, Eq, PartialEq)] 21 | #[error("Server exception {}", _0[0])] 22 | pub struct ServerError(pub Box<[Exception]>); 23 | 24 | /// This type enumerates configuration URL errors. 25 | #[derive(Debug, Error, Clone, Eq, PartialEq)] 26 | pub enum UrlError { 27 | #[error("Connection URL parameter `{}' requires feature `{}'", param, feature)] 28 | FeatureRequired { feature: String, param: String }, 29 | 30 | #[error("Invalid or incomplete connection URL")] 31 | Invalid, 32 | 33 | #[error("Invalid value `{}' for connection URL parameter `{}'.", value, param)] 34 | InvalidParamValue { param: String, value: String }, 35 | 36 | #[error("Invalid pool constraints: pool_min ({}) > pool_max ({}).", min, max)] 37 | InvalidPoolConstraints { min: u16, max: u16 }, 38 | 39 | #[error("URL parse error: {}", _0)] 40 | Parse(#[source] ParseError), 41 | 42 | #[error("Unknown connection URL parameter `{}'.", param)] 43 | UnknownParameter { param: String }, 44 | 45 | #[error("Unsupported connection URL scheme `{}'.", scheme)] 46 | UnsupportedScheme { scheme: String }, 47 | } 48 | 49 | /// This type enumerates clickhouse driver own errors. 50 | #[derive(Debug, Error, Clone, PartialEq)] 51 | pub enum DriverError { 52 | #[error("Connection to the server is closed.")] 53 | ConnectionClosed, 54 | 55 | #[error("Connection timeout.")] 56 | ConnectionTimeout, 57 | 58 | #[error("Packet {} out of order.", _0)] 59 | PacketOutOfOrder(u64), 60 | 61 | #[error("Pool was disconnected.")] 62 | PoolDisconnected, 63 | 64 | #[error("Unexpected packet.")] 65 | UnexpectedPacket { code: u64, payload: Vec }, 66 | 67 | #[error("Unsupported data type.")] 68 | UnsupportedType(SqlType), 69 | 70 | #[error("Malformed packet data.")] 71 | BrokenData, 72 | 73 | #[error("Packet too large.")] 74 | PacketTooLarge, 75 | 76 | #[error("String too long.")] 77 | StringTooLong, 78 | 79 | #[error("Bad compressed packet header.")] 80 | BadCompressedPacketHeader, 81 | 82 | #[error("Bad packet hash.")] 83 | BadHash, 84 | 85 | #[error("Block row count {} exceed the limit.", _0)] 86 | RowCountTooMany(u64), 87 | 88 | #[error("Other operation in progress.")] 89 | OperationInProgress, 90 | 91 | #[error("Index out of range.")] 92 | IndexOutOfRange, 93 | 94 | #[error("Integrity error")] 95 | IntegrityError, 96 | } 97 | 98 | /// This type enumerates clickhouse data conversion errors. 99 | #[derive(Debug, Error, Clone, PartialEq)] 100 | pub enum ConversionError { 101 | #[error("Conversion unsupported")] 102 | UnsupportedConversion, 103 | #[error("fixed string length doesn't match column type. FixedString({})", _0)] 104 | FixedStringLengthNotMatch(u32), 105 | #[error("string value '{:?}' doesn't match enum value", _0)] 106 | EnumMismatch(Vec), 107 | #[error("Unknown column type '{}'", _0)] 108 | UnknownColumnType(String), 109 | #[error("Incorrect utf8 byte sequence")] 110 | Utf8, 111 | } 112 | /// Consolidation of errors 113 | #[derive(Debug, Error)] 114 | pub enum Error { 115 | #[error("Driver error: `{}'", _0)] 116 | Driver(#[source] Box), 117 | 118 | #[error("io error: `{}`", _0)] 119 | Io(#[source] io::Error), 120 | 121 | #[error("Server error: `{}'", _0)] 122 | Server(#[source] ServerError), 123 | 124 | #[error("URL error: `{}'", _0)] 125 | Url(#[source] UrlError), 126 | 127 | #[error("Deserialize error: `{}`", _0)] 128 | Convertion(ConversionError), 129 | 130 | #[error("Other error: `{}`", _0)] 131 | Other(Cow<'static, str>), 132 | } 133 | 134 | impl Error { 135 | pub fn is_timeout(&self) -> bool { 136 | if let Error::Driver(berr) = self { 137 | **berr == DriverError::ConnectionTimeout 138 | } else { 139 | false 140 | } 141 | } 142 | } 143 | 144 | pub type Result = result::Result; 145 | 146 | impl From for Error { 147 | fn from(err: io::Error) -> Self { 148 | let kind = err.kind(); 149 | if err.get_ref().is_some() { 150 | match err.into_inner().unwrap().downcast::() { 151 | Ok(err) => Error::Driver(err), 152 | _ => Error::Io(io::Error::from(kind)), 153 | } 154 | } else { 155 | Error::Io(err) 156 | } 157 | } 158 | } 159 | 160 | impl From for io::Error { 161 | fn from(err: Error) -> Self { 162 | match err { 163 | Error::Io(error) => error, 164 | e => io::Error::new(io::ErrorKind::Other, e.to_string()), 165 | } 166 | } 167 | } 168 | 169 | impl From for UrlError { 170 | fn from(err: ParseError) -> Self { 171 | UrlError::Parse(err) 172 | } 173 | } 174 | /// NotAnError 175 | impl From for Error { 176 | fn from(_: std::convert::Infallible) -> Self { 177 | Error::Other(Cow::Borrowed("")) 178 | } 179 | } 180 | 181 | impl From for Error { 182 | fn from(err: ParseError) -> Self { 183 | Error::Url(err.into()) 184 | } 185 | } 186 | 187 | impl From for Error { 188 | fn from(_err: Utf8Error) -> Self { 189 | Error::Convertion(ConversionError::Utf8) 190 | } 191 | } 192 | 193 | impl From for Error { 194 | fn from(err: DriverError) -> Self { 195 | Error::Driver(Box::new(err)) 196 | } 197 | } 198 | 199 | impl From for Error { 200 | fn from(err: ServerError) -> Self { 201 | Error::Server(err) 202 | } 203 | } 204 | 205 | impl From for Error { 206 | fn from(err: UrlError) -> Self { 207 | Error::Url(err) 208 | } 209 | } 210 | 211 | impl From for Error { 212 | fn from(err: ConversionError) -> Self { 213 | Error::Convertion(err) 214 | } 215 | } 216 | 217 | impl From<&'static str> for Error { 218 | fn from(err: &'static str) -> Self { 219 | Error::Other(Cow::Borrowed(err)) 220 | } 221 | } 222 | 223 | impl From<()> for Error { 224 | fn from(_err: ()) -> Self { 225 | Error::Other(Cow::Borrowed("unknown error")) 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /clickhouse-driver/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! ## Clickhouse-driver 2 | //! Asynchronous pure rust tokio-based Clickhouse client library 3 | //! 4 | //! ## Quick start 5 | //! add next lines in dependencies section of `Cargo.toml` 6 | //! ```toml 7 | //! clickhouse-driver = { version="0.1.0-alpha.1", path="../path_to_package/clickhouse-driver"} 8 | //! clickhouse-driver-lz4 = { version="0.1.0", path="../path_to_package/lz4a"} 9 | //! clickhouse-driver-cthrs = { version="0.1.0", path="../path_to_package/cityhash-rs"} 10 | //! ``` 11 | //! ## Supported Clickhouse data types 12 | //! * Date | DateTime | DateTime64- read/write 13 | //! * (U)Int(8|16|32|64) - read/write 14 | //! * Float32 | Float64 - read/write 15 | //! * UUID - read/write 16 | //! * String | FixedString- read/write 17 | //! * Ipv4 | Ipv6 - read/write 18 | //! * Nullable(*) - read/write 19 | //! * Decimal - read/write 20 | //! * Enum8, Enum16 - read/write 21 | //! 22 | //! * LowCardinality(String) - read 23 | //! 24 | //! ## Connection url 25 | //! ```url 26 | //! tcp://[username:password@]host.name[:port]/database?paramname=paramvalue&... 27 | //! ``` 28 | //! for example 29 | //! ```url 30 | //! tcp://user:default@localhost/log?ping_timout=200ms&execute_timeout=5s&query_timeout=20s&pool_max=4&compression=lz4 31 | //! ``` 32 | //! - default port: 9000 33 | //! - default username: "default" 34 | //! - default database: "default" 35 | //! 36 | #![recursion_limit = "128"] 37 | #![allow(clippy::unknown_clippy_lints)] 38 | extern crate byteorder; 39 | extern crate chrono; 40 | extern crate chrono_tz; 41 | #[cfg(not(feature = "cityhash_rs"))] 42 | extern crate clickhouse_driver_cth; 43 | #[cfg(feature = "cityhash_rs")] 44 | extern crate clickhouse_driver_cthrs; 45 | 46 | extern crate core; 47 | #[macro_use] 48 | extern crate futures; 49 | extern crate hostname; 50 | #[macro_use] 51 | extern crate lazy_static; 52 | #[cfg(lz4)] 53 | extern crate clickhouse_driver_lz4a; 54 | extern crate log; 55 | extern crate parking_lot; 56 | #[cfg(test)] 57 | extern crate rand; 58 | extern crate thiserror; 59 | extern crate tokio; 60 | extern crate url; 61 | extern crate uuid; 62 | use pool::options::Options; 63 | 64 | #[cfg(not(target_endian = "little"))] 65 | compile_error!("only little-endian platforms supported"); 66 | 67 | mod client; 68 | mod compression; 69 | mod errors; 70 | mod pool; 71 | pub mod prelude; 72 | #[macro_use] 73 | mod protocol; 74 | mod sync; 75 | mod types; 76 | 77 | #[allow(dead_code)] 78 | const MAX_STRING_LEN: usize = 64 * 1024; 79 | /// Max number of rows in server block, 640K is default value 80 | const MAX_BLOCK_SIZE: usize = 640 * 1024; 81 | /// Max size of server block, bytes, 1M is default value 82 | const MAX_BLOCK_SIZE_BYTES: usize = 10 * 1024 * 1024; 83 | 84 | pub static CLIENT_NAME: &str = "Rust Native Driver"; 85 | pub const CLICK_HOUSE_REVISION: u64 = 54405; 86 | pub const CLICK_HOUSE_DBMSVERSION_MAJOR: u64 = 1; 87 | pub const CLICK_HOUSE_DBMSVERSION_MINOR: u64 = 1; 88 | 89 | lazy_static! { 90 | static ref HOSTNAME: String = { 91 | hostname::get().map_or_else( 92 | |_orig| String::new(), 93 | |s| s.into_string().unwrap_or_default(), 94 | ) 95 | }; 96 | static ref DEF_OPTIONS: Options = crate::pool::options::Options::default(); 97 | } 98 | 99 | pub fn description() -> String { 100 | format!( 101 | "{} {}.{}.{}", 102 | CLIENT_NAME, 103 | CLICK_HOUSE_DBMSVERSION_MAJOR, 104 | CLICK_HOUSE_DBMSVERSION_MINOR, 105 | CLICK_HOUSE_REVISION 106 | ) 107 | } 108 | 109 | #[test] 110 | fn test_encoder() { 111 | assert_eq!(description(), "Rust Native Driver 1.1.54405"); 112 | } 113 | -------------------------------------------------------------------------------- /clickhouse-driver/src/pool/builder.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | convert::{TryFrom, TryInto}, 3 | sync::atomic, 4 | sync::Arc, 5 | }; 6 | 7 | use tokio::sync; 8 | 9 | use crate::{ 10 | client::InnerConection, 11 | errors::{Result, UrlError}, 12 | sync::WakerSet, 13 | }; 14 | 15 | use super::{CompressionMethod, Inner, Options, Pool, POOL_STATUS_SERVE}; 16 | use parking_lot::Mutex; 17 | 18 | /// Connection pool builder 19 | /// 20 | /// # Example 21 | /// ` 22 | /// let pool = PoolBuilder::default() 23 | /// .with_database("default") 24 | /// .with_username("default") 25 | /// .with_password("") 26 | /// .add_addr("www.example.com:9000") 27 | /// .build() 28 | /// .unwrap(); 29 | /// ` 30 | pub struct PoolBuilder { 31 | addr: Vec, 32 | username: Option, 33 | password: Option, 34 | database: Option, 35 | pool_min: u16, 36 | pool_max: u16, 37 | compression: CompressionMethod, 38 | ping: bool, 39 | } 40 | 41 | impl TryFrom for Options { 42 | type Error = UrlError; 43 | 44 | fn try_from(mut value: PoolBuilder) -> std::result::Result { 45 | if value.addr.is_empty() { 46 | return Err(UrlError::Invalid); 47 | } 48 | 49 | let mut options = crate::DEF_OPTIONS.clone(); 50 | std::mem::swap(&mut options.addr, &mut value.addr); 51 | options.compression = value.compression; 52 | options.ping_before_query = value.ping; 53 | options.username = value.username.replace(options.username).unwrap_or_default(); 54 | options.password = value.password.replace(options.password).unwrap_or_default(); 55 | options.database = value.database.replace(options.database).unwrap_or_default(); 56 | 57 | options.pool_min = value.pool_min; 58 | options.pool_max = value.pool_max; 59 | Ok(options) 60 | } 61 | } 62 | 63 | impl PoolBuilder { 64 | /// Set compression option 65 | /// This make connection use LZ4 compression for block data transfer 66 | #[inline] 67 | pub fn with_compression(mut self) -> Self { 68 | self.compression = CompressionMethod::LZ4; 69 | self 70 | } 71 | /// If set, this option make connection check server availability after it 72 | /// is received from pool. 73 | #[inline] 74 | pub fn with_ping(mut self) -> Self { 75 | self.ping = true; 76 | self 77 | } 78 | /// Set connection pool boundaries 79 | /// min - set the number of idle connection that the pool can keep up to 80 | /// max - set maximum number of connection that pool can issued 81 | #[inline] 82 | pub fn with_pool(mut self, min: u16, max: u16) -> Self { 83 | self.pool_min = min; 84 | self.pool_max = max; 85 | self 86 | } 87 | 88 | /// Set the username that is used in authentication 89 | #[inline] 90 | pub fn with_username(mut self, value: impl ToString) -> Self { 91 | self.username = Some(value.to_string()); 92 | self 93 | } 94 | /// Set the default database that is used in query processing if 95 | /// the query doesn't explicitly specify another database name 96 | #[inline] 97 | pub fn with_database(mut self, value: impl ToString) -> Self { 98 | self.database = Some(value.to_string()); 99 | self 100 | } 101 | /// Set password that is used in authentication 102 | #[inline] 103 | pub fn with_password(mut self, value: impl ToString) -> Self { 104 | self.password = Some(value.to_string()); 105 | self 106 | } 107 | 108 | /// Set server host address. 109 | /// 110 | /// Address must have domain name and port number 111 | /// # Example 112 | /// ` 113 | /// PoolBuilder::new() 114 | /// .with_addr('example1.com:9000') 115 | /// .with_addr('example2.com:9000'); 116 | /// ` 117 | /// Connection pool can have multiple addresses 118 | /// In this case next connection randomly chooses any 119 | /// available one if it's reachable 120 | #[inline] 121 | pub fn add_addr(mut self, value: impl ToString) -> Self { 122 | self.addr.push(value.to_string()); 123 | self 124 | } 125 | /// Convert the Builder into Pool using specified options. 126 | /// Note! Created Pool does not have connection. 127 | /// First connection will be created by executing pool.connection() 128 | /// 129 | #[inline] 130 | pub fn build(self) -> Result { 131 | let options: Options = self.try_into()?; 132 | PoolBuilder::create(options) 133 | } 134 | /// Construct Pool from Option object 135 | pub(super) fn create(mut options: Options) -> Result { 136 | if options.pool_max < options.pool_min { 137 | return Err(UrlError::InvalidPoolConstraints { 138 | min: options.pool_min, 139 | max: options.pool_max, 140 | } 141 | .into()); 142 | } 143 | 144 | #[allow(unused_variables)] 145 | if cfg!(feature = "recycle") { 146 | let (tx, rx) = sync::mpsc::unbounded_channel::>>(); 147 | } 148 | 149 | let hosts = options.take_addr(); 150 | 151 | let inner = Arc::new(Inner { 152 | new: crossbeam::queue::ArrayQueue::new(options.pool_min as usize), 153 | options, 154 | wakers: WakerSet::new(), 155 | lock: Mutex::new(0), 156 | connections_num: atomic::AtomicUsize::new(0), 157 | //wait: atomic::AtomicUsize::new(0), 158 | #[cfg(feature = "recycle")] 159 | recycler: Some(rx), 160 | hosts, 161 | close: POOL_STATUS_SERVE.into(), 162 | }); 163 | 164 | let mut pool = Pool { 165 | inner, 166 | #[cfg(feature = "recycle")] 167 | drop: tx, 168 | }; 169 | pool.inner.spawn_recycler(); 170 | Ok(pool) 171 | } 172 | } 173 | 174 | impl Default for PoolBuilder { 175 | fn default() -> Self { 176 | PoolBuilder { 177 | addr: Vec::new(), 178 | username: None, 179 | password: None, 180 | database: None, 181 | pool_min: crate::DEF_OPTIONS.pool_min, 182 | pool_max: crate::DEF_OPTIONS.pool_max, 183 | ping: false, 184 | compression: CompressionMethod::None, 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /clickhouse-driver/src/pool/disconnect.rs: -------------------------------------------------------------------------------- 1 | use super::{Inner, Pool, POOL_STATUS_SERVE, POOL_STATUS_STOPPING}; 2 | use crate::errors::{DriverError, Result}; 3 | use std::future::Future; 4 | use std::pin::Pin; 5 | use std::sync::{atomic::Ordering, Arc}; 6 | use std::task::{Context, Poll}; 7 | 8 | /// DisconnectPool future 9 | pub struct DisconnectPool { 10 | pool_inner: Arc, 11 | } 12 | 13 | impl DisconnectPool { 14 | #[inline(always)] 15 | pub(super) fn new(pool: Pool) -> DisconnectPool { 16 | DisconnectPool { 17 | pool_inner: pool.inner, 18 | } 19 | } 20 | } 21 | 22 | impl Future for DisconnectPool { 23 | type Output = Result<()>; 24 | 25 | fn poll(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll { 26 | let this = self.as_mut(); 27 | 28 | let stop = this 29 | .pool_inner 30 | .close 31 | .compare_exchange_weak( 32 | POOL_STATUS_SERVE, 33 | POOL_STATUS_STOPPING, 34 | Ordering::AcqRel, 35 | Ordering::Relaxed, 36 | ) 37 | .is_err(); 38 | 39 | this.pool_inner.wakers.notify_all(); 40 | 41 | // TODO: waiting for connections to close 42 | if stop { 43 | Poll::Ready(Ok(())) 44 | } else { 45 | Poll::Ready(Err(DriverError::PoolDisconnected.into())) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /clickhouse-driver/src/pool/mod.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryInto; 2 | use std::{ 3 | fmt, 4 | sync::atomic::{self, Ordering}, 5 | sync::Arc, 6 | }; 7 | 8 | #[cfg(feature = "recycle")] 9 | use tokio::sync; 10 | 11 | pub use builder::PoolBuilder; 12 | use crossbeam::queue; 13 | pub use options::CompressionMethod; 14 | pub use options::Options; 15 | use parking_lot::Mutex; 16 | use tokio::time::delay_for; 17 | use util::*; 18 | 19 | use crate::{ 20 | client::{disconnect, Connection, InnerConection}, 21 | errors::{Error, Result}, 22 | sync::WakerSet, 23 | }; 24 | 25 | use self::disconnect::DisconnectPool; 26 | use crate::errors::DriverError; 27 | use futures::Future; 28 | use std::pin::Pin; 29 | use std::task::{Context, Poll}; 30 | 31 | pub mod builder; 32 | mod disconnect; 33 | pub mod options; 34 | 35 | #[cfg(feature = "recycle")] 36 | mod recycler; 37 | mod util; 38 | 39 | /// Normal operation status 40 | const POOL_STATUS_SERVE: i8 = 0; 41 | /// Pool in progress of shutting down. 42 | /// New connection request will be rejected 43 | const POOL_STATUS_STOPPING: i8 = 1; 44 | /// Pool is stopped. 45 | /// All connections are freed 46 | #[allow(dead_code)] 47 | const POOL_STATUS_STOPPED: i8 = 2; 48 | 49 | /// Pool implementation (inner sync structure) 50 | pub(crate) struct Inner { 51 | new: queue::ArrayQueue>, 52 | /// The number of issued connections 53 | /// This value is in range of 0 .. options.max_pool 54 | lock: Mutex, 55 | /// Pool options 56 | pub(crate) options: Options, 57 | /// Used for notification of tasks which wait for available connection 58 | wakers: WakerSet, 59 | #[cfg(feature = "recycle")] 60 | recycler: Option>>>, 61 | /// Server host addresses 62 | hosts: Vec, 63 | /// The number of active connections that is taken by tasks 64 | connections_num: atomic::AtomicUsize, 65 | /// Number of tasks in waiting queue 66 | //wait: atomic::AtomicUsize, 67 | /// Pool status flag 68 | close: atomic::AtomicI8, 69 | } 70 | 71 | impl Inner { 72 | #[cfg(not(feature = "recycle"))] 73 | #[inline] 74 | fn spawn_recycler(self: &mut Arc) {} 75 | 76 | #[cfg(feature = "recycle")] 77 | fn spawn_recycler(self: &mut Arc) { 78 | use recycler::Recycler; 79 | 80 | let dropper = if let Some(inner) = Arc::get_mut(self) { 81 | inner.recycler.take() 82 | } else { 83 | None 84 | }; 85 | 86 | //use ttl_check_inerval::TtlCheckInterval; 87 | if let Some(dropper) = dropper { 88 | // Spawn the Recycler. 89 | tokio::spawn(Recycler::new(Arc::clone(self), dropper)); 90 | } 91 | } 92 | 93 | fn closed(&self) -> bool { 94 | self.close.load(atomic::Ordering::Relaxed) > POOL_STATUS_SERVE 95 | } 96 | 97 | /// Take back connection to pool if connection is not deterogated 98 | /// and pool is not full . Recycle in other case. 99 | pub(crate) fn return_connection(&self, conn: Box) { 100 | // NOTE: It's safe to call it out of tokio runtime 101 | let conn = if conn.is_ok() 102 | && conn.info.flag == 0 103 | && !self.closed() 104 | && self.new.len() < self.options.pool_min as usize 105 | { 106 | let lock = self.lock.lock(); 107 | match self.new.push(conn) { 108 | Ok(_) => { 109 | self.wakers.notify_one(); 110 | drop(lock); 111 | return; 112 | } 113 | Err(econn) => econn.0, 114 | } 115 | } else { 116 | conn 117 | }; 118 | { 119 | let mut lock = self.lock.lock(); 120 | *lock -= 1; 121 | //self.issued.fetch_sub(1, Ordering::AcqRel); 122 | disconnect(conn); 123 | // NOTE! Release right before notify 124 | drop(lock); 125 | } 126 | self.wakers.notify_one(); 127 | } 128 | } 129 | 130 | struct AwaitConnection<'a, F> { 131 | key: Option, 132 | wakers: &'a WakerSet, 133 | inner: &'a Inner, 134 | f: F, 135 | } 136 | 137 | impl bool> Future for AwaitConnection<'_, F> { 138 | type Output = bool; 139 | 140 | fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 141 | let me = unsafe { Pin::get_unchecked_mut(self) }; 142 | let lock = me.inner.lock.lock(); 143 | if let Some(key) = me.key.take() { 144 | if me.wakers.remove_if_notified(key, cx) { 145 | me.key = None; 146 | Poll::Ready((me.f)(*lock, me.inner)) 147 | } else { 148 | Poll::Pending 149 | } 150 | } else if (me.f)(*lock, me.inner) { 151 | Poll::Ready(true) 152 | } else { 153 | let key = me.wakers.insert(cx); 154 | me.key = Some(key); 155 | Poll::Pending 156 | } 157 | } 158 | } 159 | 160 | /// Reference to a asynchronous Clickhouse connections pool. 161 | /// It can be cloned and shared between threads. 162 | #[derive(Clone)] 163 | pub struct Pool { 164 | pub(crate) inner: Arc, 165 | #[cfg(feature = "recycle")] 166 | drop: sync::mpsc::UnboundedSender>>, 167 | } 168 | 169 | impl fmt::Debug for Pool { 170 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 171 | let issued = unsafe { *self.inner.lock.data_ptr() }; 172 | f.debug_struct("Pool") 173 | .field("inner.min", &self.inner.options.pool_min) 174 | .field("inner.max", &self.inner.options.pool_max) 175 | .field("queue len", &self.inner.new.len()) 176 | .field("issued", &issued) 177 | .finish() 178 | } 179 | } 180 | 181 | impl Pool { 182 | /// Create pool object from Options object or url string. 183 | /// 184 | /// # Example 185 | /// ``` 186 | /// use clickhouse_driver::prelude::*; 187 | /// let pool = Pool::create("tcp://username:password@localhost/db?compression=lz4"); 188 | /// ``` 189 | #[inline] 190 | pub fn create(options: T) -> Result 191 | where 192 | Error: From, 193 | T: TryInto, 194 | { 195 | let options = options.try_into()?; 196 | PoolBuilder::create(options) 197 | } 198 | 199 | /// Return pool current status. 200 | /// `idle` - number of idle connections in pool 201 | /// `issued` - total number of issued connections 202 | /// `wait` - number of tasks waiting for a connection 203 | #[inline(always)] 204 | pub fn info(&self) -> PoolInfo { 205 | let inner = &self.inner; 206 | 207 | util::PoolInfo { 208 | idle: inner.new.len(), 209 | issued: unsafe { *inner.lock.data_ptr() }, 210 | wait: inner.wakers.len(), 211 | } 212 | } 213 | 214 | /// Return a connection from poll or create new if 215 | /// the poll doesn't have idle connections 216 | pub async fn connection(&self) -> Result { 217 | let inner = &*self.inner; 218 | loop { 219 | let ok = AwaitConnection { 220 | key: None, 221 | wakers: &inner.wakers, 222 | inner: &inner, 223 | f: |issued: usize, inner: &Inner| { 224 | inner.closed() 225 | || !inner.new.is_empty() 226 | || issued < inner.options.pool_max as usize 227 | }, 228 | } 229 | .await; 230 | if !ok { 231 | continue; 232 | } 233 | 234 | if inner.closed() { 235 | return Err(DriverError::PoolDisconnected.into()); 236 | } 237 | 238 | if let Ok(conn) = self.inner.new.pop() { 239 | let mut conn = Connection::new(self.clone(), conn); 240 | 241 | if inner.options.ping_before_query { 242 | let mut c = inner.options.send_retries; 243 | loop { 244 | if conn.ping().await.is_ok() { 245 | return Ok(conn); 246 | } 247 | 248 | if c <= 1 { 249 | break; 250 | } 251 | 252 | delay_for(inner.options.retry_timeout).await; 253 | c -= 1; 254 | } 255 | 256 | conn.set_deteriorated(); 257 | self.return_connection(conn); 258 | continue; 259 | } else { 260 | return Ok(conn); 261 | } 262 | } else { 263 | let mut lock = inner.lock.lock(); 264 | 265 | if *lock < inner.options.pool_max as usize { 266 | // reserve quota... 267 | *lock += 1; 268 | // ...and goto create new connection releasing lock for other clients 269 | break; 270 | } 271 | }; 272 | } 273 | // create new connection 274 | for addr in self.get_addr_iter() { 275 | match InnerConection::init(&inner.options, addr).await { 276 | Ok(conn) => { 277 | return Ok(Connection::new(self.clone(), conn)); 278 | } 279 | Err(err) => { 280 | // On timeout repeat connection with another address 281 | if !err.is_timeout() { 282 | return Err(err); 283 | } 284 | } 285 | } 286 | } 287 | 288 | // Release quota if failed to establish new connection 289 | let mut lock = inner.lock.lock(); 290 | *lock -= 1; 291 | inner.wakers.notify_any(); 292 | drop(lock); 293 | Err(crate::errors::DriverError::ConnectionClosed.into()) 294 | } 295 | 296 | /// Take a connection back to pool 297 | #[inline] 298 | pub fn return_connection(&self, mut conn: Connection) { 299 | // NOTE: It's safe to call it out of tokio runtime 300 | let pool = conn.pool.take(); 301 | debug_assert_eq!(pool.as_ref().unwrap(), self); 302 | 303 | self.inner.return_connection(conn.take()); 304 | } 305 | 306 | // #[cfg(not(feature = "recycle"))] 307 | // #[inline] 308 | // fn send_to_recycler(&self, conn: Box) { 309 | // disconnect(conn); 310 | // } 311 | // 312 | // #[cfg(feature = "recycle")] 313 | // #[inline] 314 | // fn send_to_recycler(&self, conn: Box) { 315 | // if let Err(conn) = self.drop.send(Some(conn)) { 316 | // let conn = conn.0.unwrap(); 317 | // // This _probably_ means that the Runtime is shutting down, and that the Recycler was 318 | // // dropped rather than allowed to exit cleanly. 319 | // if self.inner.close.load(atomic::Ordering::SeqCst) != POOL_STATUS_STOPPED { 320 | // // Yup, Recycler was forcibly dropped! 321 | // // All we can do here is try the non-pool drop path for Conn. 322 | // drop(conn); 323 | // } else { 324 | // unreachable!("Recycler exited while connections still exist"); 325 | // } 326 | // } 327 | // } 328 | /// Close the pool and all issued connections 329 | pub fn disconnect(self) -> DisconnectPool { 330 | DisconnectPool::new(self) 331 | } 332 | /// Iterate over available host 333 | fn get_addr_iter(&'_ self) -> AddrIter<'_> { 334 | let inner = &self.inner; 335 | let index = if inner.hosts.len() > 1 { 336 | inner.connections_num.fetch_add(1, Ordering::Relaxed) 337 | } else { 338 | inner.connections_num.load(Ordering::Relaxed) 339 | }; 340 | util::AddrIter::new(inner.hosts.as_slice(), index, self.options().send_retries) 341 | } 342 | /// Return Option object used for creation pool 343 | /// @note! the option does not have hosts 344 | #[inline] 345 | pub fn options(&self) -> &Options { 346 | &self.inner.options 347 | } 348 | } 349 | 350 | impl PartialEq for Pool { 351 | fn eq(&self, other: &Pool) -> bool { 352 | Arc::ptr_eq(&self.inner, &other.inner) 353 | } 354 | } 355 | 356 | #[cfg(test)] 357 | mod test { 358 | use super::builder::PoolBuilder; 359 | use super::Result; 360 | 361 | #[tokio::test] 362 | async fn test_build_pool() -> Result<()> { 363 | let pool = PoolBuilder::default() 364 | .with_database("default") 365 | .with_username("default") 366 | .add_addr("www.yandex.ru:9000") 367 | .build() 368 | .unwrap(); 369 | 370 | assert_eq!(pool.options().username, "default"); 371 | assert_eq!(pool.inner.hosts[0], "www.yandex.ru:9000"); 372 | 373 | Ok(()) 374 | } 375 | } 376 | -------------------------------------------------------------------------------- /clickhouse-driver/src/pool/recycler.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | future::Future, 3 | pin::Pin, 4 | sync::{atomic::Ordering, Arc}, 5 | task::{Context, Poll}, 6 | }; 7 | 8 | use futures::stream::Stream; 9 | use tokio::sync::mpsc; 10 | 11 | use super::{Inner, InnerConection, POOL_STATUS_STOPPED}; 12 | 13 | pub(crate) struct Recycler { 14 | dropped: mpsc::UnboundedReceiver>>, 15 | inner: Arc, 16 | eof: bool, 17 | } 18 | 19 | impl Recycler { 20 | pub(crate) fn new( 21 | inner: Arc, 22 | dropped: mpsc::UnboundedReceiver>>, 23 | ) -> Recycler { 24 | Recycler { 25 | inner, 26 | dropped, 27 | eof: false, 28 | } 29 | } 30 | } 31 | 32 | impl Future for Recycler { 33 | type Output = (); 34 | 35 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 36 | //let mut close = self.inner.close.load(Ordering::Acquire); 37 | loop { 38 | match Pin::new(&mut self.dropped).poll_next(cx) { 39 | Poll::Ready(Some(Some(_conn))) => {} 40 | Poll::Ready(Some(None)) => { 41 | break; 42 | } 43 | Poll::Ready(None) => { 44 | // no more connections are coming -- time to exit! 45 | self.inner 46 | .close 47 | .store(POOL_STATUS_STOPPED, Ordering::Release); 48 | self.eof = true; 49 | } 50 | Poll::Pending => { 51 | if self.eof { 52 | let wait = self.inner.wait.load(Ordering::Acquire); 53 | if wait == 0 { 54 | break; 55 | } 56 | //awake waiting pool task and repeat 57 | self.inner.notifyer.notify(); 58 | } 59 | 60 | return Poll::Pending; 61 | } 62 | } 63 | } 64 | Poll::Ready(()) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /clickhouse-driver/src/pool/util.rs: -------------------------------------------------------------------------------- 1 | /// Server host addresses iterator 2 | /// 3 | /// New establishing connection chooses random host address from 4 | /// provided list. This implementation use round-robin strategy for 5 | /// selecting. A host may be temporary unavailable, so we repeatedly 6 | /// check it again up to 'repeat' count times . 7 | pub(crate) struct AddrIter<'a> { 8 | hosts: &'a [String], 9 | start: u16, 10 | i: u16, 11 | max: u16, 12 | } 13 | 14 | impl<'a> AddrIter<'a> { 15 | #[inline] 16 | pub(super) fn new(hosts: &'a [String], start: usize, repeat: u8) -> AddrIter<'a> { 17 | AddrIter { 18 | hosts, 19 | i: 0, 20 | start: (start % hosts.len()) as u16, 21 | max: (hosts.len() * repeat as usize) as u16, 22 | } 23 | } 24 | } 25 | 26 | impl<'a> Iterator for AddrIter<'a> { 27 | type Item = &'a str; 28 | 29 | fn next(&mut self) -> Option { 30 | if self.i < self.max { 31 | let r = &self.hosts[(self.i + self.start) as usize % (self.hosts.len())]; 32 | self.i += 1; 33 | Some(r) 34 | } else { 35 | None 36 | } 37 | } 38 | } 39 | 40 | /// Connection pool status 41 | /// - The number of idle connections in pool 42 | /// - The total number of issued connections including idle and active 43 | /// - The number of tasks that are waiting for available connection 44 | #[derive(Debug)] 45 | pub struct PoolInfo { 46 | pub idle: usize, 47 | pub issued: usize, 48 | pub wait: usize, 49 | } 50 | -------------------------------------------------------------------------------- /clickhouse-driver/src/prelude.rs: -------------------------------------------------------------------------------- 1 | pub use crate::{ 2 | client::Connection, 3 | pool::options::{CompressionMethod, Options}, 4 | //client::QueryResult, 5 | pool::{Pool, PoolBuilder}, 6 | protocol::{ 7 | block::{Block, ServerBlock}, 8 | column::{Deserialize, Row}, 9 | }, 10 | }; 11 | pub mod types { 12 | pub use crate::protocol::value::{ 13 | ValueDateTime, ValueDateTime64, ValueDecimal32, ValueDecimal64, ValueUuid, 14 | }; 15 | #[cfg(feature = "int128")] 16 | pub use crate::types::Decimal128; 17 | pub use crate::types::{Decimal, Decimal32, Decimal64, DecimalBits}; 18 | } 19 | pub mod errors { 20 | pub use crate::errors::*; 21 | } 22 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/block.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | fmt, 4 | io::{self, Write}, 5 | iter::Iterator, 6 | }; 7 | 8 | use super::code::*; 9 | use super::column::{AsInColumn, ColumnDataAdapter, Deserialize, Row}; 10 | use super::encoder::Encoder; 11 | use super::value::IntoColumn; 12 | use super::ServerWriter; 13 | use crate::client::ServerInfo; 14 | use crate::compression::LZ4CompressionWrapper; 15 | use crate::types::{Field, FIELD_NONE, FIELD_NULLABLE}; 16 | use chrono_tz::Tz; 17 | 18 | pub struct RowIterator<'a> { 19 | block: &'a ServerBlock, 20 | id: u64, 21 | } 22 | 23 | impl<'a> Iterator for RowIterator<'a> { 24 | type Item = Row<'a>; 25 | fn next(&mut self) -> Option> { 26 | if self.id >= self.block.rows { 27 | None 28 | } else { 29 | let id = self.id; 30 | self.id += 1; 31 | let row = unsafe { Row::create(self.block, id) }; 32 | Some(row) 33 | } 34 | } 35 | } 36 | 37 | pub struct ItemIterator<'a, D: Deserialize> { 38 | block: &'a ServerBlock, 39 | id: u64, 40 | phantom: PhantomData<&'a D>, 41 | } 42 | 43 | impl<'a, D: Deserialize> Iterator for ItemIterator<'a, D> { 44 | type Item = D; 45 | fn next(&mut self) -> Option { 46 | if self.id >= self.block.rows { 47 | None 48 | } else { 49 | let id = self.id; 50 | self.id += 1; 51 | let row = unsafe { Row::create(self.block, id) }; 52 | Some(::deserialize(row).expect("unexpected deserialization error")) 53 | } 54 | } 55 | } 56 | 57 | /// Inserting data column struct 58 | /// Clickhouse serialization format 59 | /// ------column------------- 60 | /// FNM TNM DATA 61 | /// |---|---|---------| 62 | /// FNM - field name, Clickhouse table column name 63 | /// TNM - type name, Clickhouse sql serialized field type (Int64, String, FixedString(10)...) 64 | /// DATA - serialized data array, has data specific format. Integer data (u)int8|16|32|64|128, 65 | /// float f32|f64, Decimal and other fixed length data 66 | /// (Date, DateTime, UUID, Enum8, Enum16...) are serialized as array of little-endian binary representation. 67 | /// String column is serialized as Variant String - VarInt string length + string byte array 68 | /// FixedString is serialized as array of string data (without length) 69 | /// Nullable data type precedes array of null flags represented as array of u8 where 0-null, 1-notnull 70 | pub(crate) struct BlockInfo { 71 | pub(super) cols: u64, 72 | pub(super) rows: u64, 73 | pub(super) overflow: bool, 74 | pub(super) bucket: u32, 75 | } 76 | 77 | impl std::fmt::Debug for BlockInfo { 78 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 79 | f.debug_struct("BlockInfo") 80 | .field("columns", &self.cols) 81 | .field("rows", &self.rows) 82 | .field("overflow", &self.overflow) 83 | .field("bucket", &self.bucket) 84 | .finish() 85 | } 86 | } 87 | 88 | impl BlockInfo { 89 | /// Clickhouse empty block is used as a marker of end of data transfer 90 | pub(super) fn is_empty(&self) -> bool { 91 | self.rows == 0 && self.cols == 0 92 | } 93 | } 94 | /// Output data holder 95 | pub struct Block<'b> { 96 | columns: Vec>, 97 | /// Clickhouse overflow flag. @note not used now 98 | overflow: u8, 99 | /// The number of rows in each column 100 | rows: usize, 101 | /// Clickhouse table name 102 | pub(crate) table: &'b str, 103 | } 104 | 105 | impl<'b> Block<'b> { 106 | pub fn new(table: &'b str) -> Block<'b> { 107 | Block { 108 | overflow: 0, 109 | columns: Vec::new(), 110 | rows: 0, 111 | table, 112 | } 113 | } 114 | 115 | /// Returns the number of columns 116 | #[inline] 117 | pub fn column_count(&self) -> usize { 118 | self.columns.len() 119 | } 120 | /// Returns the number of rows 121 | #[inline] 122 | pub fn row_count(&self) -> usize { 123 | self.rows 124 | } 125 | 126 | /// Returns whether the block has any columns 127 | #[inline] 128 | pub fn is_empty(&self) -> bool { 129 | self.columns.is_empty() 130 | } 131 | /// Iterate over collection of columns. 132 | /// Each column has it own data, so it's wrapped by adapter that provides common interface 133 | /// for data encoding and type check 134 | pub fn column_iter(&self) -> std::slice::Iter { 135 | self.columns.iter() 136 | } 137 | 138 | /// Check if the added column has the same number of rows as the others 139 | fn set_rows(&mut self, rows: usize) { 140 | if !self.columns.is_empty() { 141 | if self.rows != rows { 142 | panic!("block columns must have the same length") 143 | } 144 | } else { 145 | self.rows = rows; 146 | }; 147 | } 148 | /// Add new column to the block 149 | /// NOTE! columns should be added in order of INSERT query 150 | pub fn add(mut self, name: &'b str, data: Vec) -> Self 151 | where 152 | T: IntoColumn<'b>, 153 | { 154 | self.set_rows(data.len()); 155 | 156 | self.columns.push(ColumnDataAdapter { 157 | name, 158 | flag: FIELD_NONE, 159 | data: IntoColumn::to_column(data), 160 | }); 161 | self 162 | } 163 | /// Add new column to block. 164 | /// In contrast to `add` method, here we add column for Nullable data types. 165 | /// Option None value is used as null data. 166 | pub fn add_nullable(mut self, name: &'b str, data: Vec>) -> Self 167 | where 168 | Option: IntoColumn<'b>, 169 | T: Default, 170 | { 171 | self.set_rows(data.len()); 172 | 173 | self.columns.push(ColumnDataAdapter { 174 | name, 175 | flag: FIELD_NULLABLE, 176 | data: IntoColumn::to_column(data), 177 | }); 178 | self 179 | } 180 | } 181 | /// Block column name and type. 182 | /// 183 | /// 184 | /// BlockColumnHeader is a part of column set of ServerBlock and used to 185 | /// retrieve raw server data and convert it to rust data type for SELECT statements; 186 | /// or to validate output data for INSERT statements. 187 | /// To prevent superfluous server load by provided bunch of incorrect data 188 | /// driver should validate the Block against server table structure 189 | /// before it been sent to server. 190 | pub struct BlockColumnHeader { 191 | pub(crate) field: Field, 192 | pub(crate) name: String, 193 | } 194 | 195 | impl fmt::Debug for BlockColumnHeader { 196 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 197 | f.debug_struct("Pool") 198 | .field("name", &self.name) 199 | .field("field.type", &self.field.sql_type) 200 | .finish() 201 | } 202 | } 203 | 204 | /// input Block column data. 205 | pub struct BlockColumn { 206 | pub(crate) header: BlockColumnHeader, 207 | pub(crate) data: Box, 208 | } 209 | 210 | impl BlockColumn { 211 | pub(crate) fn into_header(self) -> BlockColumnHeader { 212 | self.header 213 | } 214 | } 215 | 216 | impl std::fmt::Debug for BlockColumn { 217 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 218 | f.debug_struct("BlockColumn") 219 | .field("sql_type", &self.header.field.sql_type) 220 | .field("name", &self.header.name) 221 | .finish() 222 | } 223 | } 224 | 225 | #[derive(Debug)] 226 | pub struct ServerBlock { 227 | pub(crate) columns: Vec, 228 | pub(crate) rows: u64, 229 | pub(crate) timezone: Tz, 230 | } 231 | 232 | impl ServerBlock { 233 | #[inline] 234 | pub(crate) fn into_columns(self) -> Vec { 235 | self.columns 236 | } 237 | 238 | #[inline] 239 | pub(crate) fn into_headers(self) -> Vec { 240 | self.into_columns() 241 | .into_iter() 242 | .map(|c| c.into_header()) 243 | .collect() 244 | } 245 | 246 | pub fn iter_rows(&self) -> RowIterator { 247 | RowIterator { block: self, id: 0 } 248 | } 249 | pub fn iter(&self) -> ItemIterator { 250 | ItemIterator { 251 | block: self, 252 | id: 0, 253 | phantom: PhantomData, 254 | } 255 | } 256 | 257 | #[inline] 258 | pub fn column_count(&self) -> u64 { 259 | self.columns.len() as u64 260 | } 261 | #[inline] 262 | pub fn row_count(&self) -> u64 { 263 | self.rows 264 | } 265 | } 266 | 267 | /// Provide specific implementation of ServerWriter trait for Empty and Data blocks 268 | pub(crate) trait AsBlock { 269 | fn dump(&self, cx: &ServerInfo, writer: &mut dyn Write) -> std::io::Result<()>; 270 | } 271 | 272 | impl ServerWriter for B { 273 | /// Serialize Block according to server capabilities and client settings. 274 | /// Compress content by LZ4 if corresponding option is set 275 | fn write(&self, cx: &ServerInfo, writer: &mut dyn Write) -> std::io::Result<()> { 276 | CLIENT_DATA.encode(writer)?; 277 | 278 | // Temporary table 279 | if cx.revision >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES { 280 | ().encode(writer)?; 281 | } 282 | 283 | if !cx.compression.is_none() { 284 | let mut compress = LZ4CompressionWrapper::new(writer); 285 | self.dump(cx, &mut compress)?; 286 | compress.flush() 287 | } else { 288 | self.dump(cx, writer)?; 289 | writer.flush() 290 | } 291 | } 292 | } 293 | /// Empty block message is used as specific SERVER_DATA message 294 | /// and indicate the end of data stream 295 | /// In response to this message Clickhouse returns success of failure status 296 | pub struct EmptyBlock; 297 | 298 | /// Optimized byte sequence for empty block 299 | impl AsBlock for EmptyBlock { 300 | /// Write block content to output stream 301 | #[inline] 302 | fn dump(&self, cx: &ServerInfo, writer: &mut dyn Write) -> std::io::Result<()> { 303 | let revision = cx.revision; 304 | if revision >= DBMS_MIN_REVISION_WITH_BLOCK_INFO { 305 | // 1 [0] - ? 306 | // 0 [1] - overflow 307 | // 2 [2] - ? 308 | // -1 [3..7] - bucket num as int32 309 | // 0 [8] - ? 310 | 311 | [1u8, 0, 2, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0] 312 | .as_ref() 313 | .encode(writer) 314 | } else { 315 | // columns, rows 316 | [0u8, 0u8].as_ref().encode(writer) 317 | } 318 | } 319 | } 320 | /// OutputBlockWrapper is adapter that combine provided by caller 321 | /// columns data (Block) and 322 | /// Clickhouse server table metadata ( BlockColumnHeader[] ) 323 | pub(super) struct OutputBlockWrapper<'b> { 324 | pub(super) inner: &'b Block<'b>, 325 | pub(super) columns: &'b Vec, 326 | } 327 | 328 | impl OutputBlockWrapper<'_> { 329 | fn is_empty(&self) -> bool { 330 | self.columns.is_empty() 331 | } 332 | } 333 | 334 | impl<'b> AsBlock for OutputBlockWrapper<'b> { 335 | /// Write block content to output stream 336 | fn dump(&self, cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()> { 337 | if self.is_empty() { 338 | return EmptyBlock.dump(cx, writer); 339 | } 340 | 341 | let revision = cx.revision; 342 | if revision >= DBMS_MIN_REVISION_WITH_BLOCK_INFO { 343 | [1u8, self.inner.overflow, 2, 0xFF, 0xFF, 0xFF, 0xFF, 0] 344 | .as_ref() 345 | .encode(writer)?; 346 | }; 347 | 348 | (self.columns.len() as u64).encode(writer)?; 349 | (self.inner.rows as u64).encode(writer)?; 350 | 351 | for (head, col) in self.columns.iter().zip(self.inner.columns.iter()) { 352 | head.name.encode(writer)?; 353 | head.field.encode(writer)?; 354 | col.data.encode(&head.field, writer)?; 355 | } 356 | 357 | Ok(()) 358 | } 359 | } 360 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/code.rs: -------------------------------------------------------------------------------- 1 | //pub const DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO: u64 = 54060; 2 | //pub const DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS: u64 = 54429; 3 | 4 | pub const DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES: u32 = 50264; 5 | pub const DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS: u32 = 51554; 6 | pub const DBMS_MIN_REVISION_WITH_BLOCK_INFO: u32 = 51903; 7 | pub const DBMS_MIN_REVISION_WITH_CLIENT_INFO: u32 = 54032; 8 | pub const DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE: u32 = 54058; 9 | pub const DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO: u32 = 54060; 10 | //pub const DBMS_MIN_REVISION_WITH_TABLES_STATUS 54226 11 | //pub const DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337 12 | pub const DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME: u32 = 54372; 13 | pub const DBMS_MIN_REVISION_WITH_VERSION_PATCH: u32 = 54401; 14 | #[allow(dead_code)] 15 | pub const DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE: u32 = 54405; 16 | 17 | // Client message packet types 18 | pub const CLIENT_HELLO: u64 = 0; 19 | pub const CLIENT_QUERY: u64 = 1; 20 | pub const CLIENT_DATA: u64 = 2; 21 | pub const CLIENT_CANCEL: u64 = 3; 22 | pub const CLIENT_PING: u64 = 4; 23 | 24 | pub const STATE_COMPLETE: u8 = 2; 25 | 26 | // Server message packet types 27 | pub const SERVER_HELLO: u64 = 0; 28 | pub const SERVER_DATA: u64 = 1; 29 | pub const SERVER_EXCEPTION: u64 = 2; 30 | pub const SERVER_PROGRESS: u64 = 3; 31 | pub const SERVER_PONG: u64 = 4; 32 | pub const SERVER_END_OF_STREAM: u64 = 5; 33 | pub const SERVER_PROFILE_INFO: u64 = 6; 34 | // pub const SERVER_TOTALS: u64 = 7; 35 | // pub const SERVER_EXTREMES: u64 = 8; 36 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/decoder.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::{ConversionError, DriverError, Result}; 2 | use core::marker::PhantomData; 3 | use std::future::Future; 4 | use std::pin::Pin; 5 | use std::task::{Context, Poll}; 6 | use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt}; 7 | 8 | /// Read string data encoded as VarInt(length) + bytearray 9 | pub(crate) struct ReadVString<'a, T: FromBytes, R> { 10 | length_: usize, 11 | data: Vec, 12 | inner: Pin<&'a mut R>, 13 | _marker: PhantomData<&'a T>, 14 | } 15 | 16 | pub trait FromBytes: Sized { 17 | fn from_bytes(bytes: &mut Vec) -> Result; 18 | } 19 | 20 | impl FromBytes for String { 21 | #[inline] 22 | fn from_bytes(bytes: &mut Vec) -> Result { 23 | let b = std::mem::take(bytes); 24 | String::from_utf8(b).map_err(|_e| ConversionError::Utf8.into()) 25 | } 26 | } 27 | 28 | impl FromBytes for Vec { 29 | #[inline] 30 | fn from_bytes(bytes: &mut Vec) -> Result { 31 | Ok(std::mem::take(bytes)) 32 | } 33 | } 34 | 35 | impl<'a, T: FromBytes, R: AsyncRead> ReadVString<'a, T, R> { 36 | pub(crate) fn new(reader: &'a mut R, length: usize) -> ReadVString<'a, T, R> { 37 | let data = unsafe { 38 | let mut v = Vec::with_capacity(length); 39 | v.set_len(length); 40 | v 41 | }; 42 | let inner = unsafe { Pin::new_unchecked(reader) }; 43 | ReadVString { 44 | length_: 0, 45 | data, 46 | inner, 47 | _marker: PhantomData, 48 | } 49 | } 50 | 51 | fn poll_get(&mut self, cx: &mut Context<'_>) -> Poll> { 52 | loop { 53 | if self.length_ == self.data.len() { 54 | return FromBytes::from_bytes(&mut self.data).into(); 55 | } else { 56 | self.length_ += ready!(self 57 | .inner 58 | .as_mut() 59 | .poll_read(cx, &mut self.data[self.length_..])?); 60 | } 61 | } 62 | } 63 | } 64 | 65 | impl<'a, T: FromBytes, R: AsyncRead> Future for ReadVString<'a, T, R> { 66 | type Output = Result; 67 | 68 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 69 | let me = &mut *self; 70 | me.poll_get(cx) 71 | } 72 | } 73 | /// Read VarInt 74 | pub(crate) struct ReadVInt<'a, R> { 75 | value: u64, 76 | i: u8, 77 | inner: Pin<&'a mut R>, 78 | } 79 | 80 | impl<'a, R: AsyncRead> ReadVInt<'a, R> { 81 | fn new(reader: &'a mut R) -> ReadVInt<'a, R> { 82 | let inner = unsafe { Pin::new_unchecked(reader) }; 83 | ReadVInt { 84 | value: 0, 85 | i: 0, 86 | inner, 87 | } 88 | } 89 | 90 | fn poll_get(&mut self, cx: &mut Context<'_>) -> Poll> { 91 | let mut b = [0u8; 1]; 92 | loop { 93 | //let inner: Pin<&mut R> = unsafe{ Pin::new_unchecked(self.inner) }; 94 | if 0 == ready!(self.inner.as_mut().poll_read(cx, &mut b)?) { 95 | return Poll::Ready(Err(DriverError::BrokenData.into())); 96 | } 97 | let b = b[0]; 98 | 99 | self.value |= ((b & 0x7f) as u64) << (self.i); 100 | self.i += 7; 101 | 102 | if b < 0x80 { 103 | return Poll::Ready(Ok(self.value)); 104 | }; 105 | 106 | if self.i > 63 { 107 | return Poll::Ready(Err(DriverError::BrokenData.into())); 108 | }; 109 | } 110 | } 111 | } 112 | 113 | impl<'a, R: AsyncRead> Future for ReadVInt<'a, R> { 114 | type Output = Result; 115 | 116 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 117 | let me = &mut *self; 118 | me.poll_get(cx) 119 | } 120 | } 121 | 122 | pub struct ValueReader { 123 | inner: R, 124 | } 125 | 126 | impl ValueReader { 127 | pub(super) fn new(reader: R) -> ValueReader { 128 | ValueReader { inner: reader } 129 | } 130 | //TODO: Optimize reading note that reader is buffered data 131 | pub(super) fn read_vint(&mut self) -> ReadVInt<'_, R> { 132 | ReadVInt::new(&mut self.inner) 133 | } 134 | //TODO: Optimize reading note that reader is buffered data 135 | pub(super) fn read_string(&mut self, len: u64) -> ReadVString<'_, T, R> { 136 | ReadVString::new(&mut self.inner, len as usize) 137 | } 138 | 139 | #[inline] 140 | pub(super) fn as_mut(&mut self) -> &mut R { 141 | &mut self.inner 142 | } 143 | } 144 | 145 | pub(crate) struct Skip<'a, R> { 146 | value: usize, 147 | inner: Pin<&'a mut R>, 148 | } 149 | 150 | impl<'a, R: AsyncBufRead> Skip<'a, R> { 151 | pub(super) fn poll_skip(&mut self, cx: &mut Context<'_>) -> Poll> { 152 | while self.value > 0 { 153 | let buf = ready!(self.inner.as_mut().poll_fill_buf(cx)?); 154 | let n = std::cmp::min(self.value, buf.len()); 155 | self.inner.as_mut().consume(n); 156 | self.value -= n; 157 | } 158 | Ok(()).into() 159 | } 160 | } 161 | 162 | impl<'a, R: AsyncBufRead> Future for Skip<'a, R> { 163 | type Output = Result<()>; 164 | 165 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 166 | let me = &mut *self; 167 | me.poll_skip(cx) 168 | } 169 | } 170 | 171 | impl ValueReader { 172 | pub(super) fn skip(&mut self, len: u64) -> Skip<'_, R> { 173 | Skip { 174 | value: len as usize, 175 | inner: Pin::new(&mut self.inner), 176 | } 177 | } 178 | 179 | pub(super) async fn read_byte(&mut self) -> Result { 180 | let mut buf = [0u8; 1]; 181 | self.inner.read_exact(&mut buf[..]).await?; 182 | 183 | Ok(buf[0]) 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/encoder.rs: -------------------------------------------------------------------------------- 1 | use byteorder::WriteBytesExt; 2 | use std::io::{Result, Write}; 3 | use uuid::Uuid; 4 | 5 | pub trait Encoder { 6 | fn encode(&self, writer: &mut dyn Write) -> Result<()>; 7 | } 8 | 9 | impl Encoder for u64 { 10 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 11 | //let mut i = 0; 12 | let mut mx = *self; 13 | 14 | while mx >= 0x80 { 15 | writer.write_u8(mx as u8 | 0x80)?; 16 | mx >>= 7; 17 | //i += 1; 18 | } 19 | 20 | writer.write_u8(mx as u8)?; 21 | Ok(()) 22 | } 23 | } 24 | 25 | impl Encoder for u8 { 26 | #[inline] 27 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 28 | writer.write_u8(*self)?; 29 | Ok(()) 30 | } 31 | } 32 | 33 | impl Encoder for &[u8] { 34 | #[inline] 35 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 36 | writer.write_all(*self)?; 37 | Ok(()) 38 | } 39 | } 40 | 41 | impl Encoder for &str { 42 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 43 | let l: u64 = self.len() as u64; 44 | l.encode(writer)?; 45 | writer.write_all(self.as_bytes())?; 46 | Ok(()) 47 | } 48 | } 49 | 50 | impl Encoder for String { 51 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 52 | let l: u64 = self.len() as u64; 53 | l.encode(writer)?; 54 | writer.write_all(self.as_bytes())?; 55 | Ok(()) 56 | } 57 | } 58 | 59 | /// Empty string or 0u8 byte 60 | impl Encoder for () { 61 | #[inline] 62 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 63 | writer.write_u8(0)?; 64 | Ok(()) 65 | } 66 | } 67 | 68 | impl Encoder for Uuid { 69 | fn encode(&self, writer: &mut dyn Write) -> Result<()> { 70 | if self.is_nil() { 71 | writer.write_u8(0)?; 72 | return Ok(()); 73 | } 74 | 75 | let bytes = self.as_bytes(); 76 | debug_assert!(bytes.len() == 16); 77 | 16u8.encode(writer)?; 78 | writer.write_all(bytes)?; 79 | Ok(()) 80 | } 81 | } 82 | 83 | pub struct StringEncoderAdapter { 84 | buf: Vec, 85 | inner: W, 86 | } 87 | 88 | impl StringEncoderAdapter { 89 | pub fn new(writer: W) -> StringEncoderAdapter { 90 | StringEncoderAdapter { 91 | buf: Vec::new(), 92 | inner: writer, 93 | } 94 | } 95 | } 96 | 97 | impl Write for StringEncoderAdapter { 98 | fn write(&mut self, buf: &[u8]) -> Result { 99 | self.buf.write(buf) 100 | } 101 | 102 | fn flush(&mut self) -> Result<()> { 103 | let l: u64 = self.buf.len() as u64; 104 | l.encode(&mut self.inner)?; 105 | self.inner.write_all(&self.buf[..])?; 106 | 107 | self.buf.clear(); 108 | Ok(()) 109 | } 110 | } 111 | 112 | impl Drop for StringEncoderAdapter { 113 | fn drop(&mut self) { 114 | self.flush().expect("TODO") 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/insert.rs: -------------------------------------------------------------------------------- 1 | use futures::TryFutureExt; 2 | use std::marker::Unpin; 3 | use tokio::io::{AsyncRead, AsyncWrite}; 4 | 5 | use super::block::{Block, BlockColumnHeader, EmptyBlock, OutputBlockWrapper, ServerBlock}; 6 | use super::command::ResponseStream; 7 | use crate::errors::{ConversionError, DriverError, Result}; 8 | 9 | use super::ServerWriter; 10 | use crate::protocol::column::{ColumnDataAdapter, EnumIndex}; 11 | 12 | const DEFAULT_INSERT_BUFFER_SIZE: usize = 8 * 1024; 13 | 14 | pub struct InsertSink<'a, R: AsyncRead + AsyncWrite + Unpin + Send> { 15 | pub(crate) inner: ResponseStream<'a, R>, 16 | pub(crate) buf: Vec, 17 | #[allow(dead_code)] 18 | pub(crate) columns: Vec, 19 | } 20 | 21 | impl<'a, R: AsyncRead + AsyncWrite + Unpin + Send> Drop for InsertSink<'a, R> { 22 | fn drop(&mut self) { 23 | self.inner.clear_pending() 24 | } 25 | } 26 | 27 | impl<'a, R: AsyncRead + AsyncWrite + Unpin + Send> InsertSink<'a, R> { 28 | pub(crate) fn new(tcpstream: ResponseStream<'a, R>, block: ServerBlock) -> InsertSink<'a, R> { 29 | let buf = Vec::with_capacity(DEFAULT_INSERT_BUFFER_SIZE); 30 | 31 | let mut columns = block.into_headers(); 32 | // prepare Enum8 and Enum16 for string->data conversion by sorting index by string 33 | for column in columns.iter_mut() { 34 | if let Some(meta) = column.field.get_meta_mut() { 35 | meta.index.sort_unstable_by(EnumIndex::fn_sort_str); 36 | } 37 | } 38 | 39 | InsertSink { 40 | inner: tcpstream, 41 | buf, 42 | columns, 43 | } 44 | } 45 | /// Send another block of data to Clickhouse server. 46 | /// It's part of streamed insert API. 47 | /// Streamed insert is the most effective method when you need 48 | /// to load into server huge data . 49 | pub async fn next(&mut self, data: &Block<'_>) -> Result<()> { 50 | self.buf.clear(); 51 | // The number of Columns must be the same! 52 | // As a precaution check it 53 | if data.column_count() != self.columns.len() { 54 | return Err(DriverError::BrokenData.into()); 55 | } 56 | // TODO: split huge block on chunks less then MAX_BLOCK_SIZE size each 57 | // Now the caller responsible to split data 58 | if data.row_count() > crate::MAX_BLOCK_SIZE { 59 | return Err(DriverError::BrokenData.into()); 60 | } 61 | let compatible = |(head, col): (&BlockColumnHeader, &ColumnDataAdapter)| { 62 | head.name.eq(col.name) 63 | && head.field.flag == col.flag 64 | && col.data.is_compatible(&head.field) 65 | }; 66 | // For efficiency we check input data and column data format compatibility only once 67 | // before serialization. We just check if columns names and nullability flags of 68 | // table structure and provided data block match to each over. 69 | // Also we make basic check the ability to convert Sql data type to rust data. 70 | if !self.columns.iter().zip(data.column_iter()).all(compatible) { 71 | return Err(ConversionError::UnsupportedConversion.into()); 72 | } 73 | 74 | // TODO: get rid of intermediate buffer. Write block right into stream 75 | OutputBlockWrapper { 76 | columns: &self.columns, 77 | inner: data, 78 | } 79 | .write(self.inner.info_ref(), &mut self.buf)?; 80 | 81 | self.inner 82 | .write(self.buf.as_slice()) 83 | .map_err(Into::into) 84 | .await 85 | } 86 | /// Commits last inserted blocks returning server insert status. 87 | /// @note. Clickhouse does not support ASID transactions. 88 | /// There is no way to revert a transaction. 89 | /// Commit just allow to get status of previously inserted blocks. 90 | /// If it returns an error, you can send last blocks again. 91 | /// Clickhouse keeps hashes of last 100 blocks. So you can safely 92 | /// without duplication commit only every hundredth block. 93 | /// 94 | /// # Example 95 | /// for (i,chunk) in chanks.enumerate(){ 96 | /// con.insert(Block::new("log") 97 | /// .add("id", chunk.vec_of_id) 98 | /// .add("message", chunk.vec_of_messages)).await?; 99 | /// if i%100 == 0 { 100 | /// status = con.commit().await?; 101 | /// if status.is_err(){ 102 | /// todo with error 103 | /// } 104 | /// } 105 | /// } 106 | pub async fn commit(&mut self) -> Result<()> { 107 | self.buf.clear(); 108 | EmptyBlock.write(self.inner.info_ref(), &mut self.buf)?; 109 | self.inner.write(self.buf.as_slice()).await?; 110 | 111 | if let Some(packet) = self.inner.next().await? { 112 | return Err(DriverError::PacketOutOfOrder(packet.code()).into()); 113 | } 114 | // Disable fuse. it allows us to make intermediate commits 115 | // self.inner.set_fuse(); 116 | Ok(()) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/mod.rs: -------------------------------------------------------------------------------- 1 | //! # Clickhouse Server protocol 2 | //! Clickhouse uses for communication proprietary protocol and bespoke binary message formats. 3 | //! The protocol is message-oriented request-response based and uses TCP (TLS) as transport layer. 4 | //! Official documentation doesn't have detailed information about it. 5 | //! This driver implementation is entirely based on source code of others open-source clickhouse drivers 6 | //! and tcp dump analysis. 7 | //! 8 | //! 9 | //! ## SERVER_DATA and CLIENT_DATA 10 | //! --------message---------- 11 | //! PKID TMPF BLK or CBLK ( 1 or more) 12 | //! 1 0 13 | //! | - | - | ----------- | 14 | //! 15 | //! PKID - Packet ID (1-CLIENT DATA; 2-SERVER DATA), 1 byte 16 | //! TMPF - Temporary Table Name. (0-NULL) 17 | //! BLK - Raw Block 18 | //! CBLK - Compressed Block 19 | //! 20 | //! --------raw block------------ 21 | //! BI CN RN COLS 22 | //! |--|--|--|------| 23 | //! 24 | //! BI - Block Info 25 | //! CN - Number of Columns (varint) 26 | //! RN - Number of Rows (varint) 27 | //! COLS - Columns serialized data, data type specific 28 | //! 29 | //! --------block info--------- 30 | //! ? OVF ? BUCK ? 31 | //! |-| - |-|----|-| 32 | //! 33 | //! OVF - Overflow flag, 0/1 34 | //! BUCK - Bucket number, -1 - not specified (default) 35 | //! 36 | //! --------compressed block---- 37 | //! CTH ALG CSZ DSZ LZ4 BLK 38 | //! |---| - |---|---|-------| 39 | //! 40 | //! CTH - 16 byte CityHash 1.0 Hashsum CityHash128(ALG---LZ4 BLK) 41 | //! ALG - 1 byte Algorithm of compression. 0x82 - LZ4 42 | //! CSZ - Compressed Block size , [ALG .. LZ4 BLK] 43 | //! DSZ - Decompressed Block size 44 | //! LZ4 BLK - LZ4 compressed Raw Block 45 | //! 46 | //! ## Block data 47 | //! Block contains series of columns each of them has header and body 48 | //! -------- block (BLK ) --------------------- 49 | //! 1-column 2-column next 50 | //! | BCH | BCB | BCH | BCB | .... 51 | //! BCH - Block column header 52 | //! BCB - Block column body 53 | //! 54 | //! --------- block header (BCH) ------------- 55 | //! | NM | TYPE | CD | 56 | //! NM - table column name 57 | //! TYPE - table column type and metadata (i.e Enum type has it dictionary) 58 | //! CD - column data. 59 | //! 60 | //! Column data format is type dependant. 61 | //! ## Column values serialization 62 | //! Each data type has it own binary format 63 | //! 64 | //! (U)Int8(16|32|64) and Float32|Float64 data types encoded as the corresponding numeric type in little endian order. 65 | //! String data type is encoded as sequence of length + data, there length is varInt encoded 66 | //! FixedString(len) is encoded as sequence of data with fixed length 67 | //! Date - 2 byte values. Little endian 16 bit signed integer - number of days from epoch (1 Jan 1970 ) 68 | //! DateTime - 4 byte value. Little endian 32 bit signed integer - number of seconds from epoch 69 | //! DateTime64 - 8 byte value. Little endian 64 bit signed integer - number of ticks from epoch. 70 | //! The size in seconds of each tick is defined by scale factor of DateTime64 type metadata. 71 | //! Scale is 1 byte integer and can take values from 0 to 9. 72 | //! 0 - each second is 1 tick 73 | //! 1 - each second is 10 ticks 74 | //! n - each second is 10^n ticks 75 | //! UUID - 16 byte value 76 | //! IPv4 and IPv6 as corresponding 4 and 16 byte big-endian values (network endian) 77 | //! Enum8 and Enum16 encoded as array of 1 (or 2 for Enum16) integer values which are keys in 78 | //! enum dictionary 79 | //! 80 | //! Nullable(BaseType) has additional array of nullable flags. This array has 1-byte values of 81 | //! 0 - (not null ) , 1 - null. The array has the same length as BaseType data. 82 | //! Null array is stored in column format before Base type. 83 | 84 | use crate::client::ServerInfo; 85 | use crate::errors::{ConversionError, Result}; 86 | use crate::types::Field; 87 | use std::io::{self, Write}; 88 | #[cfg(feature = "int128")] 89 | use value::ValueDecimal128; 90 | use value::{ 91 | ValueDate, ValueDateTime, ValueDateTime64, ValueDecimal32, ValueDecimal64, ValueIp4, ValueIp6, 92 | ValueUuid, 93 | }; 94 | 95 | #[cfg(feature = "extra")] 96 | mod simd; 97 | 98 | pub mod block; 99 | pub mod code; 100 | pub mod column; 101 | pub mod command; 102 | mod decoder; 103 | pub mod encoder; 104 | pub mod insert; 105 | pub mod packet; 106 | pub mod query; 107 | pub mod value; 108 | 109 | /// This trait provides common interface for client request serialization. 110 | /// ServerInfo parameter keeps server specific options (revision, compression method ...) 111 | /// and defines encoded rules, version specific options, and timezone. 112 | pub(crate) trait ServerWriter { 113 | fn write(&self, cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()>; 114 | } 115 | 116 | /// Read the RIP register (instruction pointer). 117 | /// Used to detect code duplicates 118 | #[cfg(rustc_nightly)] 119 | #[feature(asm)] 120 | #[allow(dead_code)] 121 | #[inline(always)] 122 | pub(crate) fn rip() -> u64 { 123 | let rip: u64; 124 | unsafe { 125 | asm!("lea {}, 0[rip]", out(reg) rip); 126 | } 127 | rip 128 | } 129 | 130 | /// Gateway from rust to dynamic sql data of Clickhouse Row 131 | #[derive(Debug)] 132 | pub enum ValueRefEnum<'a> { 133 | String(&'a [u8]), 134 | Array8(&'a [u8]), 135 | Array16(&'a [u16]), 136 | Array32(&'a [u32]), 137 | Array64(&'a [u64]), 138 | Array128(&'a [u128]), 139 | UInt8(u8), 140 | UInt16(u16), 141 | UInt32(u32), 142 | UInt64(u64), 143 | Int8(i8), 144 | Int16(i16), 145 | Int32(i32), 146 | Int64(i64), 147 | #[cfg(feature = "int128")] 148 | UInt128(u128), 149 | Float32(f32), 150 | Float64(f64), 151 | Date(ValueDate), 152 | DateTime(ValueDateTime), 153 | DateTime64(ValueDateTime64), 154 | Enum(i16), 155 | Ip4(ValueIp4), 156 | Ip6(ValueIp6), 157 | Uuid(ValueUuid), 158 | Decimal32(ValueDecimal32), 159 | Decimal64(ValueDecimal64), 160 | #[cfg(feature = "int128")] 161 | Decimal128(ValueDecimal128), 162 | } 163 | 164 | /// Row value getter interface 165 | pub trait Value<'a, T: 'a> { 166 | fn get(&'a self, _: &'a Field) -> Result> { 167 | Err(ConversionError::UnsupportedConversion.into()) 168 | } 169 | } 170 | 171 | #[cfg(test)] 172 | mod test { 173 | use super::ValueRefEnum; 174 | use crate::protocol::column::{EnumIndex, ValueRef}; 175 | use crate::types::FieldMeta; 176 | use std::mem::size_of; 177 | 178 | macro_rules! into_boxed { 179 | ($s: expr) => { 180 | $s.to_owned().into_boxed_str().into_boxed_bytes() 181 | }; 182 | } 183 | 184 | #[test] 185 | fn test_meta_index() { 186 | let index = vec![ 187 | EnumIndex(1i16, into_boxed!("yes")), 188 | EnumIndex(-1i16, into_boxed!("n/a")), 189 | EnumIndex(0i16, into_boxed!("no")), 190 | EnumIndex(-2i16, into_boxed!("unknown")), 191 | ]; 192 | let mut meta = FieldMeta { index }; 193 | //sort by value 194 | meta.index.sort_unstable_by_key(EnumIndex::fn_sort_val); 195 | assert_eq!(meta.index[0].0, -2); 196 | assert_eq!(meta.index[1].0, -1); 197 | assert_eq!(meta.index[2].0, 0); 198 | assert_eq!(meta.index[3].0, 1); 199 | 200 | assert_eq!(meta.val2str(-1), b"n/a"); 201 | assert_eq!(meta.val2str(0), b"no"); 202 | assert_eq!(meta.val2str(-2), b"unknown"); 203 | assert_eq!(meta.val2str(1), b"yes"); 204 | 205 | meta.index.sort_unstable_by(EnumIndex::fn_sort_str); 206 | 207 | assert_eq!(meta.str2val(b"no").unwrap(), 0i16); 208 | assert_eq!(meta.str2val(b"yes").unwrap(), 1i16); 209 | assert_eq!(meta.str2val(b"unknown").unwrap(), -2i16); 210 | assert_eq!(meta.str2val(b"n/a").unwrap(), -1i16); 211 | assert!(meta.str2val(b"some other").is_err()); 212 | } 213 | 214 | #[test] 215 | fn test_valueref_size() { 216 | let valueref_size = size_of::(); 217 | let valuerefenum_size = size_of::(); 218 | 219 | assert_eq!(valueref_size, valuerefenum_size); 220 | assert!( 221 | valueref_size <= 24, 222 | "ValueRef should be smaller than 32 bytes. 16 bytes - data + 8 bytes descriptor" 223 | ); 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/packet.rs: -------------------------------------------------------------------------------- 1 | use super::code::*; 2 | use super::encoder::Encoder; 3 | use super::query::Query; 4 | use super::ServerWriter; 5 | use crate::client::ServerInfo; 6 | use crate::pool::Options; 7 | use crate::protocol::block::{EmptyBlock, ServerBlock}; 8 | use chrono_tz::Tz; 9 | use std::{io, io::Write}; 10 | // OUT 11 | 12 | pub(crate) trait Command: ServerWriter {} 13 | 14 | pub(crate) struct Hello<'a> { 15 | pub(crate) opt: &'a Options, 16 | } 17 | 18 | impl Command for Hello<'_> {} 19 | 20 | impl<'a> ServerWriter for Hello<'a> { 21 | fn write(&self, _cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()> { 22 | CLIENT_HELLO.encode(writer)?; 23 | 24 | crate::CLIENT_NAME.encode(writer)?; 25 | crate::CLICK_HOUSE_DBMSVERSION_MAJOR.encode(writer)?; 26 | crate::CLICK_HOUSE_DBMSVERSION_MINOR.encode(writer)?; 27 | crate::CLICK_HOUSE_REVISION.encode(writer)?; 28 | 29 | self.opt.database.encode(writer)?; 30 | self.opt.username.encode(writer)?; 31 | self.opt.password.encode(writer)?; 32 | Ok(()) 33 | } 34 | } 35 | 36 | pub(crate) struct Ping; 37 | 38 | impl Command for Ping {} 39 | 40 | impl ServerWriter for Ping { 41 | fn write(&self, _cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()> { 42 | CLIENT_PING.encode(writer) 43 | } 44 | } 45 | 46 | pub(crate) struct Cancel; 47 | 48 | impl Command for Cancel {} 49 | 50 | impl ServerWriter for Cancel { 51 | fn write(&self, _cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()> { 52 | CLIENT_CANCEL.encode(writer) 53 | } 54 | } 55 | 56 | pub(crate) struct Execute { 57 | pub(crate) query: Query, 58 | } 59 | 60 | impl Command for Execute {} 61 | 62 | impl ServerWriter for Execute { 63 | fn write(&self, cx: &ServerInfo, writer: &mut dyn Write) -> io::Result<()> { 64 | CLIENT_QUERY.encode(writer)?; 65 | ().encode(writer)?; 66 | // Query string (SELECT, INSERT or DDL ) 67 | self.query.write(cx, writer)?; 68 | // Write empty block as a marker of the stream end 69 | EmptyBlock.write(cx, writer)?; 70 | Ok(()) 71 | } 72 | } 73 | 74 | // IN 75 | 76 | pub(crate) struct ProfileInfo { 77 | pub rows: u64, 78 | pub bytes: u64, 79 | pub blocks: u64, 80 | pub applied_limit: u8, 81 | pub rows_before_limit: u64, 82 | pub calculated_rows_before_limit: u8, 83 | } 84 | 85 | impl std::fmt::Debug for ProfileInfo { 86 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 87 | f.debug_struct("ProfileInfo") 88 | .field("rows", &self.rows) 89 | .field("bytes", &self.bytes) 90 | .field("blocks", &self.blocks) 91 | .field("application limit", &self.applied_limit) 92 | .field("rows before limit", &self.rows_before_limit) 93 | .field( 94 | "calculated rows before limit", 95 | &self.calculated_rows_before_limit, 96 | ) 97 | .finish() 98 | } 99 | } 100 | 101 | pub struct Statistics { 102 | rows: u64, 103 | bytes: u64, 104 | total: u64, 105 | } 106 | 107 | impl std::fmt::Debug for Statistics { 108 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 109 | f.debug_struct("Statistics") 110 | .field("rows", &self.rows) 111 | .field("bytes", &self.bytes) 112 | .field("total", &self.total) 113 | .finish() 114 | } 115 | } 116 | 117 | #[derive(Debug)] 118 | pub(crate) enum Response { 119 | Pong, 120 | Data(ServerBlock), 121 | Hello(String, u64, u64, u64, Tz), 122 | // Eos, 123 | // Profile(u64, u64, u64, u8, u8), 124 | // Totals, 125 | // Extremes, 126 | } 127 | 128 | impl Response { 129 | pub(crate) fn code(&self) -> u64 { 130 | match self { 131 | Response::Pong => SERVER_PONG, 132 | Response::Data(_) => SERVER_DATA, 133 | Response::Hello(..) => SERVER_HELLO, 134 | // Response::Eos => SERVER_END_OF_STREAM, 135 | // Response::Extremes => SERVER_EXTREMES, 136 | // Response::Totals => SERVER_TOTALS, 137 | // Response::Excepion(..) => SERVER_EXCEPTION, 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/query.rs: -------------------------------------------------------------------------------- 1 | use super::block::Block; 2 | use super::code::*; 3 | use super::encoder::Encoder; 4 | use super::{ServerInfo, ServerWriter}; 5 | use std::io::{Result, Write}; 6 | use uuid::Uuid; 7 | 8 | /// Query encapsulates Clickhouse sql command 9 | pub struct Query { 10 | pub(crate) id: Option, 11 | pub(crate) query: String, 12 | } 13 | 14 | macro_rules! add_col { 15 | ($query:ident, $name:expr) => { 16 | $query.push_str("`"); 17 | $query.push_str(&$name); 18 | $query.push_str("`"); 19 | }; 20 | } 21 | 22 | impl Query { 23 | /// Construct `INSERT` statement from Block columns names 24 | pub(crate) fn from_block(block: &Block) -> Query { 25 | let mut query: String = format!("INSERT INTO {} (", block.table); 26 | query.reserve(256); 27 | let mut iter = block.column_iter(); 28 | if let Some(col) = iter.next() { 29 | add_col!(query, col.name); 30 | 31 | for col in iter { 32 | query.push_str(","); 33 | add_col!(query, col.name); 34 | } 35 | } 36 | query.push_str(") VALUES"); 37 | Query { query, id: None } 38 | } 39 | } 40 | 41 | impl From<&str> for Query { 42 | fn from(sql: &str) -> Self { 43 | let id = Uuid::new_v4(); 44 | Query { 45 | id: Some(id), 46 | query: sql.into(), 47 | } 48 | } 49 | } 50 | 51 | impl From for Query { 52 | fn from(sql: String) -> Self { 53 | Query::from(sql.as_str()) 54 | } 55 | } 56 | 57 | impl ServerWriter for Query { 58 | fn write(&self, cx: &ServerInfo, writer: &mut dyn Write) -> Result<()> { 59 | if cx.revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO { 60 | // Query kind 61 | 1u8.encode(writer)?; 62 | // Initial user 63 | ().encode(writer)?; 64 | // Query id 65 | self.id.unwrap_or_default().encode(writer)?; 66 | // Network address 67 | "[::ffff:127.0.0.1]:0".encode(writer)?; 68 | // Network iface 69 | 1u8.encode(writer)?; 70 | // OS user name 71 | ().encode(writer)?; 72 | // Hostname 73 | crate::HOSTNAME.encode(writer)?; 74 | // Client name and version 75 | crate::CLIENT_NAME.encode(writer)?; 76 | crate::CLICK_HOUSE_DBMSVERSION_MAJOR.encode(writer)?; 77 | crate::CLICK_HOUSE_DBMSVERSION_MINOR.encode(writer)?; 78 | crate::CLICK_HOUSE_REVISION.encode(writer)?; 79 | 80 | if cx.revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO { 81 | // Quota key 82 | ().encode(writer)?; 83 | } 84 | 85 | if cx.revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH { 86 | // Client version patch 87 | 0u8.encode(writer)?; 88 | }; 89 | }; 90 | 91 | "max_block_size".encode(writer)?; 92 | (crate::MAX_BLOCK_SIZE as u64).encode(writer)?; 93 | let ro = cx.readonly; 94 | if ro != 0 { 95 | "readonly".encode(writer)?; 96 | ro.encode(writer)? 97 | } 98 | // Empty string end up settings block 99 | ().encode(writer)?; 100 | 101 | // Stage:Complete 102 | STATE_COMPLETE.encode(writer)?; 103 | let compression: u8 = if cx.compression.is_none() { 0 } else { 1 }; 104 | 105 | // Compression (disable=0, enable=1) 106 | compression.encode(writer)?; 107 | 108 | // Query string 109 | self.query.encode(writer)?; 110 | writer.flush() 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /clickhouse-driver/src/protocol/simd.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_arch = "x86")] 2 | use core::arch::x86::*; 3 | #[cfg(target_arch = "x86_64")] 4 | use core::arch::x86_64::*; 5 | 6 | /// Convert byte array to array of bits 7 | /// 0 byte - bit set 8 | /// 1 byte - bit not set 9 | #[allow(dead_code)] 10 | #[target_feature(enable = "avx2")] 11 | #[allow(clippy::cast_ptr_alignment)] 12 | pub unsafe fn load_packed_flag(flag: &[u8]) -> Box<[u32]> { 13 | let wcount: usize = flag.len() / 32; 14 | let rem = 32 - (flag.len() & 0x1f); 15 | 16 | let mut res: Vec = Vec::with_capacity(if rem < 32 { wcount + 1 } else { wcount }); 17 | 18 | res.set_len(res.capacity()); 19 | 20 | let one: __m256i = _mm256_set1_epi8(0x01); 21 | let mut src: *const __m256i = flag.as_ptr() as *const __m256i; 22 | let mut dst = res.as_mut_ptr(); 23 | for _ in 0..wcount { 24 | let x = _mm256_lddqu_si256(src); 25 | let m = _mm256_sub_epi8(x, one); 26 | *dst = _mm256_movemask_epi8(m) as u32; 27 | src = src.add(1); 28 | dst = dst.add(1); 29 | } 30 | // last 1 to 32 bytes 31 | if rem < 32 { 32 | // read last 32 bytes 33 | src = flag.as_ptr().add(flag.len() - 32) as *const __m256i; 34 | let x = _mm256_lddqu_si256(src); 35 | let m = _mm256_sub_epi8(x, one); 36 | *dst = (_mm256_movemask_epi8(m) as u32) >> rem; 37 | } 38 | 39 | res.into_boxed_slice() 40 | } 41 | 42 | #[cfg(test)] 43 | mod test { 44 | use super::*; 45 | #[test] 46 | fn load_packed_flag_bound() { 47 | let b1 = [ 48 | 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 49 | 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50 | 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 51 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 52 | 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 53 | ]; 54 | let packed = unsafe { load_packed_flag(&b1[..]) }; 55 | 56 | //println!("res {:b} {:b} {:b}",packed[0],packed[1],packed[2]); 57 | assert_eq!( 58 | packed.as_ref(), 59 | &[0x1e0a01e3_u32, 0xffffffff, 0x00, 0xff00ff] 60 | ); 61 | } 62 | 63 | #[test] 64 | fn load_packed_flag_unbound() { 65 | let b1 = [ 66 | 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 67 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 68 | ]; 69 | let packed = unsafe { load_packed_flag(&b1[..]) }; 70 | assert_eq!(packed.as_ref(), &[0x1e0a01e3_u32, 0xff00]); 71 | // swap 2 bytes of each word 72 | let b1 = [ 73 | 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 74 | 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 75 | ]; 76 | let packed = unsafe { load_packed_flag(&b1[..]) }; 77 | assert_eq!(packed.as_ref(), &[0x0a1e01e3_u32, 0xff]); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /clickhouse-driver/src/sync/mod.rs: -------------------------------------------------------------------------------- 1 | // TODO: draw up condvar such a https://docs.rs/async-std/1.6.2/async_std/sync/struct.Condvar.html 2 | // Condvar is better in task synchronization than tokio notifier in our case. 3 | // While stopping pool we have to signal all pending tasks that it is not possible for notifier. 4 | // 5 | mod condvar {} 6 | mod waker_set; 7 | pub use waker_set::WakerSet; 8 | -------------------------------------------------------------------------------- /clickhouse-driver/src/sync/waker_set.rs: -------------------------------------------------------------------------------- 1 | //! A common utility for building synchronization primitives. 2 | //! 3 | //! When an async operation is blocked, it needs to register itself somewhere so that it can be 4 | //! notified later on. The `WakerSet` type helps with keeping track of such async operations and 5 | //! notifying them when they may make progress. 6 | 7 | use crossbeam::utils::Backoff; 8 | use slab::Slab; 9 | use std::cell::UnsafeCell; 10 | use std::ops::{Deref, DerefMut}; 11 | use std::sync::atomic::{AtomicUsize, Ordering}; 12 | use std::task::{Context, Waker}; 13 | 14 | /// Set when the entry list is locked. 15 | #[allow(clippy::identity_op)] 16 | const LOCKED: usize = 1 << 0; 17 | 18 | /// Set when there is at least one entry that has already been notified. 19 | const NOTIFIED: usize = 1 << 1; 20 | 21 | /// Set when there is at least one notifiable entry. 22 | const NOTIFIABLE: usize = 1 << 2; 23 | 24 | /// Inner representation of `WakerSet`. 25 | struct Inner { 26 | /// A list of entries in the set. 27 | /// 28 | /// Each entry has an optional waker associated with the task that is executing the operation. 29 | /// If the waker is set to `None`, that means the task has been woken up but hasn't removed 30 | /// itself from the `WakerSet` yet. 31 | /// 32 | /// The key of each entry is its index in the `Slab`. 33 | entries: Slab>, 34 | 35 | /// The number of notifiable entries. 36 | notifiable: usize, 37 | } 38 | 39 | /// A set holding wakers. 40 | pub struct WakerSet { 41 | /// Holds three bits: `LOCKED`, `NOTIFY_ONE`, and `NOTIFY_ALL`. 42 | flag: AtomicUsize, 43 | 44 | /// A set holding wakers. 45 | inner: UnsafeCell, 46 | } 47 | 48 | unsafe impl Sync for WakerSet {} 49 | 50 | #[allow(dead_code)] 51 | impl WakerSet { 52 | /// Creates a new `WakerSet`. 53 | #[inline] 54 | pub fn new() -> WakerSet { 55 | WakerSet { 56 | flag: AtomicUsize::new(0), 57 | inner: UnsafeCell::new(Inner { 58 | entries: Slab::new(), 59 | notifiable: 0, 60 | }), 61 | } 62 | } 63 | 64 | /// Inserts a waker for a blocked operation and returns a key associated with it. 65 | #[cold] 66 | pub fn insert(&self, cx: &Context<'_>) -> usize { 67 | let w = cx.waker().clone(); 68 | let mut inner = self.lock(); 69 | 70 | let key = inner.entries.insert(Some(w)); 71 | inner.notifiable += 1; 72 | key 73 | } 74 | 75 | /// Removes the waker of an operation. 76 | #[cold] 77 | pub fn remove(&self, key: usize) { 78 | let mut inner = self.lock(); 79 | 80 | if inner.entries.remove(key).is_some() { 81 | inner.notifiable -= 1; 82 | } 83 | } 84 | 85 | /// If the waker for this key is still waiting for a notification, then update 86 | /// the waker for the entry, and return false. If the waker has been notified, 87 | /// treat the entry as completed and return true. 88 | pub fn remove_if_notified(&self, key: usize, cx: &Context<'_>) -> bool { 89 | let mut inner = self.lock(); 90 | 91 | match &mut inner.entries[key] { 92 | None => { 93 | inner.entries.remove(key); 94 | true 95 | } 96 | Some(w) => { 97 | // We were never woken, so update instead 98 | if !w.will_wake(cx.waker()) { 99 | *w = cx.waker().clone(); 100 | } 101 | false 102 | } 103 | } 104 | } 105 | 106 | /// Removes the waker of a cancelled operation. 107 | /// 108 | /// Returns `true` if another blocked operation from the set was notified. 109 | #[cold] 110 | pub fn cancel(&self, key: usize) -> bool { 111 | let mut inner = self.lock(); 112 | 113 | match inner.entries.remove(key) { 114 | Some(_) => inner.notifiable -= 1, 115 | None => { 116 | // The operation was cancelled and notified so notify another operation instead. 117 | for (_, opt_waker) in inner.entries.iter_mut() { 118 | // If there is no waker in this entry, that means it was already woken. 119 | if let Some(w) = opt_waker.take() { 120 | w.wake(); 121 | inner.notifiable -= 1; 122 | return true; 123 | } 124 | } 125 | } 126 | } 127 | 128 | false 129 | } 130 | 131 | /// Notifies a blocked operation if none have been notified already. 132 | /// 133 | /// Returns `true` if an operation was notified. 134 | #[inline] 135 | pub fn notify_any(&self) -> bool { 136 | // Use `SeqCst` ordering to synchronize with `Lock::drop()`. 137 | let flag = self.flag.load(Ordering::SeqCst); 138 | 139 | if flag & NOTIFIED == 0 && flag & NOTIFIABLE != 0 { 140 | self.notify(Notify::Any) 141 | } else { 142 | false 143 | } 144 | } 145 | 146 | /// Notifies one additional blocked operation. 147 | /// 148 | /// Returns `true` if an operation was notified. 149 | #[inline] 150 | pub fn notify_one(&self) -> bool { 151 | // Use `SeqCst` ordering to synchronize with `Lock::drop()`. 152 | if self.flag.load(Ordering::SeqCst) & NOTIFIABLE != 0 { 153 | self.notify(Notify::One) 154 | } else { 155 | false 156 | } 157 | } 158 | 159 | /// Notifies all blocked operations. 160 | /// 161 | /// Returns `true` if at least one operation was notified. 162 | #[inline] 163 | pub fn notify_all(&self) -> bool { 164 | // Use `SeqCst` ordering to synchronize with `Lock::drop()`. 165 | if self.flag.load(Ordering::SeqCst) & NOTIFIABLE != 0 { 166 | self.notify(Notify::All) 167 | } else { 168 | false 169 | } 170 | } 171 | 172 | /// Notifies blocked operations, either one or all of them. 173 | /// 174 | /// Returns `true` if at least one operation was notified. 175 | #[cold] 176 | fn notify(&self, n: Notify) -> bool { 177 | let mut inner = &mut *self.lock(); 178 | let mut notified = false; 179 | 180 | for (_, opt_waker) in inner.entries.iter_mut() { 181 | // If there is no waker in this entry, that means it was already woken. 182 | if let Some(w) = opt_waker.take() { 183 | w.wake(); 184 | inner.notifiable -= 1; 185 | notified = true; 186 | 187 | if n == Notify::One { 188 | break; 189 | } 190 | } 191 | 192 | if n == Notify::Any { 193 | break; 194 | } 195 | } 196 | 197 | notified 198 | } 199 | 200 | /// Locks the list of entries. 201 | fn lock(&self) -> Lock<'_> { 202 | let backoff = Backoff::new(); 203 | while self.flag.fetch_or(LOCKED, Ordering::Acquire) & LOCKED != 0 { 204 | backoff.snooze(); 205 | } 206 | Lock { waker_set: self } 207 | } 208 | 209 | pub fn len(&self) -> usize { 210 | let inner = &mut *self.lock(); 211 | inner 212 | .entries 213 | .iter() 214 | .filter(|(_, item)| item.is_some()) 215 | .count() 216 | } 217 | } 218 | 219 | /// A guard holding a `WakerSet` locked. 220 | struct Lock<'a> { 221 | waker_set: &'a WakerSet, 222 | } 223 | 224 | impl Drop for Lock<'_> { 225 | #[inline] 226 | fn drop(&mut self) { 227 | let mut flag = 0; 228 | 229 | // Set the `NOTIFIED` flag if there is at least one notified entry. 230 | if self.entries.len() - self.notifiable > 0 { 231 | flag |= NOTIFIED; 232 | } 233 | 234 | // Set the `NOTIFIABLE` flag if there is at least one notifiable entry. 235 | if self.notifiable > 0 { 236 | flag |= NOTIFIABLE; 237 | } 238 | 239 | // Use `SeqCst` ordering to synchronize with `WakerSet::lock_to_notify()`. 240 | self.waker_set.flag.store(flag, Ordering::SeqCst); 241 | } 242 | } 243 | 244 | impl Deref for Lock<'_> { 245 | type Target = Inner; 246 | 247 | #[inline] 248 | fn deref(&self) -> &Inner { 249 | unsafe { &*self.waker_set.inner.get() } 250 | } 251 | } 252 | 253 | impl DerefMut for Lock<'_> { 254 | #[inline] 255 | fn deref_mut(&mut self) -> &mut Inner { 256 | unsafe { &mut *self.waker_set.inner.get() } 257 | } 258 | } 259 | 260 | /// Notification strategy. 261 | #[derive(Clone, Copy, Eq, PartialEq)] 262 | enum Notify { 263 | /// Make sure at least one entry is notified. 264 | Any, 265 | /// Notify one additional entry. 266 | One, 267 | /// Notify all entries. 268 | All, 269 | } 270 | -------------------------------------------------------------------------------- /clickhouse-driver/src/types/decimal.rs: -------------------------------------------------------------------------------- 1 | use crate::types::SCALE; 2 | use std::convert::From; 3 | use std::fmt::{self, Display, Formatter, Result}; 4 | use std::ops::{Div, Mul, Neg, Sub}; 5 | 6 | pub trait DecimalBits { 7 | /// Common precision for this datatype 8 | fn precision() -> u8; 9 | /// Check if datatype can hold value with certain precision 10 | /// For i32 it's 0-9, i64: 10-18, i128: 19-27 11 | fn fit(precision: u8) -> bool; 12 | } 13 | macro_rules! bits { 14 | ($t:ty => $min:expr, $max: expr) => { 15 | impl DecimalBits for $t { 16 | #[inline] 17 | fn precision() -> u8 { 18 | $max 19 | } 20 | #[allow(unused_comparisons)] 21 | fn fit(precision: u8) -> bool { 22 | precision <= $max && precision >= $min 23 | } 24 | } 25 | }; 26 | } 27 | 28 | bits!(i32 => 0, 9); 29 | bits!(i64 => 10, 18); 30 | #[cfg(feature = "int128")] 31 | bits!(i128 => 19, 27); 32 | 33 | /// Provides arbitrary-precision floating point decimal. 34 | #[derive(Clone)] 35 | pub struct Decimal { 36 | pub(crate) underlying: T, 37 | pub(crate) precision: u8, 38 | pub(crate) scale: u8, 39 | } 40 | 41 | pub type Decimal32 = Decimal; 42 | pub type Decimal64 = Decimal; 43 | #[cfg(feature = "int128")] 44 | pub type Decimal128 = Decimal; 45 | 46 | impl fmt::Debug for Decimal { 47 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 48 | f.debug_struct("Decimal") 49 | .field("underlying", &self.underlying) 50 | .field("precision", &self.precision) 51 | .field("scale", &self.scale) 52 | .finish() 53 | } 54 | } 55 | 56 | impl Decimal { 57 | #[inline] 58 | pub fn from_parts(underlying: T, precision: u8, scale: u8) -> Decimal { 59 | Decimal { 60 | underlying, 61 | precision, 62 | scale, 63 | } 64 | } 65 | 66 | #[inline] 67 | pub fn from(underlying: T, scale: u8) -> Decimal { 68 | Decimal { 69 | underlying, 70 | precision: ::precision(), 71 | scale, 72 | } 73 | } 74 | 75 | #[inline(always)] 76 | pub fn internal(&self) -> T { 77 | self.underlying 78 | } 79 | 80 | #[inline(always)] 81 | pub fn set_internal(&mut self, value: T) { 82 | self.underlying = value; 83 | } 84 | 85 | /// Determines how many decimal digits fraction can have. 86 | #[inline] 87 | pub fn scale(&self) -> usize { 88 | self.scale as usize 89 | } 90 | } 91 | 92 | impl Default for Decimal { 93 | fn default() -> Self { 94 | Decimal { 95 | underlying: Default::default(), 96 | precision: T::precision(), 97 | scale: 0, 98 | } 99 | } 100 | } 101 | 102 | fn format(val: T, scale: usize, f: &mut Formatter<'_>) -> Result 103 | where 104 | T: Display 105 | + Default 106 | + From 107 | + Copy 108 | + PartialOrd 109 | + Neg 110 | + Sub 111 | + Mul 112 | + Div, 113 | { 114 | if scale == 0 { 115 | f.write_fmt(format_args!("{}", val)) 116 | } else { 117 | let (val, sign) = if val < Default::default() { 118 | (-val, "-") 119 | } else { 120 | (val, "") 121 | }; 122 | // 128 bit workaround. Because MAGNITUDE has maximum 10^18 value 123 | // larger values can be obtained as 10^x = 10^18 * 10^(x-18) 124 | // it's case for i128 values only and can be optimized 125 | let div: T = if scale < 19 { 126 | SCALE[scale].into() 127 | } else { 128 | Into::::into(SCALE[18]) * Into::::into(SCALE[scale - 18]) 129 | }; 130 | 131 | let h = val / div; 132 | let r = val - h * div; 133 | 134 | f.write_fmt(format_args!( 135 | "{sign}{h:}.{r:0>scale$}", 136 | h = h, 137 | r = r, 138 | scale = scale, 139 | sign = sign 140 | )) 141 | } 142 | } 143 | 144 | impl Display for Decimal { 145 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 146 | format(self.underlying as i64, self.scale as usize, f) 147 | } 148 | } 149 | 150 | impl Display for Decimal { 151 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 152 | format(self.underlying, self.scale as usize, f) 153 | } 154 | } 155 | #[cfg(feature = "int128")] 156 | impl Display for Decimal { 157 | fn fmt(&self, f: &mut Formatter<'_>) -> Result { 158 | format(self.underlying, self.scale as usize, f) 159 | } 160 | } 161 | 162 | #[cfg(test)] 163 | mod test { 164 | use super::*; 165 | #[test] 166 | fn test_decimal_format() { 167 | assert_eq!("1002", Decimal::from(1002_i32, 0).to_string()); 168 | assert_eq!("-1002", Decimal::from(-1002_i32, 0).to_string()); 169 | assert_eq!("100.2", Decimal::from(1002_i32, 1).to_string()); 170 | assert_eq!("10.02", Decimal::from(1002_i32, 2).to_string()); 171 | assert_eq!("0.1002", Decimal::from(1002_i32, 4).to_string()); 172 | assert_eq!("0.001002", Decimal::from(1002_i32, 6).to_string()); 173 | assert_eq!("-10.02", Decimal::from(-1002_i32, 2).to_string()); 174 | assert_eq!("-0.001002", Decimal::from(-1002_i32, 6).to_string()); 175 | } 176 | 177 | #[test] 178 | fn test_decimal_range() { 179 | assert_eq!(i32::fit(8), true); 180 | assert_ne!(i32::fit(10), true); 181 | assert_ne!(i64::fit(6), true); 182 | assert_eq!(i64::fit(14), true); 183 | } 184 | 185 | #[test] 186 | #[cfg(feature = "int128")] 187 | fn test_decimal_format_128() { 188 | assert_eq!( 189 | "3.14159265358979323846", 190 | Decimal::from(3_1415926535_8979323846_i128, 20).to_string() 191 | ); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /clickhouse-driver/src/types/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "int128")] 2 | pub use crate::protocol::value::ValueDecimal128; 3 | pub use crate::protocol::value::{ 4 | ValueDate, ValueDateTime, ValueDateTime64, ValueDecimal32, ValueDecimal64, ValueIp4, ValueIp6, 5 | ValueUuid, 6 | }; 7 | use chrono_tz::Tz; 8 | #[cfg(feature = "int128")] 9 | pub use decimal::Decimal128; 10 | pub use decimal::{Decimal, Decimal32, Decimal64, DecimalBits}; 11 | use std::fmt; 12 | use std::io; 13 | use std::io::Write; 14 | 15 | use crate::errors::{ConversionError, Result}; 16 | use crate::protocol::column::EnumIndex; 17 | use crate::protocol::encoder::{Encoder, StringEncoderAdapter}; 18 | pub use parser::parse_type_field; 19 | use std::fmt::Formatter; 20 | 21 | mod decimal; 22 | mod parser; 23 | 24 | /// Column field metadata 25 | /// for now it's used for Enum data type `value <-> string` transcode 26 | /// in serialization/deserialization process 27 | pub struct FieldMeta { 28 | pub(crate) index: Vec>, 29 | } 30 | 31 | impl FieldMeta { 32 | /// Searches enum value by string representation 33 | /// Return i16 value regardless of the data type Enum8 or Enum16 34 | /// For Enum8 returning value can be safely cast to i8 35 | /// 36 | /// Require index to be sorted by str. 37 | /// `meta.index.sort_unstable_by(EnumIndex::fn_sort_str)` 38 | #[inline] 39 | pub(crate) fn str2val(&self, title: &[u8]) -> Result { 40 | let index = self 41 | .index 42 | .binary_search_by(|item| item.1.as_ref().cmp(title)) 43 | .map_err(|_| ConversionError::EnumMismatch(title.to_vec()))?; 44 | 45 | debug_assert!(index < self.index.len(), "enum index corrupted"); 46 | Ok(unsafe { self.index.get_unchecked(index).0 }) 47 | } 48 | #[inline] 49 | pub(crate) fn val2str(&self, val: i16) -> &[u8] { 50 | let index = self 51 | .index 52 | .binary_search_by_key(&val, EnumIndex::fn_sort_val) 53 | .expect("enum index corrupted"); 54 | 55 | debug_assert!(index < self.index.len(), "enum index corrupted"); 56 | unsafe { self.index.get_unchecked(index).1.as_ref() } 57 | } 58 | } 59 | 60 | pub const FIELD_NONE: u8 = 0x00; 61 | pub const FIELD_NULLABLE: u8 = 0x01; 62 | pub const FIELD_LOWCARDINALITY: u8 = 0x02; 63 | pub const FIELD_ARRAY: u8 = 0x04; 64 | 65 | pub struct Field { 66 | pub(crate) sql_type: SqlType, 67 | pub(crate) flag: u8, 68 | pub(crate) depth: u8, 69 | meta: Option, 70 | } 71 | 72 | impl fmt::Debug for Field { 73 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 74 | f.debug_struct("Field") 75 | .field("sql_type", &self.sql_type) 76 | .field("flag", &self.flag) 77 | .field("depth", &self.depth) 78 | .finish() 79 | } 80 | } 81 | 82 | impl Clone for Field { 83 | fn clone(&self) -> Self { 84 | Field { 85 | sql_type: self.sql_type, 86 | flag: self.flag, 87 | depth: self.depth, 88 | meta: { 89 | match &self.meta { 90 | None => None, 91 | Some(meta) => Some(FieldMeta { 92 | index: meta.index.to_vec(), 93 | }), 94 | } 95 | }, 96 | } 97 | } 98 | } 99 | 100 | impl Field { 101 | pub fn get_meta(&self) -> Option<&FieldMeta> { 102 | self.meta.as_ref() 103 | } 104 | pub fn get_meta_mut(&mut self) -> Option<&mut FieldMeta> { 105 | self.meta.as_mut() 106 | } 107 | 108 | #[inline] 109 | pub fn is_nullable(&self) -> bool { 110 | (self.flag & FIELD_NULLABLE) == FIELD_NULLABLE 111 | } 112 | #[inline] 113 | pub fn is_array(&self) -> bool { 114 | (self.flag & FIELD_ARRAY) == FIELD_ARRAY 115 | } 116 | #[inline] 117 | pub fn is_lowcardinality(&self) -> bool { 118 | (self.flag & FIELD_LOWCARDINALITY) == FIELD_LOWCARDINALITY 119 | } 120 | 121 | fn write_enum(meta: &FieldMeta, writer: &mut dyn Write) -> io::Result<()> { 122 | let mut iter = meta.index.iter(); 123 | write!(writer, "(")?; 124 | if let Some(item) = iter.next() { 125 | //writer.write_fmt( )?; 126 | write!(writer, "{}", item)?; 127 | } 128 | for item in iter { 129 | write!(writer, ",{}", item)?; 130 | } 131 | 132 | write!(writer, ")")?; 133 | Ok(()) 134 | } 135 | } 136 | 137 | impl Encoder for Field { 138 | fn encode(&self, writer: &mut dyn Write) -> io::Result<()> { 139 | let mut type_adapter = StringEncoderAdapter::new(writer); 140 | 141 | if self.is_nullable() { 142 | write!(type_adapter, "Nullable(")?; 143 | }; 144 | 145 | write!(type_adapter, "{}", self.sql_type)?; 146 | 147 | if self.sql_type == SqlType::Enum8 || self.sql_type == SqlType::Enum16 { 148 | Field::write_enum( 149 | self.meta.as_ref().expect("enum index corrupted"), 150 | &mut type_adapter, 151 | )?; 152 | } 153 | 154 | if self.is_nullable() { 155 | write!(type_adapter, ")")?; 156 | }; 157 | Ok(()) 158 | } 159 | } 160 | 161 | #[derive(Copy, Clone, Debug, PartialEq)] 162 | pub enum SqlType { 163 | UInt8, 164 | UInt16, 165 | UInt32, 166 | UInt64, 167 | Int8, 168 | Int16, 169 | Int32, 170 | Int64, 171 | String, 172 | FixedString(u32), 173 | Float32, 174 | Float64, 175 | Date, 176 | DateTime, 177 | DateTime64(u8, Tz), 178 | Ipv4, 179 | Ipv6, 180 | Uuid, 181 | Decimal(u8, u8), 182 | Enum8, 183 | Enum16, 184 | // type placeholders. don't really instantiate this types 185 | Array, 186 | LowCardinality, 187 | } 188 | 189 | impl fmt::Display for SqlType { 190 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 191 | let name = match self { 192 | SqlType::String => "String", 193 | SqlType::UInt8 => "UInt8", 194 | SqlType::UInt16 => "UInt16", 195 | SqlType::UInt32 => "UInt32", 196 | SqlType::UInt64 => "UInt64", 197 | SqlType::Int8 => "Int8", 198 | SqlType::Int16 => "Int16", 199 | SqlType::Int32 => "Int32", 200 | SqlType::Int64 => "Int64", 201 | SqlType::Uuid => "UUID", 202 | SqlType::Ipv4 => "IPv4", 203 | SqlType::Ipv6 => "IPv6", 204 | SqlType::Float32 => "Float32", 205 | SqlType::Float64 => "Float64", 206 | SqlType::Date => "Date", 207 | SqlType::DateTime => "DateTime", 208 | SqlType::FixedString(p) => return f.write_fmt(format_args!("FixedString({})", p)), 209 | SqlType::DateTime64(p, _) => return f.write_fmt(format_args!("DateTime64({})", p)), 210 | SqlType::Decimal(p, s) => return f.write_fmt(format_args!("Decimal({},{})", p, s)), 211 | SqlType::Enum8 => "Enum8", 212 | SqlType::Enum16 => "Enum16", 213 | _ => unimplemented!(""), 214 | }; 215 | 216 | f.write_str(name) 217 | } 218 | } 219 | 220 | pub(crate) static SCALE: &[i64] = &[ 221 | 1, 222 | 10, 223 | 100, 224 | 1000, 225 | 10000, 226 | 100_000, 227 | 1_000_000, 228 | 10_000_000, 229 | 100_000_000, 230 | 1_000_000_000, 231 | 10_000_000_000, 232 | 100_000_000_000, 233 | 1_000_000_000_000, 234 | 10_000_000_000_000, 235 | 100_000_000_000_000, 236 | 1_000_000_000_000_000, 237 | 10_000_000_000_000_000, 238 | 100_000_000_000_000_000, 239 | 1_000_000_000_000_000_000, 240 | ]; 241 | 242 | #[cfg(test)] 243 | mod test { 244 | use super::Field; 245 | use crate::protocol::column::EnumIndex; 246 | use crate::protocol::encoder::Encoder; 247 | use crate::types::{FieldMeta, SqlType, FIELD_NONE, FIELD_NULLABLE}; 248 | 249 | macro_rules! into_boxed { 250 | ($s: expr) => { 251 | $s.to_owned().into_boxed_str().into_boxed_bytes() 252 | }; 253 | } 254 | macro_rules! into_string { 255 | ($v: ident ) => { 256 | String::from_utf8_lossy($v.as_slice()) 257 | }; 258 | } 259 | 260 | #[test] 261 | fn test_field_format_basic() { 262 | let mut buf = Vec::new(); 263 | let f = Field { 264 | sql_type: SqlType::String, 265 | flag: FIELD_NONE, 266 | depth: 0, 267 | meta: None, 268 | }; 269 | 270 | f.encode(&mut buf).unwrap(); 271 | assert_eq!(into_string!(buf), "\u{6}String") 272 | } 273 | #[test] 274 | fn test_field_format_nullable() { 275 | let mut buf = Vec::new(); 276 | let f = Field { 277 | sql_type: SqlType::String, 278 | flag: FIELD_NULLABLE, 279 | depth: 0, 280 | meta: None, 281 | }; 282 | f.encode(&mut buf).unwrap(); 283 | assert_eq!(into_string!(buf), "\u{10}Nullable(String)"); 284 | } 285 | 286 | #[test] 287 | fn test_field_format_param() { 288 | let mut buf = Vec::new(); 289 | let f = Field { 290 | sql_type: SqlType::FixedString(20), 291 | flag: FIELD_NULLABLE, 292 | depth: 0, 293 | meta: None, 294 | }; 295 | f.encode(&mut buf).unwrap(); 296 | assert_eq!(into_string!(buf), "\u{19}Nullable(FixedString(20))"); 297 | buf.clear(); 298 | let f = Field { 299 | sql_type: SqlType::Decimal(18, 4), 300 | flag: FIELD_NULLABLE, 301 | depth: 0, 302 | meta: None, 303 | }; 304 | f.encode(&mut buf).unwrap(); 305 | assert_eq!(into_string!(buf), "\u{17}Nullable(Decimal(18,4))"); 306 | } 307 | #[test] 308 | fn test_field_format_enum() { 309 | let mut buf = Vec::new(); 310 | let index = vec![ 311 | EnumIndex(0, into_boxed!("no")), 312 | EnumIndex(1, into_boxed!("yes")), 313 | EnumIndex(2, into_boxed!("n'a")), 314 | ]; 315 | let f = Field { 316 | sql_type: SqlType::Enum8, 317 | flag: FIELD_NONE, 318 | depth: 0, 319 | meta: Some(FieldMeta { index }), 320 | }; 321 | f.encode(&mut buf).unwrap(); 322 | assert_eq!(into_string!(buf), "$Enum8('no' = 0,'yes' = 1,'n\\'a' = 2)"); 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /clickhouse-driver/tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | use clickhouse_driver::prelude::{Options, Pool}; 3 | use std::convert::TryInto; 4 | use std::env; 5 | use url::Url; 6 | 7 | pub fn db_url() -> String { 8 | env::var("DATABASE_URL").unwrap_or_else(|_| { 9 | "tcp://localhost?execute_timeout=5s&query_timeout=20s&pool_max=4&compression=lz4".into() 10 | }) 11 | } 12 | 13 | pub fn get_config() -> Options { 14 | let database_url = db_url(); 15 | database_url.try_into().unwrap() 16 | } 17 | 18 | /// Replace connection parameters in 19 | /// the environment with specified in `url` argument 20 | /// Preserve host,port,database,user info 21 | pub fn get_pool_extend(url: &str) -> Pool { 22 | let mut config_url = Url::parse(db_url().as_str()).unwrap(); 23 | let url = Url::parse(url).unwrap(); 24 | 25 | config_url.set_query(url.query()); 26 | Pool::create(config_url).unwrap() 27 | } 28 | 29 | pub fn get_pool() -> Pool { 30 | let url = db_url(); 31 | Pool::create(url).unwrap() 32 | } 33 | -------------------------------------------------------------------------------- /clickhouse-driver/tests/pool.rs: -------------------------------------------------------------------------------- 1 | use clickhouse_driver::prelude::errors; 2 | use clickhouse_driver::prelude::*; 3 | use std::io; 4 | use std::time::Duration; 5 | use tokio::{self, time::delay_for}; 6 | 7 | mod common; 8 | use common::{get_config, get_pool, get_pool_extend}; 9 | 10 | // macro_rules! check { 11 | // ($f:expr) => { 12 | // match $f { 13 | // Ok(r) => r, 14 | // Err(err) => { 15 | // eprintln!("{:?}", err); 16 | // Default::default() 17 | // } 18 | // } 19 | // }; 20 | // } 21 | 22 | #[tokio::test] 23 | async fn test_connection_pool() -> io::Result<()> { 24 | let pool = get_pool(); 25 | 26 | let mut h: Vec<_> = (0..10) 27 | .map(|_| { 28 | let pool = pool.clone(); 29 | 30 | tokio::spawn(async move { 31 | let mut conn = pool.connection().await.unwrap(); 32 | conn.ping().await.expect("ping ok"); 33 | delay_for(Duration::new(2, 0)).await; 34 | }) 35 | }) 36 | .collect(); 37 | 38 | for (_, hnd) in h.iter_mut().enumerate() { 39 | hnd.await?; 40 | } 41 | 42 | Ok(()) 43 | } 44 | 45 | #[tokio::test] 46 | async fn test_conn_race() -> io::Result<()> { 47 | let pool = get_pool_extend("tcp://localhost/?pool_max=10&pool_min=10&ping_before_query=false"); 48 | let now = std::time::Instant::now(); 49 | const COUNT: u32 = 1000; 50 | const MSEC_100: u32 = 100000000; 51 | 52 | let mut h: Vec<_> = (0..COUNT) 53 | .map(|_| { 54 | let pool = pool.clone(); 55 | 56 | tokio::spawn(async move { 57 | let _ = pool.connection().await.unwrap(); 58 | delay_for(Duration::new(0, MSEC_100)).await; 59 | }) 60 | }) 61 | .collect(); 62 | 63 | for (_, hnd) in h.iter_mut().enumerate() { 64 | hnd.await?; 65 | } 66 | 67 | let elapsed = now.elapsed().as_secs(); 68 | println!("elapsed {}", elapsed); 69 | assert!(elapsed < (COUNT as u64 / 10 + 5)); 70 | 71 | Ok(()) 72 | } 73 | 74 | #[tokio::test] 75 | async fn test_ping() -> errors::Result<()> { 76 | let pool = get_pool(); 77 | let mut conn = pool.connection().await?; 78 | conn.ping().await?; 79 | 80 | let config = get_config().set_timeout(Duration::from_nanos(1)); 81 | 82 | let pool = Pool::create(config).unwrap(); 83 | let mut conn = pool.connection().await?; 84 | let err_timeout = conn.ping().await; 85 | 86 | // assert!(err_timeout.unwrap_err().is_timeout()); 87 | 88 | 89 | let config = get_config().set_timeout(Duration::from_nanos(1)); 90 | 91 | let pool = Pool::create(config).unwrap(); 92 | let mut conn = pool.connection().await?; 93 | let _err_timeout = conn.ping().await; 94 | 95 | // assert!(err_timeout.unwrap_err().is_timeout()); 96 | 97 | Ok(()) 98 | } 99 | 100 | #[tokio::test] 101 | async fn test_readonly_mode() -> errors::Result<()> { 102 | let pool = get_pool(); 103 | let mut conn = pool.connection().await?; 104 | conn.execute("DROP TABLE IF EXISTS test_readonly1").await?; 105 | conn.execute("DROP TABLE IF EXISTS test_readonly2").await?; 106 | let res = conn 107 | .execute("CREATE TABLE IF NOT EXISTS test_readonly0(id UInt64) ENGINE=Memory") 108 | .await; 109 | assert!(res.is_ok()); 110 | drop(conn); 111 | drop(pool); 112 | 113 | let pool = get_pool_extend("tcp://localhost/?readonly=2"); 114 | 115 | let mut conn = pool.connection().await?; 116 | let res = conn 117 | .execute("CREATE TABLE IF NOT EXISTS test_readonly1(id UInt64) ENGINE=Memory") 118 | .await; 119 | assert!(res.is_err()); 120 | //println!("{:?}",res.unwrap_err()); 121 | 122 | let pool = get_pool_extend("tcp://localhost/?readonly=1"); 123 | let mut conn = pool.connection().await?; 124 | let res = conn 125 | .execute("CREATE TABLE IF NOT EXISTS test_readonly2(id UInt64) ENGINE=Memory") 126 | .await; 127 | 128 | assert!(res.is_err()); 129 | 130 | let data1 = vec![0u64, 1, 3, 4, 5, 6]; 131 | let block = { Block::new("test_readonly").add("id", data1) }; 132 | let insert = conn.insert(&block).await; 133 | assert!(insert.is_err()); 134 | drop(insert); 135 | 136 | let query_result = conn.query("SELECT count(*) FROM main").await; 137 | assert!(query_result.is_ok()); 138 | let mut query_result = query_result.unwrap(); 139 | while let Some(_block) = query_result.next().await? {} 140 | drop(query_result); 141 | drop(conn); 142 | drop(pool); 143 | 144 | Ok(()) 145 | } 146 | -------------------------------------------------------------------------------- /clickhouse-driver/tests/query.rs: -------------------------------------------------------------------------------- 1 | use chrono::{DateTime, Utc}; 2 | use clickhouse_driver::prelude::errors; 3 | use clickhouse_driver::prelude::*; 4 | use std::net::{Ipv4Addr, Ipv6Addr}; 5 | use uuid::Uuid; 6 | mod common; 7 | use clickhouse_driver::prelude::types::Decimal32; 8 | use common::{get_config, get_pool}; 9 | 10 | macro_rules! get { 11 | ($row:ident, $idx: expr, $msg: expr) => { 12 | $row.value($idx)?.expect($msg) 13 | }; 14 | ($row:ident, $idx: expr) => { 15 | get!($row, $idx, "unexpected error") 16 | }; 17 | } 18 | 19 | type CHDT = DateTime; 20 | 21 | #[tokio::test] 22 | async fn test_query_ddl() -> errors::Result<()> { 23 | let pool = get_pool(); 24 | let mut conn = pool.connection().await?; 25 | conn.execute("DROP TABLE IF EXISTS rust2").await?; 26 | conn.execute("CREATE TABLE rust2(x Int64) ENGINE=Memory") 27 | .await?; 28 | Ok(()) 29 | } 30 | 31 | #[tokio::test] 32 | async fn test_query_compress() -> errors::Result<()> { 33 | let config = get_config(); 34 | 35 | let pool = Pool::create(config.set_compression(CompressionMethod::LZ4)).unwrap(); 36 | { 37 | let mut conn = pool.connection().await?; 38 | 39 | let mut qr = conn.query("SELECT lcs FROM main LIMIT 1000").await?; 40 | while let Some(_block) = qr.next().await? {} 41 | assert_eq!(qr.is_pending(), false); 42 | } 43 | 44 | drop(pool); 45 | let config = get_config(); 46 | 47 | let pool = Pool::create(config.set_compression(CompressionMethod::None)).unwrap(); 48 | let mut conn = pool.connection().await?; 49 | 50 | let mut qr = conn.query("SELECT lcs FROM main LIMIT 1000").await?; 51 | while let Some(block) = qr.next().await? { 52 | for row in block.iter_rows() { 53 | let _lcs: &str = row.value(0)?.unwrap(); 54 | //println!("{}",lcs); 55 | } 56 | } 57 | assert_eq!(qr.is_pending(), false); 58 | 59 | drop(pool); 60 | let pool = get_pool(); 61 | { 62 | let mut conn = pool.connection().await?; 63 | 64 | let mut qr = conn.query("SELECT lcs FROM main LIMIT 1000").await?; 65 | while let Some(block) = qr.next().await? { 66 | for row in block.iter_rows() { 67 | let _lcs: &str = row.value(0)?.unwrap(); 68 | //println!("{}", lcs); 69 | } 70 | } 71 | assert_eq!(qr.is_pending(), false); 72 | } 73 | Ok(()) 74 | } 75 | 76 | #[tokio::test] 77 | async fn test_query_pending() -> errors::Result<()> { 78 | let pool = get_pool(); 79 | let mut conn = pool.connection().await?; 80 | 81 | let mut query_result = conn.query("SELECT i64 FROM main").await?; 82 | 83 | let mut i: u32 = 0; 84 | while let Some(_block) = query_result.next().await? { 85 | i += 1; 86 | if i == 1 { 87 | assert_eq!(query_result.is_pending(), true); 88 | } 89 | } 90 | 91 | assert_eq!(query_result.is_pending(), false); 92 | drop(query_result); 93 | Ok(()) 94 | } 95 | 96 | #[tokio::test] 97 | async fn test_query_string() -> errors::Result<()> { 98 | let pool = get_pool(); 99 | let mut conn = pool.connection().await?; 100 | 101 | let mut query_result = conn.query("SELECT title FROM main").await?; 102 | 103 | while let Some(block) = query_result.next().await? { 104 | for (j, row) in block.iter_rows().enumerate() { 105 | let s: &str = get!(row, 0); 106 | println!("{:4}:{}", j, s); 107 | } 108 | } 109 | 110 | Ok(()) 111 | } 112 | 113 | #[tokio::test] 114 | async fn test_query_enum() -> errors::Result<()> { 115 | let pool = get_pool(); 116 | let mut conn = pool.connection().await?; 117 | 118 | let mut query_result = conn.query("SELECT e8,e16 FROM main").await?; 119 | 120 | while let Some(block) = query_result.next().await? { 121 | for row in block.iter_rows() { 122 | let e8: &str = get!(row, 0); 123 | let e16: &str = get!(row, 1); 124 | println!("'{}'='{}'", e8, e16); 125 | } 126 | } 127 | 128 | Ok(()) 129 | } 130 | 131 | #[tokio::test] 132 | async fn test_query_nullable() -> errors::Result<()> { 133 | let pool = get_pool(); 134 | let mut conn = pool.connection().await?; 135 | 136 | let mut query_result = conn.query("SELECT n FROM main WHERE n=NULL").await?; 137 | 138 | while let Some(block) = query_result.next().await? { 139 | for row in block.iter_rows() { 140 | let n: Option = row.value(0)?; 141 | assert!(n.is_none()); 142 | } 143 | } 144 | drop(query_result); 145 | 146 | let mut query_result = conn.query("SELECT n FROM main WHERE n=1").await?; 147 | 148 | while let Some(block) = query_result.next().await? { 149 | for row in block.iter_rows() { 150 | let n: Option = row.value(0)?; 151 | assert!(n.is_some()); 152 | assert_eq!(n.unwrap(), 1u16); 153 | } 154 | } 155 | Ok(()) 156 | } 157 | 158 | #[tokio::test] 159 | async fn test_query_lowcardinality() -> errors::Result<()> { 160 | let pool = get_pool(); 161 | let mut conn = pool.connection().await?; 162 | 163 | let mut query_result = conn 164 | .query("SELECT lcs FROM mainx WHERE lcs='May' LIMIT 1000") 165 | .await?; 166 | 167 | while let Some(block) = query_result.next().await? { 168 | for row in block.iter_rows() { 169 | let lcs: &str = row.value(0)?.unwrap(); 170 | assert_eq!(lcs, "May"); 171 | } 172 | } 173 | drop(query_result); 174 | let mut query_result = conn 175 | .query("SELECT lcs FROM mainx WHERE lcs IS NULL LIMIT 1000") 176 | .await?; 177 | 178 | while let Some(block) = query_result.next().await? { 179 | for row in block.iter_rows() { 180 | let lcs: Option<&str> = row.value(0)?; 181 | assert!(lcs.is_none()); 182 | } 183 | } 184 | 185 | Ok(()) 186 | } 187 | 188 | #[tokio::test] 189 | async fn test_query_array() -> errors::Result<()> { 190 | let pool = get_pool(); 191 | let mut conn = pool.connection().await?; 192 | 193 | let mut query_result = conn 194 | .query("SELECT a8,a16,a32,a64,ad,adt,adc,aip4,aip6 FROM mainx LIMIT 100 ") 195 | .await?; 196 | 197 | while let Some(block) = query_result.next().await? { 198 | for row in block.iter_rows() { 199 | let a8: &[u8] = get!(row, 0); 200 | let a16: &[u16] = get!(row, 1); 201 | let a32: &[u32] = get!(row, 2); 202 | let a64: &[u64] = get!(row, 3); 203 | let ad: Vec> = get!(row, 4); 204 | let adt: Vec> = get!(row, 5); 205 | let adc: Vec = get!(row, 6); 206 | let aip4: Vec = get!(row, 7); 207 | let aip6: Vec = get!(row, 8); 208 | 209 | println!( 210 | "{:?} {:?} {:?} {:?} {:?} {:?} {:?} {:?} {:?}", 211 | a8, a16, a32, a64, ad, adt, adc, aip4, aip6 212 | ); 213 | } 214 | } 215 | Ok(()) 216 | } 217 | 218 | #[tokio::test] 219 | async fn test_query_deserialize() -> errors::Result<()> { 220 | let pool = get_pool(); 221 | let mut conn = pool.connection().await?; 222 | 223 | #[derive(Debug)] 224 | struct RowObject { 225 | uuid: Uuid, 226 | title: String, 227 | dt: CHDT, 228 | ip: Ipv4Addr, 229 | } 230 | macro_rules! field { 231 | ($row:ident, $idx: expr, $err: expr) => { 232 | $row.value($idx)?.ok_or_else($err) 233 | }; 234 | ($row:ident, $idx: expr) => { 235 | field!($row, $idx, || { 236 | errors::ConversionError::UnsupportedConversion 237 | }) 238 | }; 239 | } 240 | 241 | impl Deserialize for RowObject { 242 | fn deserialize(row: Row) -> errors::Result { 243 | let err = || errors::ConversionError::UnsupportedConversion; 244 | 245 | let uuid: Uuid = field!(row, 0, err)?; 246 | let dt: CHDT = field!(row, 1, err)?; 247 | let title: &str = field!(row, 2, err)?; 248 | let ip: Ipv4Addr = field!(row, 3, err)?; 249 | 250 | Ok(RowObject { 251 | uuid, 252 | dt, 253 | title: title.to_string(), 254 | ip, 255 | }) 256 | } 257 | } 258 | 259 | let mut query_result = conn 260 | .query("SELECT uuid, t, title, ip4 FROM main LIMIT 10 ") 261 | .await?; 262 | 263 | while let Some(block) = query_result.next().await? { 264 | for row in block.iter::() { 265 | println!("{:?}", row); 266 | } 267 | } 268 | 269 | Ok(()) 270 | } 271 | -------------------------------------------------------------------------------- /derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-driver-derive" 3 | version = "0.1.1" 4 | description = "Derive macros for ClickHouse Asynchronous Driver." 5 | authors = ["Dmitry Dulesov "] 6 | edition = "2018" 7 | 8 | [dependencies] 9 | syn = "1.0.31" 10 | quote = "1.0.7" 11 | 12 | [lib] 13 | proc-macro = true 14 | -------------------------------------------------------------------------------- /derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate proc_macro; 2 | #[macro_use] 3 | extern crate quote; 4 | extern crate syn; 5 | 6 | use proc_macro::TokenStream; 7 | 8 | use syn::export::TokenStream2; 9 | use syn::{parse_macro_input, DeriveInput}; 10 | 11 | #[proc_macro_derive(IsCommand)] 12 | pub fn command_derive(input: TokenStream) -> TokenStream { 13 | let ast = parse_macro_input!(input as DeriveInput); 14 | let gen = command_impl(&ast); 15 | gen.into() 16 | } 17 | 18 | fn command_impl(ast: &syn::DeriveInput) -> TokenStream2 { 19 | let gen = &ast.generics; 20 | let name = &ast.ident; 21 | quote! { 22 | impl#gen Command for #name#gen { 23 | 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | clickhouse: 4 | image: yandex/clickhouse-server 5 | ports: 6 | - 8123:8123 7 | - 9000:9000 -------------------------------------------------------------------------------- /lz4a/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-driver-lz4" 3 | description = "LZ4 for ClickHouse Asynchronous Driver." 4 | version = "0.1.0" 5 | authors = ["Dmitry Dulesov "] 6 | edition = "2018" 7 | build = "build.rs" 8 | links = "clickhouse-driver" 9 | license = "MIT" 10 | repository = "https://github.com/ddulesov/clickhouse_driver" 11 | include = ["liblz4/lib/**","benches/**","src/**","build.rs"] 12 | 13 | [dependencies] 14 | libc = "0.2" 15 | 16 | [build-dependencies] 17 | cc = "1.0.54" 18 | 19 | [dev-dependencies] 20 | rand = "0.7.3" 21 | -------------------------------------------------------------------------------- /lz4a/benches/bench.rs: -------------------------------------------------------------------------------- 1 | #![cfg(rustc_nightly)] 2 | #![feature(test)] 3 | #![feature(core_intrinsics)] 4 | 5 | extern crate libc; 6 | extern crate test; 7 | 8 | use test::Bencher; 9 | 10 | use libc::c_char; 11 | use rand::prelude::*; 12 | 13 | //use core::intrinsics::size_of; 14 | use lz4a::*; 15 | 16 | fn data(len: usize) -> Vec { 17 | let mut b: Vec = Vec::with_capacity(len as usize); 18 | unsafe { 19 | b.set_len(len as usize); 20 | } 21 | 22 | b 23 | } 24 | 25 | #[bench] 26 | fn bench_compress_static_1m(b: &mut Bencher) { 27 | let mut input = data(1024 * 1024); 28 | rand::thread_rng().fill(&mut input[..]); 29 | 30 | let cz: usize = LZ4_CompressInplaceBufferSize(1024 * 1024); 31 | let mut output: Vec = Vec::with_capacity(cz); 32 | unsafe { 33 | output.set_len(cz as usize); 34 | } 35 | let mut code: i32 = 0; 36 | 37 | b.iter(|| { 38 | code = unsafe { 39 | LZ4_compress_default( 40 | input.as_ptr() as *const c_char, 41 | output.as_mut_ptr() as *mut c_char, 42 | (input.len()) as i32, 43 | output.len() as i32, 44 | ) 45 | }; 46 | }); 47 | } 48 | 49 | #[bench] 50 | fn bench_decompress_static_1m(b: &mut Bencher) { 51 | let mut input = data(1024 * 1024); 52 | rand::thread_rng().fill(&mut input[..]); 53 | 54 | let cz: usize = LZ4_CompressInplaceBufferSize(1024 * 1024); 55 | let mut output: Vec = Vec::with_capacity(cz); 56 | 57 | unsafe { 58 | output.set_len(cz as usize); 59 | } 60 | 61 | let mut code = unsafe { 62 | LZ4_compress_default( 63 | input.as_ptr() as *const c_char, 64 | output.as_mut_ptr() as *mut c_char, 65 | (input.len()) as i32, 66 | output.len() as i32, 67 | ) 68 | }; 69 | 70 | output.truncate(code as usize); 71 | 72 | let compressed_len = code; 73 | //drop( input ); 74 | 75 | b.iter(|| unsafe { 76 | code = LZ4_decompress_safe( 77 | output.as_ptr() as *const c_char, 78 | input.as_mut_ptr() as *mut c_char, 79 | compressed_len as i32, 80 | input.len() as i32, 81 | ); 82 | }); 83 | } 84 | 85 | #[bench] 86 | fn bench_compress_highratio_1m(b: &mut Bencher) { 87 | let input = data(1024 * 1024); 88 | 89 | let cz: usize = LZ4_CompressInplaceBufferSize(1024 * 1024); 90 | let mut output: Vec = Vec::with_capacity(cz); 91 | unsafe { 92 | output.set_len(cz as usize); 93 | } 94 | let mut code: i32 = 0; 95 | 96 | b.iter(|| { 97 | code = unsafe { 98 | LZ4_compress_default( 99 | input.as_ptr() as *const c_char, 100 | output.as_mut_ptr() as *mut c_char, 101 | (input.len()) as i32, 102 | output.len() as i32, 103 | ) 104 | }; 105 | }); 106 | } 107 | 108 | #[bench] 109 | fn bench_decompress_highratio_1m(b: &mut Bencher) { 110 | let mut input = data(1024 * 1024); 111 | 112 | let cz: usize = LZ4_CompressInplaceBufferSize(1024 * 1024); 113 | let mut output: Vec = Vec::with_capacity(cz); 114 | 115 | unsafe { 116 | output.set_len(cz as usize); 117 | } 118 | 119 | let mut code = unsafe { 120 | LZ4_compress_default( 121 | input.as_ptr() as *const c_char, 122 | output.as_mut_ptr() as *mut c_char, 123 | (input.len()) as i32, 124 | output.len() as i32, 125 | ) 126 | }; 127 | 128 | output.truncate(code as usize); 129 | 130 | let compressed_len = code; 131 | //drop( input ); 132 | 133 | b.iter(|| unsafe { 134 | code = LZ4_decompress_safe( 135 | output.as_ptr() as *const c_char, 136 | input.as_mut_ptr() as *mut c_char, 137 | compressed_len as i32, 138 | input.len() as i32, 139 | ); 140 | }); 141 | } 142 | -------------------------------------------------------------------------------- /lz4a/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cc; 2 | 3 | use std::error::Error; 4 | use std::path::PathBuf; 5 | use std::{env, fs, process}; 6 | 7 | fn main() { 8 | match run() { 9 | Ok(()) => (), 10 | Err(err) => { 11 | eprintln!("{}", err); 12 | process::exit(1); 13 | } 14 | } 15 | } 16 | #[allow(clippy::single_match)] 17 | fn run() -> Result<(), Box> { 18 | let mut compiler = cc::Build::new(); 19 | compiler 20 | .file("liblz4/lib/lz4.c") 21 | .file("liblz4/lib/lz4hc.c") 22 | //.file("liblz4/lib/lz4frame.c") 23 | //.file("liblz4/lib/xxhash.c") 24 | // We always compile the C with optimization, because otherwise it is 20x slower. 25 | .opt_level(3); 26 | match env::var("TARGET") 27 | .map_err(|err| format!("reading TARGET environment variable: {}", err))? 28 | .as_str() 29 | { 30 | "i686-pc-windows-gnu" => { 31 | compiler.flag("-fno-tree-vectorize"); 32 | } 33 | _ => {} 34 | } 35 | compiler.compile("liblz4.a"); 36 | 37 | let src = env::current_dir()?.join("liblz4").join("lib"); 38 | let dst = PathBuf::from(env::var_os("OUT_DIR").ok_or("missing OUT_DIR environment variable")?); 39 | let include = dst.join("include"); 40 | fs::create_dir_all(&include) 41 | .map_err(|err| format!("creating directory {}: {}", include.display(), err))?; 42 | for e in fs::read_dir(&src)? { 43 | let e = e?; 44 | let utf8_file_name = e 45 | .file_name() 46 | .into_string() 47 | .map_err(|_| format!("unable to convert file name {:?} to UTF-8", e.file_name()))?; 48 | if utf8_file_name.ends_with(".h") { 49 | let from = e.path(); 50 | let to = include.join(e.file_name()); 51 | fs::copy(&from, &to).map_err(|err| { 52 | format!("copying {} to {}: {}", from.display(), to.display(), err) 53 | })?; 54 | } 55 | } 56 | println!("cargo:root={}", dst.display()); 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /lz4a/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | extern crate libc; 3 | #[cfg(test)] 4 | #[macro_use] 5 | extern crate std; 6 | 7 | use libc::{c_char, c_int, c_uint, size_t}; 8 | 9 | pub type LZ4FErrorCode = size_t; 10 | pub const LZ4F_VERSION: c_uint = 100; 11 | 12 | extern "C" { 13 | // int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); 14 | #[allow(non_snake_case)] 15 | pub fn LZ4_compress_default( 16 | source: *const c_char, 17 | dest: *mut c_char, 18 | sourceSize: c_int, 19 | maxDestSize: c_int, 20 | ) -> c_int; 21 | 22 | // int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); 23 | #[allow(non_snake_case)] 24 | pub fn LZ4_compress_fast( 25 | source: *const c_char, 26 | dest: *mut c_char, 27 | sourceSize: c_int, 28 | maxDestSize: c_int, 29 | acceleration: c_int, 30 | ) -> c_int; 31 | 32 | // int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); 33 | #[allow(non_snake_case)] 34 | pub fn LZ4_compress_HC( 35 | src: *const c_char, 36 | dst: *mut c_char, 37 | srcSize: c_int, 38 | dstCapacity: c_int, 39 | compressionLevel: c_int, 40 | ) -> c_int; 41 | 42 | // int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); 43 | #[allow(non_snake_case)] 44 | pub fn LZ4_decompress_safe( 45 | source: *const c_char, 46 | dest: *mut c_char, 47 | compressedSize: c_int, 48 | maxDecompressedSize: c_int, 49 | ) -> c_int; 50 | 51 | #[allow(non_snake_case)] 52 | pub fn LZ4_decompress_fast( 53 | source: *const c_char, 54 | dest: *mut c_char, 55 | originaldSize: c_int, 56 | ) -> c_int; 57 | 58 | // const char* LZ4F_getErrorName(LZ4F_errorCode_t code); 59 | pub fn LZ4F_getErrorName(code: size_t) -> *const c_char; 60 | 61 | // int LZ4_versionNumber(void) 62 | pub fn LZ4_versionNumber() -> c_int; 63 | 64 | // int LZ4_compressBound(int isize) 65 | fn LZ4_compressBound(size: c_int) -> c_int; 66 | 67 | } 68 | 69 | const LZ4_DISTANCE_MAX: usize = 65535; 70 | 71 | #[allow(non_snake_case)] 72 | #[inline] 73 | pub const fn LZ4_CompressInplaceBufferSize(decompressed: usize) -> usize { 74 | decompressed + LZ4_DISTANCE_MAX + 32 75 | } 76 | 77 | #[allow(non_snake_case)] 78 | #[inline] 79 | pub const fn LZ4_DecompressInplaceBufferSize(compressed: usize) -> usize { 80 | compressed + (compressed >> 8) + 32 81 | } 82 | 83 | #[allow(non_snake_case)] 84 | #[inline] 85 | pub fn LZ4_Decompress(src: &[u8], dst: &mut [u8]) -> i32 { 86 | unsafe { 87 | LZ4_decompress_safe( 88 | src.as_ptr() as *const c_char, 89 | dst.as_mut_ptr() as *mut c_char, 90 | src.len() as c_int, 91 | dst.len() as c_int, 92 | ) 93 | } 94 | } 95 | 96 | #[allow(non_snake_case)] 97 | #[inline] 98 | pub fn LZ4_Compress(src: &[u8], dst: &mut [u8]) -> i32 { 99 | unsafe { 100 | LZ4_compress_default( 101 | src.as_ptr() as *const c_char, 102 | dst.as_mut_ptr() as *mut c_char, 103 | src.len() as c_int, 104 | dst.len() as c_int, 105 | ) 106 | } 107 | } 108 | 109 | #[allow(non_snake_case)] 110 | #[inline] 111 | pub fn LZ4_CompressBounds(src: usize) -> usize { 112 | unsafe { LZ4_compressBound(src as c_int) as usize } 113 | } 114 | 115 | #[cfg(test)] 116 | mod test { 117 | extern crate rand; 118 | use self::rand::RngCore; 119 | use crate::*; 120 | use libc::c_int; 121 | 122 | #[test] 123 | fn test_version_number() { 124 | let version = unsafe { LZ4_versionNumber() }; 125 | assert_eq!(version, 10902 as c_int); 126 | 127 | // 640 kb original size 128 | assert_eq!(unsafe { LZ4_compressBound(640 * 1024) }, 657946); 129 | 130 | // 1Mb destination bufer 131 | assert_eq!(LZ4_CompressInplaceBufferSize(983009), 1024 * 1024); 132 | assert_eq!(LZ4_DecompressInplaceBufferSize(1044464), 1024 * 1024 - 1); 133 | } 134 | 135 | #[test] 136 | fn test_compression() { 137 | use std::vec::Vec; 138 | let mut rng = rand::thread_rng(); 139 | 140 | for sz in [600_usize, 1024, 6000, 65000, 650000].iter() { 141 | let cz: usize = LZ4_CompressInplaceBufferSize(*sz); 142 | 143 | let mut orig: Vec = Vec::with_capacity(cz); 144 | unsafe { 145 | orig.set_len(cz); 146 | rng.fill_bytes(&mut orig[..]); 147 | 148 | let margin = cz - *sz; 149 | //compress inplace 150 | //maximum compressed size 151 | let bz = LZ4_compressBound(*sz as c_int); 152 | //destination compression bufer 153 | let mut comp: Vec = Vec::with_capacity(bz as usize); 154 | 155 | comp.set_len(bz as usize); 156 | 157 | //normal compression 158 | let code = LZ4_compress_default( 159 | orig.as_ptr().add(margin) as *const c_char, 160 | comp.as_mut_ptr() as *mut c_char, 161 | (orig.len() - margin) as i32, 162 | comp.len() as i32, 163 | ); 164 | 165 | assert!(code >= 0); 166 | assert_eq!(orig.len() - margin, *sz); 167 | let compressed_sz = code as usize; 168 | 169 | //compression inplace 170 | let code = LZ4_compress_default( 171 | orig.as_ptr().add(margin) as *const c_char, 172 | orig.as_mut_ptr() as *mut c_char, 173 | (orig.len() - margin) as i32, 174 | orig.len() as i32, 175 | ); 176 | 177 | assert!(code >= 0); 178 | 179 | assert_eq!(&comp[0..compressed_sz], &orig[0..compressed_sz]); 180 | } 181 | } 182 | 183 | assert_eq!(1, 1); 184 | } 185 | 186 | #[test] 187 | fn test_decompression() { 188 | use std::vec::Vec; 189 | //let mut rng = rand::thread_rng(); 190 | 191 | for sz in [600_usize, 1024, 6000, 65000, 650000].iter() { 192 | let mut orig: Vec = Vec::with_capacity(*sz); 193 | unsafe { 194 | orig.set_len(*sz); 195 | 196 | { 197 | //it's sort of randomized data 198 | orig[0] = 1; 199 | orig[*sz / 4] = 4; 200 | orig[*sz / 2] = 7; 201 | orig[*sz * 2 / 3] = 10; 202 | orig[*sz - 1] = 1; 203 | } 204 | 205 | let bz = LZ4_compressBound(*sz as c_int) as usize; 206 | 207 | let mut comp: Vec = Vec::with_capacity(bz); 208 | comp.set_len(bz); 209 | 210 | let code = LZ4_compress_default( 211 | orig.as_ptr() as *const c_char, 212 | comp.as_mut_ptr() as *mut c_char, 213 | (orig.len()) as i32, 214 | (bz) as i32, 215 | ); 216 | 217 | assert!(code > 0); 218 | //size of compressed data 219 | println!( 220 | "orig {}; compressed {}; in buf len {}", 221 | *sz, 222 | code as usize, 223 | comp.len() 224 | ); 225 | //compressed size 226 | let cz = code as usize; 227 | 228 | let mut buf: Vec = Vec::with_capacity(*sz); 229 | buf.set_len(*sz); 230 | 231 | let code = LZ4_decompress_safe( 232 | comp.as_ptr() as *const c_char, 233 | buf.as_mut_ptr() as *mut c_char, 234 | cz as i32, 235 | *sz as i32, 236 | ); 237 | 238 | assert!(code > 0); 239 | 240 | let cz = code as usize; 241 | 242 | assert_eq!(cz, *sz); 243 | assert_eq!(&orig[0..*sz], &buf[0..cz]); 244 | } 245 | } 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /tests/bench/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | .pyc -------------------------------------------------------------------------------- /tests/bench/BENCHMARK.md: -------------------------------------------------------------------------------- 1 | ## benchmark 2 | 3 | * rustc 1.45.0 (5c1f21c3b 2020-07-13) 4 | * go-1.14 5 | * python-3.8.1 6 | * java openjdk 14.0.1 2020-04-14 7 | 8 | * platform ubuntu 20.04 LTS, x86-64, EC2/t2-small(2Gb) 9 | 10 | ## client libraries 11 | - java (clickhouse-native-jdbc ) | 2.1-stable 12 | - go (https://github.com/ClickHouse/clickhouse-go) | 1.4.1 13 | - python (https://github.com/mymarilyn/clickhouse-driver) | 0.1.4 14 | - clickhouse_rs | 0.2.0-alpha.5 15 | - clickhouse-driver(3468745c3d8a2fbf5182a8d844f1d122f90a6d20)| 0.1.0-alpha.2 (*DRIVER*) 16 | 17 | 18 | java -Xms256m -Xmx1024m -jar ./clickhouse_driver-1.0-SNAPSHOT.jar -- select 19 | 20 | ## result of SELECT benchmark, SELECT 100000 rows (id UInt64, name String, dt DateTime) 21 | milliseconds 22 | 23 | | python | java-native | go | clickhouse-rs | *DRIVER* | 24 | | --------:|------------:| --------:| -------------:|----------:| 25 | | OMK | 2900 | 9234 | 3776 | 2833 | 26 | | Killed | 2900 | 9119 | 3603 | 2821 | 27 | | | 2888 | 9032 | 3681 | 2847 | 28 | | | 2921 | 9658 | 3666 | 2867 | 29 | | | **2902** | **9260** | **3681** |**2842** | 30 | 31 | ## result of INSERT benchmark, 1000 blocks x 10000 rows of (id UInt64, name String, dt DateTime) 32 | milliseconds 33 | 34 | | python | java-native | go | clickhouse-rs | *DRIVER* | 35 | | --------:|------------:|----------:| -------------:|----------:| 36 | | 98263 | 32677 | 42395 | 32939 | 3808 | 37 | | 95706 | 32489 | 42161 | 33911 | 3584 | 38 | | 95973 | 35106 | 41783 | 33649 | 3915 | 39 | | 94287 | 35745 | 41544 | 33631 | 3750 | 40 | |**96057** | **34004** | **41970** | **33532** |**3764** | 41 | -------------------------------------------------------------------------------- /tests/bench/clickhouse-rs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-rs-bench" 3 | version = "0.1.0" 4 | authors = ["dmitry dulesov "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | clickhouse-rs={ version = "0.2.0-alpha.5", features=["tokio_io"] } 9 | tokio={version="0.2", features = ["full"]} 10 | chrono = { version="^0.4" } 11 | chrono-tz={ version = "0.5" } -------------------------------------------------------------------------------- /tests/bench/clickhouse-rs/src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate tokio; 2 | extern crate chrono; 3 | extern crate chrono_tz; 4 | 5 | extern crate clickhouse_rs; 6 | use clickhouse_rs::{Block, Pool, ClientHandle}; 7 | use std::io; 8 | use std::env; 9 | use std::time; 10 | use chrono::prelude::*; 11 | use chrono_tz::Tz; 12 | use chrono_tz::Tz::UTC; 13 | 14 | static NAMES: [&str; 5] = ["one", "two", "three", "four", "five"]; 15 | const BSIZE: u64 = 10000; 16 | const CIRCLE: u64 = 1000; 17 | 18 | /// Add some entropy to mess up LZ4 compression 19 | 20 | fn next_block(i: u64) -> Block { 21 | let now = UTC.ymd(2016, 10, 22).and_hms(12, 0, 0); 22 | //let now = chrono::offset::Utc::now(); 23 | //let now = SystemTime::now(); 24 | //let now: u64 = now.duration_since(time::UNIX_EPOCH).unwrap().as_secs(); 25 | 26 | let dt: Vec<_> = (0..BSIZE).map(|idx| now + chrono::Duration::seconds(idx as i64) ).collect(); 27 | 28 | let name: Vec<&str> = (0..BSIZE) 29 | .map(|idx| NAMES[idx as usize % NAMES.len()]) 30 | .collect(); 31 | 32 | Block::new() 33 | .column("id", vec![i as u32; BSIZE as usize]) 34 | .column("name", name) 35 | .column("dt", dt) 36 | } 37 | 38 | async fn insert_perf(mut client: ClientHandle) -> io::Result<()>{ 39 | let ddl = r" 40 | CREATE TABLE IF NOT EXISTS perf_rust1 ( 41 | id UInt32, 42 | name String, 43 | dt DateTime 44 | ) Engine=MergeTree PARTITION BY name ORDER BY dt"; 45 | 46 | client.execute("DROP TABLE IF EXISTS perf_rust1").await?; 47 | 48 | client.execute(ddl).await?; 49 | let start = time::Instant::now(); 50 | for i in 0u64..CIRCLE { 51 | let block = next_block(i); 52 | client.insert("perf_rust1", block).await?; 53 | } 54 | 55 | eprintln!("elapsed {} msec", start.elapsed().as_millis()); 56 | eprintln!("{} rows have been inserted", BSIZE * CIRCLE); 57 | Ok(()) 58 | } 59 | 60 | async fn select_perf(mut client: ClientHandle) -> io::Result<()> { 61 | let start = time::Instant::now(); 62 | let block = client.query("SELECT id,name,dt FROM perf").fetch_all().await?; 63 | let mut sum: u64 = 0; 64 | 65 | for row in block.rows() { 66 | let id: u32 = row.get("id")?; 67 | let _name: String = row.get("name")?; 68 | let _dt: DateTime = row.get("dt")?; 69 | 70 | sum += id as u64; 71 | } 72 | eprintln!("elapsed {} msec", start.elapsed().as_millis()); 73 | eprintln!("{} ", sum); 74 | 75 | 76 | Ok(()) 77 | } 78 | 79 | #[tokio::main] 80 | async fn main() -> io::Result<()> { 81 | let mut args = env::args(); 82 | args.next(); 83 | let database_url = std::env::var("DATABASE_URL") 84 | .unwrap_or_else(|_| "tcp://localhost:9000?compression=lz4".into()); 85 | 86 | println!("connect to {}", &database_url); 87 | let pool = Pool::new(database_url); 88 | let client = pool.get_handle().await?; 89 | 90 | if let Some(meth) = args.next() { 91 | match meth.as_str() { 92 | "insert" => insert_perf(client).await, 93 | "select" => select_perf(client).await, 94 | _ => panic!("provide perf method 'insert' or 'select'") 95 | } 96 | 97 | }else{ 98 | panic!("provide perf method 'insert' or 'select'") 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /tests/bench/go/go.mod: -------------------------------------------------------------------------------- 1 | module clickhouse_main 2 | 3 | go 1.13 4 | 5 | require github.com/ClickHouse/clickhouse-go v1.4.1 6 | -------------------------------------------------------------------------------- /tests/bench/go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/ClickHouse/clickhouse-go v1.4.1 h1:D9cihLg76O1ZyILLaXq1eksYzEuV010NdvucgKGGK14= 2 | github.com/ClickHouse/clickhouse-go v1.4.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= 3 | github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= 4 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 7 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 8 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 9 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 10 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 14 | -------------------------------------------------------------------------------- /tests/bench/go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/ClickHouse/clickhouse-go" 7 | "log" 8 | "net/url" 9 | "os" 10 | "time" 11 | ) 12 | 13 | func perf_insert(connect *sql.DB) { 14 | _, err := connect.Exec(`DROP TABLE IF EXISTS perf_go`) 15 | if err != nil { 16 | log.Fatal(err) 17 | } 18 | 19 | _, err = connect.Exec(` 20 | CREATE TABLE IF NOT EXISTS perf_go ( 21 | id UInt32, 22 | name String, 23 | dt DateTime 24 | ) Engine=MergeTree PARTITION BY name ORDER BY dt 25 | `) 26 | 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | NAMES := [5]string{"one", "two", "three", "four", "five"} 32 | start := time.Now() 33 | 34 | for l := 0; l < 1000; l++ { 35 | tx, _ := connect.Begin() 36 | stmt, err := tx.Prepare("INSERT INTO perf_go (id, name, dt) VALUES (?, ?, ?)") 37 | 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | now := time.Now() 43 | 44 | for i := 0; i < 10000; i++ { 45 | if _, err := stmt.Exec( 46 | l, 47 | 48 | NAMES[i%len(NAMES)], 49 | now.Add(time.Duration(i)*time.Second), 50 | ); err != nil { 51 | log.Fatal(err) 52 | } 53 | } 54 | 55 | if err := tx.Commit(); err != nil { 56 | log.Fatal(err) 57 | } 58 | 59 | stmt.Close() 60 | } 61 | elapsed := time.Now().Sub(start) 62 | fmt.Printf("elapsed %v msec\n", elapsed.Milliseconds()) 63 | } 64 | 65 | func perf_select(connect *sql.DB) { 66 | start := time.Now() 67 | 68 | rows, err := connect.Query("SELECT id,name,dt FROM perf") 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | defer rows.Close() 73 | var sum uint64 = 0 74 | 75 | for rows.Next() { 76 | var ( 77 | name string 78 | id uint32 79 | dt time.Time 80 | ) 81 | if err := rows.Scan(&id, &name, &dt); err != nil { 82 | log.Fatal(err) 83 | } 84 | sum += uint64(id) 85 | } 86 | 87 | if err := rows.Err(); err != nil { 88 | log.Fatal(err) 89 | } 90 | 91 | elapsed := time.Now().Sub(start) 92 | fmt.Printf("elapsed %v msec\n", elapsed.Milliseconds()) 93 | fmt.Printf(" sum=%d", sum) 94 | } 95 | 96 | func main() { 97 | 98 | db_url := os.Getenv("DATABASE_URL") 99 | 100 | if db_url != "" { 101 | u, err := url.Parse(db_url) 102 | if err != nil { 103 | log.Fatal(err) 104 | } 105 | q := u.Query() 106 | if q.Get("compression") != "" { 107 | q.Del("compression") 108 | q.Set("compress", "true") 109 | } 110 | 111 | if v, ok := u.User.Password(); ok { 112 | q.Set("password", v) 113 | } 114 | 115 | v := u.User.Username() 116 | q.Set("username", v) 117 | 118 | u.RawQuery = q.Encode() 119 | 120 | db_url = u.String() 121 | } else { 122 | db_url = "tcp://default@localhost/default?compress=true" 123 | } 124 | 125 | fmt.Printf("url = %v\n", db_url) 126 | 127 | connect, err := sql.Open("clickhouse", db_url) 128 | if err != nil { 129 | log.Fatal(err) 130 | } 131 | if err := connect.Ping(); err != nil { 132 | if exception, ok := err.(*clickhouse.Exception); ok { 133 | fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace) 134 | } else { 135 | fmt.Println(err) 136 | } 137 | return 138 | } 139 | 140 | //connection established 141 | meth := os.Args[1] 142 | if meth == "insert" { 143 | perf_insert(connect) 144 | } else if meth == "select" { 145 | perf_select(connect) 146 | } else { 147 | fmt.Printf("method not provided") 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /tests/bench/java/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.github.ddulesov 5 | clickhouse_driver 6 | jar 7 | 1.0-SNAPSHOT 8 | clickhouse_driver 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 4.13.1 15 | test 16 | 17 | 18 | com.github.housepower 19 | clickhouse-native-jdbc 20 | 2.1-stable 21 | 22 | 23 | 24 | 25 | 26 | org.apache.maven.plugins 27 | maven-assembly-plugin 28 | 29 | 30 | package 31 | 32 | single 33 | 34 | 35 | 36 | 37 | 38 | com.github.ddulesov.App 39 | 40 | 41 | 42 | 43 | jar-with-dependencies 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /tests/bench/java/src/main/java/com/github/ddulesov/App.java: -------------------------------------------------------------------------------- 1 | package com.github.ddulesov; 2 | 3 | import java.net.URI; 4 | import java.net.URISyntaxException; 5 | import java.sql.*; 6 | import java.util.Map; 7 | 8 | public class App 9 | { 10 | static void insert(Connection connection) throws URISyntaxException, SQLException{ 11 | final Statement stmt = connection.createStatement(); 12 | stmt.executeQuery("DROP TABLE IF EXISTS perf_java"); 13 | stmt.executeQuery("CREATE TABLE IF NOT EXISTS perf_java (" + 14 | " id UInt32," + 15 | " name String," + 16 | " dt DateTime " + 17 | ") Engine=MergeTree PARTITION BY name ORDER BY dt"); 18 | stmt.close(); 19 | 20 | final String[] Names = {"one","two","three","four","five"}; 21 | 22 | long startTime = System.nanoTime(); 23 | final PreparedStatement pstmt = connection.prepareStatement("INSERT INTO perf_java VALUES(?, ?, ?)"); 24 | 25 | if(pstmt==null){ 26 | return; 27 | } 28 | 29 | long now = System.currentTimeMillis(); 30 | for (int j = 1; j <= 1000; j++) { 31 | for (int i = 1; i <= 10000; i++) { 32 | pstmt.setInt(1,i); 33 | pstmt.setString( 2, Names[ i % Names.length] ); 34 | pstmt.setTimestamp(3, new java.sql.Timestamp(now + i ) ); 35 | pstmt.addBatch(); 36 | } 37 | pstmt.executeBatch(); 38 | pstmt.clearBatch(); 39 | } 40 | 41 | pstmt.close(); 42 | 43 | long stopTime = System.nanoTime(); 44 | System.out.format("elapsed %d msec", (stopTime - startTime)/1000000); 45 | connection.close(); 46 | } 47 | 48 | static void select(Connection connection) throws URISyntaxException, SQLException{ 49 | long startTime = System.nanoTime(); 50 | final Statement stmt = connection.createStatement(); 51 | ResultSet rs = stmt.executeQuery("SELECT id, name, dt FROM perf"); 52 | long sum = 0; 53 | 54 | while (rs.next()) { 55 | sum += rs.getInt(1); 56 | final String name = rs.getString(2); 57 | final java.sql.Timestamp time = rs.getTimestamp(3); 58 | } 59 | 60 | long stopTime = System.nanoTime(); 61 | System.out.format("elapsed %d msec\n", (stopTime - startTime)/1000000); 62 | System.out.format("sum %d\n",sum); 63 | connection.close(); 64 | } 65 | 66 | public static void main( String[] args ) throws URISyntaxException, SQLException { 67 | Map env = System.getenv(); 68 | final String url = env.containsKey("DATABASE_URL")? 69 | env.get("DATABASE_URL"): "clickhouse://127.0.0.1:9000"; 70 | 71 | // transform a url to an adopted for this driver form 72 | final URI uri = URI.create(url); 73 | 74 | String username = uri.getUserInfo(); 75 | String password = ""; 76 | String query = uri.getQuery(); 77 | 78 | String[] userinfo = uri.getUserInfo().split(":"); 79 | if ( userinfo.length>1 ){ 80 | username = userinfo[0]; 81 | password = userinfo[1]; 82 | query = query + "&password="+password; 83 | } 84 | 85 | final URI uri2 = new URI("jdbc:clickhouse", username, uri.getHost(), 86 | (uri.getPort()==-1)?9000: uri.getPort(), 87 | uri.getPath(), 88 | query, 89 | uri.getFragment()); 90 | // Show resulted url 91 | System.out.format("url: '%s'\n",uri2); 92 | 93 | final Connection connection = DriverManager.getConnection(uri2.toString()); 94 | 95 | if (args.length>1 && args[1].equals("insert")){ 96 | App.insert(connection); 97 | return; 98 | } 99 | if (args.length>1 && args[1].equals("select")){ 100 | App.select(connection); 101 | return; 102 | } 103 | System.err.println("specify bench method `select` or `insert` in the first argument"); 104 | 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /tests/bench/java/src/test/java/com/github/ddulesov/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.github.ddulesov; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/bench/py/bench.py: -------------------------------------------------------------------------------- 1 | from clickhouse_driver import Client 2 | import datetime 3 | import pytz 4 | import os 5 | import datetime 6 | import time 7 | import sys 8 | 9 | from urllib.parse import urlparse 10 | 11 | url = os.environ.get('DATABASE_URL',"tcp://default@localhost/default?compression=lz4") 12 | 13 | url = urlparse(url)._replace(scheme='clickhouse').geturl() 14 | 15 | client = Client.from_url(url) 16 | 17 | 18 | ddl = ("CREATE TABLE IF NOT EXISTS perf_py (" 19 | "id UInt32," 20 | "name String," 21 | "dt DateTime " 22 | ") Engine=MergeTree PARTITION BY name ORDER BY dt") 23 | 24 | client.execute("DROP TABLE IF EXISTS perf_py") 25 | client.execute( ddl ) 26 | 27 | NAMES = ["one","two","three","four","five"]; 28 | 29 | BSIZE = 10000 30 | CIRCLE = 1000 31 | 32 | def next_block(i): 33 | now = datetime.datetime.now() 34 | sec = datetime.timedelta(seconds=1) 35 | 36 | block = [] 37 | for i in range(BSIZE): 38 | block.append([i, NAMES[ i % len(NAMES) ], now + i * sec ] ) 39 | 40 | return block 41 | 42 | 43 | def select_perf(): 44 | start_time = time.time_ns() 45 | settings = {'max_block_size': 100000} 46 | rows = client.execute_iter("SELECT * FROM perf",settings=settings) 47 | 48 | for row in rows: 49 | pass 50 | 51 | print("elapsed %s msec" % ( (time.time_ns() - start_time)/1000000 )) 52 | 53 | 54 | def insert_perf(): 55 | 56 | start_time = time.time_ns() 57 | 58 | for i in range(0,CIRCLE): 59 | client.execute('INSERT INTO perf_py (id, name, dt) VALUES', next_block(i) ) 60 | 61 | print("elapsed %s msec" % ( (time.time_ns() - start_time)/1000000 )) 62 | 63 | if __name__ == "__main__": 64 | if len( sys.argv )<2: 65 | print("specify perf test. 'insert' or 'select'. bench.py .") 66 | sys.exit(1) 67 | perf_name = sys.argv[1] 68 | 69 | f = globals()[ perf_name + "_perf" ] 70 | f() 71 | -------------------------------------------------------------------------------- /tests/create.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE main( 2 | i64 Int64, 3 | u64 UInt64, 4 | i32 Int32, 5 | u32 UInt32, 6 | i16 Int16, 7 | u16 UInt16, 8 | i8 Int8, 9 | u8 UInt8, 10 | f32 Float32, 11 | f64 Float64, 12 | title String, 13 | lcs LowCardinality(String), 14 | mon FixedString(3), 15 | d Date, 16 | t DateTime, 17 | dt64 DateTime64(3,'Europe/Moscow'), 18 | uuid UUID, 19 | e8 Enum8('yes'=1, 'no'=0, 'n/a'=-1), 20 | e16 Enum16('rust'=10,'go'=1,'c++'=2,'python'=3,'java'=4,'c#'=5), 21 | ip4 IPv4, 22 | ip6 IPv6, 23 | n Nullable(UInt16), 24 | dm1 Decimal(9,2), dm2 Decimal(12,6) 25 | ) ENGINE=MergeTree() ORDER BY d PARTITION BY i64 -------------------------------------------------------------------------------- /tests/createx.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE mainx(i64 Int64, 2 | a8 Array(Int8), 3 | a16 Array(Int16), 4 | a32 Array(UInt32), 5 | a64 Array(UInt64), 6 | a8d Array(Array(UInt8)), 7 | aip4 Array(IPv4), 8 | aip6 Array(IPv6), 9 | ad Array(Date), 10 | adt Array(DateTime), 11 | adc Array(Decimal(9,2)), 12 | lcs LowCardinality(Nullable(String)) 13 | ) ENGINE=MergeTree() ORDER BY i64 PARTITION BY i64 --------------------------------------------------------------------------------