├── .cargo └── audit.toml ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── licenserc.yaml └── workflows │ └── ci.yaml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE ├── README.md ├── clickhouse ├── Cargo.toml ├── README.md ├── examples │ └── simple.rs ├── src │ ├── binary │ │ ├── encoder.rs │ │ ├── mod.rs │ │ ├── parser.rs │ │ ├── read_ex.rs │ │ └── uvarint.rs │ ├── cmd.rs │ ├── connection.rs │ ├── error_codes.rs │ ├── errors.rs │ ├── lib.rs │ ├── protocols │ │ ├── mod.rs │ │ ├── protocol_exception.rs │ │ ├── protocol_hello.rs │ │ ├── protocol_query.rs │ │ └── protocol_type.rs │ └── types │ │ ├── block │ │ ├── block_info.rs │ │ ├── builder.rs │ │ ├── chunk_iterator.rs │ │ ├── compressed.rs │ │ ├── mod.rs │ │ └── row.rs │ │ ├── column │ │ ├── array.rs │ │ ├── chrono_datetime.rs │ │ ├── chunk.rs │ │ ├── column_data.rs │ │ ├── concat.rs │ │ ├── date.rs │ │ ├── datetime64.rs │ │ ├── decimal.rs │ │ ├── enums.rs │ │ ├── factory.rs │ │ ├── fixed_string.rs │ │ ├── ip.rs │ │ ├── iter.rs │ │ ├── list.rs │ │ ├── mod.rs │ │ ├── nullable.rs │ │ ├── numeric.rs │ │ ├── string.rs │ │ ├── string_pool.rs │ │ └── tuple.rs │ │ ├── date_converter.rs │ │ ├── decimal.rs │ │ ├── enums.rs │ │ ├── from_sql.rs │ │ ├── mod.rs │ │ ├── options.rs │ │ ├── query.rs │ │ ├── stat_buffer.rs │ │ ├── value.rs │ │ └── value_ref.rs └── tests │ └── it │ ├── binary.rs │ ├── errors.rs │ ├── main.rs │ └── types │ ├── block │ ├── builder.rs │ ├── compressed.rs │ └── mod.rs │ ├── column │ ├── array.rs │ ├── concat.rs │ ├── date.rs │ ├── datetime64.rs │ ├── factory.rs │ ├── iter.rs │ ├── list.rs │ ├── mod.rs │ ├── string_pool.rs │ └── tuple.rs │ ├── decimal.rs │ ├── from_sql.rs │ ├── mod.rs │ ├── opions.rs │ ├── value.rs │ └── value_ref.rs ├── components └── micromarshal │ ├── Cargo.toml │ ├── README.md │ ├── src │ ├── lib.rs │ ├── marshal.rs │ └── unmarshal.rs │ └── tests │ └── it │ ├── main.rs │ └── statbuffer.rs ├── mysql ├── Cargo.toml ├── README.md ├── examples │ ├── serve_auth.rs │ ├── serve_one.rs │ └── serve_secure.rs ├── src │ ├── commands.rs │ ├── errorcodes.rs │ ├── lib.rs │ ├── packet_reader.rs │ ├── packet_writer.rs │ ├── params.rs │ ├── resultset.rs │ ├── tests │ │ ├── commands.rs │ │ ├── mod.rs │ │ ├── packet.rs │ │ └── value │ │ │ ├── decode.rs │ │ │ ├── encode.rs │ │ │ └── mod.rs │ ├── tls.rs │ ├── value │ │ ├── decode.rs │ │ ├── encode.rs │ │ └── mod.rs │ └── writers.rs └── tests │ ├── it │ ├── async.rs │ ├── main.rs │ └── secure.rs │ └── ssl │ ├── server.crt │ └── server.key ├── rust-toolchain.toml └── rustfmt.toml /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = ["RUSTSEC-2020-0159", "RUSTSEC-2020-0071"] 3 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | I hereby agree to the terms of the CLA available at: https://databend.rs/dev/policies/cla/ 2 | 3 | ## Summary 4 | 5 | Summary about this PR 6 | 7 | -------------------------------------------------------------------------------- /.github/licenserc.yaml: -------------------------------------------------------------------------------- 1 | header: 2 | license: 3 | spdx-id: Apache-2.0 4 | copyright-owner: Datafuse Labs 5 | paths-ignore: 6 | # Ignore hidden files 7 | - ".cargo" 8 | - ".github" 9 | - ".gitignore" 10 | # Ignore LICENSE and Makefile 11 | - "LICENSE" 12 | # Ignore docs and generated files 13 | - "**/*.md" 14 | - "**/*.yaml" 15 | - "**/*.toml" 16 | - "**/*.lock" 17 | - "**/*.crt" 18 | - "**/*.key" 19 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | 2 | name: CI 3 | 4 | on: 5 | push: 6 | pull_request: 7 | schedule: 8 | - cron: '25 4 * * *' 9 | 10 | concurrency: 11 | group: ${{ github.ref }}-${{ github.workflow }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | license: 16 | name: Check license header 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff 21 | with: 22 | config: .github/licenserc.yaml 23 | 24 | security: 25 | name: Audit security 26 | runs-on: ubuntu-latest 27 | steps: 28 | - uses: actions/checkout@v4 29 | - name: Install cargo-aduit 30 | run: cargo install cargo-audit 31 | - name: Audit dependencies 32 | run: cargo audit 33 | 34 | style: 35 | name: Audit style 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v4 39 | - name: Check clippy 40 | run: cargo clippy --workspace --all-targets --all-features -- -D warnings 41 | - name: Check format 42 | run: cargo fmt --all -- --check 43 | 44 | test: 45 | name: Run tests 46 | strategy: 47 | matrix: 48 | os: [ubuntu-latest, macos-latest] 49 | 50 | runs-on: ${{ matrix.os }} 51 | 52 | steps: 53 | - uses: actions/checkout@v4 54 | - name: Install mysql-client 55 | if: ${{ contains(matrix.os, 'macos') }} 56 | run: brew install mysql@8.4 57 | - run: cargo build 58 | - run: | 59 | export PATH="/opt/homebrew/opt/mysql@8.4/bin:$PATH" 60 | cargo test --lib --bins --tests -- --include-ignored 61 | 62 | pass: 63 | name: All tests passed 64 | runs-on: ubuntu-latest 65 | needs: 66 | - security 67 | - style 68 | - test 69 | steps: 70 | - run: exit 0 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ project files 2 | .idea/* 3 | 4 | /target 5 | Cargo.lock 6 | .DS_Store 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This is a Rust project, so [rustup](https://rustup.rs/) is the best place to start. 4 | 5 | This is a pure rust project, so only `cargo` is needed. 6 | 7 | - `cargo check` to analyze the current package and report errors. 8 | - `cargo build` to compile the current package. 9 | - `cargo clippy` to catch common mistakes and improve code. 10 | - `cargo test` to run unit tests. 11 | 12 | Useful tips: 13 | 14 | - Check/Build/Test/Clippy all code: `cargo --workspace` 15 | - Check/Build/Test/Clippy specific package: `cargo --package ` 16 | - Test specific function: `cargo test --package opensrv-mysql r#async::it_connects` 17 | 18 | For changelog: 19 | 20 | ``` 21 | pip install git-changelog 22 | git-changelog -o CHANGELOG.md -c angular -t keepachangelog . 23 | ``` 24 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["clickhouse", "mysql", "components/micromarshal"] 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenSrv 2 | 3 | [![Build Status]][actions] 4 | 5 | [Build Status]: https://img.shields.io/github/actions/workflow/status/datafuselabs/opensrv/ci.yaml?branch=main 6 | [actions]: https://github.com/datafuselabs/opensrv/actions?query=branch%3Amain 7 | 8 | Async bindings for emulating database servers. Currently, support for ClickHouse and MySQL/MariaDB is provided. 9 | 10 | ## Status 11 | 12 | OpenSrv is in **alpha** stage, the goal is to provide high performance and highly reliable service compatibility for Databend and others. Welcome any feedback at [Discussions](https://github.com/datafuselabs/opensrv/discussions)! 13 | 14 | ## Examples 15 | 16 | You may be looking for: 17 | 18 | - [ClickHouse Examples](./clickhouse/examples) 19 | - [MySQL/MariaDB Examples](./mysql/examples) 20 | 21 | 22 | ## Projects 23 | 24 | - [Databend](https://github.com/datafuselabs/databend/): A powerful cloud data warehouse. Built for elasticity and efficiency. Free and open. Also available in the cloud: 25 | - [GreptimeDB](https://github.com/GreptimeTeam/greptimedb): An open-source, cloud-native, distributed time-series database. 26 | - [CeresDB](https://github.com/CeresDB/ceresdb): A high-performance, distributed, cloud native time-series database that can handle both time-series and analytics workloads. 27 | 28 | ## Contributing 29 | 30 | Check out the [CONTRIBUTING.md](./CONTRIBUTING.md) guide for more details on getting started with contributing to this project. 31 | 32 | ## Getting help 33 | 34 | Submit [issues](https://github.com/datafuselabs/opensrv/issues/new/choose) for bug report or asking questions in [discussion](https://github.com/datafuselabs/opensrv/discussions/new?category=q-a). 35 | 36 | ## License 37 | 38 | Licensed under Apache License, Version 2.0. 39 | 40 | -------------------------------------------------------------------------------- /clickhouse/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "opensrv-clickhouse" 3 | version = "0.7.0" 4 | authors = ["Databend Authors "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | description = "Bindings for emulating a ClickHouse server." 8 | readme = "README.md" 9 | repository = "https://github.com/datafuselabs/opensrv" 10 | keywords = ["api-bindings", "database", "sql", "mock"] 11 | categories = ["api-bindings", "network-programming", "database-implementations"] 12 | 13 | [lib] 14 | doctest = false 15 | test = false 16 | 17 | [features] 18 | tls = ["tokio-native-tls"] 19 | 20 | [dependencies] 21 | async-trait = "0.1.52" 22 | byteorder = "1.4.3" 23 | bytes = "1.1.0" 24 | chrono = { version = "0.4.20", default-features = false, features = ["std"] } 25 | chrono-tz = "0.8.0" 26 | combine = "4.6.3" 27 | hostname = "0.3.1" 28 | lz4 = "1.23.2" 29 | micromarshal = { version = "0.7.0", path = "../components/micromarshal" } 30 | naive-cityhash = "0.2.0" 31 | once_cell = "1.9.0" 32 | thiserror = "1.0.30" 33 | tokio = { version = "1.17.0", features = ["full"] } 34 | tokio-native-tls = { version = "0.3.0", optional = true } 35 | tokio-stream = "0.1.8" 36 | tracing = "0.1.31" 37 | url = "2.2.2" 38 | uuid = "1.1.0" 39 | 40 | [dev-dependencies] 41 | futures = "0.3.21" 42 | futures-core = "0.3.21" 43 | futures-sink = "0.3.21" 44 | futures-util = { version = "0.3.21", features = ["sink"] } 45 | rand = "0.8.5" 46 | -------------------------------------------------------------------------------- /clickhouse/README.md: -------------------------------------------------------------------------------- 1 | # OpenSrv - ClickHouse 2 | 3 | **Bindings for emulating a ClickHouse server.** 4 | 5 | ## Usage 6 | 7 | See the full example [here](./examples/simple.rs) 8 | 9 | ```rust 10 | struct Session { 11 | last_progress_send: Instant, 12 | metadata: ClickHouseMetadata, 13 | } 14 | 15 | #[async_trait::async_trait] 16 | impl opensrv_clickhouse::ClickHouseSession for Session { 17 | async fn execute_query(&self, ctx: &mut CHContext, connection: &mut Connection) -> Result<()> { 18 | let query = ctx.state.query.clone(); 19 | tracing::debug!("Receive query {}", query); 20 | 21 | let start = Instant::now(); 22 | 23 | // simple logic for insert 24 | if query.starts_with("INSERT") || query.starts_with("insert") { 25 | // ctx.state.out 26 | let sample_block = Block::new().column("abc", Vec::::new()); 27 | let (sender, rec) = mpsc::channel(4); 28 | ctx.state.out = Some(sender); 29 | connection.write_block(&sample_block).await?; 30 | 31 | let sent_all_data = ctx.state.sent_all_data.clone(); 32 | tokio::spawn(async move { 33 | let mut rows = 0; 34 | let mut stream = ReceiverStream::new(rec); 35 | while let Some(block) = stream.next().await { 36 | rows += block.row_count(); 37 | println!( 38 | "got insert block: {:?}, total_rows: {}", 39 | block.row_count(), 40 | rows 41 | ); 42 | } 43 | sent_all_data.notify_one(); 44 | }); 45 | return Ok(()); 46 | } 47 | 48 | let mut clickhouse_stream = SimpleBlockStream { 49 | idx: 0, 50 | start: 10, 51 | end: 24, 52 | blocks: 10, 53 | }; 54 | 55 | while let Some(block) = clickhouse_stream.next().await { 56 | let block = block?; 57 | connection.write_block(&block).await?; 58 | 59 | if self.last_progress_send.elapsed() >= Duration::from_millis(10) { 60 | let progress = self.get_progress(); 61 | connection 62 | .write_progress(progress, ctx.client_revision) 63 | .await?; 64 | } 65 | } 66 | 67 | let duration = start.elapsed(); 68 | tracing::debug!( 69 | "ClickHouseHandler executor cost:{:?}, statistics:{:?}", 70 | duration, 71 | "xxx", 72 | ); 73 | Ok(()) 74 | } 75 | 76 | fn metadata(&self) -> &ClickHouseMetadata { 77 | &self.metadata 78 | } 79 | 80 | fn get_progress(&self) -> Progress { 81 | Progress { 82 | rows: 100, 83 | bytes: 1000, 84 | total_rows: 1000, 85 | } 86 | } 87 | } 88 | ``` 89 | 90 | ## Getting help 91 | 92 | Submit [issues](https://github.com/datafuselabs/opensrv/issues/new/choose) for bug report or asking questions in [discussion](https://github.com/datafuselabs/opensrv/discussions/new?category=q-a). 93 | 94 | ## Credits 95 | 96 | This project used to be [sundy-li/clickhouse-srv](https://github.com/sundy-li/clickhouse-srv). 97 | 98 | ## License 99 | 100 | Licensed under Apache License, Version 2.0. -------------------------------------------------------------------------------- /clickhouse/src/binary/encoder.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::binary; 16 | use crate::types::StatBuffer; 17 | use micromarshal::Marshal; 18 | 19 | const MAX_VARINT_LEN64: usize = 10; 20 | 21 | #[derive(Default)] 22 | pub struct Encoder { 23 | buffer: Vec, 24 | } 25 | 26 | impl Encoder { 27 | pub fn new() -> Self { 28 | Encoder { buffer: Vec::new() } 29 | } 30 | pub fn get_buffer(self) -> Vec { 31 | self.buffer 32 | } 33 | 34 | pub fn get_buffer_ref(&self) -> &[u8] { 35 | self.buffer.as_ref() 36 | } 37 | 38 | pub fn uvarint(&mut self, v: u64) { 39 | let mut scratch = [0u8; MAX_VARINT_LEN64]; 40 | let ln = binary::put_uvarint(&mut scratch[..], v); 41 | self.write_bytes(&scratch[..ln]); 42 | } 43 | 44 | pub fn string(&mut self, text: impl AsRef) { 45 | let bytes = text.as_ref().as_bytes(); 46 | self.byte_string(bytes); 47 | } 48 | 49 | pub fn byte_string(&mut self, source: impl AsRef<[u8]>) { 50 | self.uvarint(source.as_ref().len() as u64); 51 | self.write_bytes(source.as_ref()); 52 | } 53 | 54 | pub fn write(&mut self, value: T) 55 | where 56 | T: Copy + Marshal + StatBuffer, 57 | { 58 | let mut buffer = T::buffer(); 59 | value.marshal(buffer.as_mut()); 60 | self.write_bytes(buffer.as_ref()); 61 | } 62 | 63 | pub fn write_bytes(&mut self, b: &[u8]) { 64 | self.buffer.extend_from_slice(b); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /clickhouse/src/binary/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub use self::encoder::Encoder; 16 | pub use self::parser::Parser; 17 | pub use self::read_ex::ReadEx; 18 | pub use self::uvarint::put_uvarint; 19 | 20 | mod encoder; 21 | mod parser; 22 | mod read_ex; 23 | mod uvarint; 24 | -------------------------------------------------------------------------------- /clickhouse/src/binary/parser.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Read; 16 | 17 | use chrono_tz::Tz; 18 | 19 | use crate::binary::ReadEx; 20 | use crate::errors::DriverError; 21 | use crate::errors::Error; 22 | use crate::errors::Result; 23 | use crate::protocols::HelloRequest; 24 | use crate::protocols::Packet; 25 | use crate::protocols::QueryRequest; 26 | use crate::protocols::{self}; 27 | use crate::types::Block; 28 | 29 | /// The internal clickhouse client request parser. 30 | pub struct Parser { 31 | reader: T, 32 | 33 | tz: Tz, 34 | } 35 | 36 | /// The parser can be used to parse clickhouse client requests 37 | impl Parser { 38 | /// Creates a new parser that parses the data behind the reader. More 39 | /// than one value can be behind the reader in which case the parser can 40 | /// be invoked multiple times. In other words: the stream does not have 41 | /// to be terminated. 42 | pub(crate) fn new(reader: T, tz: Tz) -> Parser { 43 | Self { reader, tz } 44 | } 45 | 46 | pub(crate) fn parse_packet( 47 | &mut self, 48 | hello: &Option, 49 | compress: bool, 50 | ) -> Result { 51 | let packet = self.reader.read_uvarint()?; 52 | 53 | match packet { 54 | protocols::CLIENT_PING => Ok(Packet::Ping), 55 | protocols::CLIENT_CANCEL => Ok(Packet::Cancel), 56 | protocols::CLIENT_DATA | protocols::CLIENT_SCALAR => { 57 | Ok(self.parse_data(packet == protocols::CLIENT_SCALAR, compress)?) 58 | } 59 | protocols::CLIENT_QUERY => Ok(self.parse_query(hello, compress)?), 60 | protocols::CLIENT_HELLO => Ok(self.parse_hello()?), 61 | 62 | _ => Err(Error::Driver(DriverError::UnknownPacket { packet })), 63 | } 64 | } 65 | 66 | fn parse_hello(&mut self) -> Result { 67 | Ok(Packet::Hello(HelloRequest::read_from(&mut self.reader)?)) 68 | } 69 | 70 | fn parse_query(&mut self, hello: &Option, _compress: bool) -> Result { 71 | match hello { 72 | Some(ref hello) => { 73 | let query = QueryRequest::read_from(&mut self.reader, hello)?; 74 | Ok(Packet::Query(query)) 75 | } 76 | _ => Err(Error::Driver(DriverError::UnexpectedPacket)), 77 | } 78 | } 79 | 80 | fn parse_data(&mut self, _scalar: bool, compress: bool) -> Result { 81 | let _temporary_table = self.reader.read_string()?; 82 | let block = Block::load(&mut self.reader, self.tz, compress)?; 83 | Ok(Packet::Data(block)) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /clickhouse/src/binary/read_ex.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io; 16 | use std::mem::MaybeUninit; 17 | 18 | use micromarshal::Unmarshal; 19 | 20 | use crate::errors::DriverError; 21 | use crate::errors::Error; 22 | use crate::errors::Result; 23 | use crate::types::column::StringPool; 24 | use crate::types::StatBuffer; 25 | 26 | pub trait ReadEx { 27 | fn read_bytes(&mut self, rv: &mut [u8]) -> Result<()>; 28 | fn read_scalar(&mut self) -> Result 29 | where 30 | V: Copy + Unmarshal + StatBuffer; 31 | fn read_string(&mut self) -> Result; 32 | fn read_len_encode_bytes(&mut self) -> Result>; 33 | fn skip_string(&mut self) -> Result<()>; 34 | fn read_uvarint(&mut self) -> Result; 35 | fn read_str_into_buffer(&mut self, pool: &mut StringPool) -> Result<()>; 36 | } 37 | 38 | const MAX_STACK_BUFFER_LEN: usize = 1024; 39 | 40 | impl ReadEx for T 41 | where 42 | T: io::Read, 43 | { 44 | fn read_bytes(&mut self, rv: &mut [u8]) -> Result<()> { 45 | let mut i = 0; 46 | while i < rv.len() { 47 | let res_nread = { 48 | let buf = &mut rv[i..]; 49 | self.read(buf) 50 | }; 51 | match res_nread { 52 | Ok(0) => { 53 | let ret = io::Error::new(io::ErrorKind::WouldBlock, "would block"); 54 | return Err(ret.into()); 55 | } 56 | Ok(nread) => i += nread, 57 | Err(e) => return Err(From::from(e)), 58 | } 59 | } 60 | Ok(()) 61 | } 62 | 63 | fn read_scalar(&mut self) -> Result 64 | where 65 | V: Copy + Unmarshal + StatBuffer, 66 | { 67 | let mut buffer = V::buffer(); 68 | self.read_bytes(buffer.as_mut())?; 69 | Ok(V::unmarshal(buffer.as_ref())) 70 | } 71 | 72 | fn read_string(&mut self) -> Result { 73 | let str_len = self.read_uvarint()? as usize; 74 | let mut buffer = vec![0_u8; str_len]; 75 | self.read_bytes(buffer.as_mut())?; 76 | Ok(String::from_utf8(buffer)?) 77 | } 78 | 79 | fn read_len_encode_bytes(&mut self) -> Result> { 80 | let str_len = self.read_uvarint()? as usize; 81 | let mut buffer = vec![0_u8; str_len]; 82 | self.read_bytes(buffer.as_mut())?; 83 | 84 | Ok(buffer) 85 | } 86 | 87 | fn skip_string(&mut self) -> Result<()> { 88 | let str_len = self.read_uvarint()? as usize; 89 | 90 | if str_len <= MAX_STACK_BUFFER_LEN { 91 | unsafe { 92 | let mut buffer: [MaybeUninit; MAX_STACK_BUFFER_LEN] = 93 | MaybeUninit::uninit().assume_init(); 94 | self.read_bytes( 95 | &mut *(&mut buffer[..str_len] as *mut [MaybeUninit] as *mut [u8]), 96 | )?; 97 | } 98 | } else { 99 | let mut buffer = vec![0_u8; str_len]; 100 | self.read_bytes(buffer.as_mut())?; 101 | } 102 | 103 | Ok(()) 104 | } 105 | 106 | fn read_uvarint(&mut self) -> Result { 107 | let mut x = 0_u64; 108 | let mut s = 0_u32; 109 | let mut i = 0_usize; 110 | loop { 111 | let b: u8 = self.read_scalar()?; 112 | 113 | if b < 0x80 { 114 | if i == 9 && b > 1 { 115 | return Err(Error::Driver(DriverError::Overflow)); 116 | } 117 | return Ok(x | (u64::from(b) << s)); 118 | } 119 | 120 | x |= u64::from(b & 0x7f) << s; 121 | s += 7; 122 | 123 | i += 1; 124 | } 125 | } 126 | 127 | fn read_str_into_buffer(&mut self, pool: &mut StringPool) -> Result<()> { 128 | let str_len = self.read_uvarint()? as usize; 129 | let buffer = pool.allocate(str_len); 130 | self.read_bytes(buffer)?; 131 | Ok(()) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /clickhouse/src/binary/uvarint.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // put_uvarint encodes a uint64 into buf and returns the number of bytes written. 16 | // If the buffer is too small, put_uvarint will panic. 17 | pub fn put_uvarint(mut buffer: impl AsMut<[u8]>, x: u64) -> usize { 18 | let mut i = 0; 19 | let mut mx = x; 20 | let buf = buffer.as_mut(); 21 | while mx >= 0x80 { 22 | buf[i] = mx as u8 | 0x80; 23 | mx >>= 7; 24 | i += 1; 25 | } 26 | buf[i] = mx as u8; 27 | i + 1 28 | } 29 | -------------------------------------------------------------------------------- /clickhouse/src/cmd.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::binary::Encoder; 16 | use crate::connection::Connection; 17 | use crate::error_codes::WRONG_PASSWORD; 18 | use crate::errors::Error; 19 | use crate::errors::Result; 20 | use crate::errors::ServerError; 21 | use crate::protocols::HelloResponse; 22 | use crate::protocols::Packet; 23 | use crate::protocols::Stage; 24 | use crate::protocols::SERVER_PONG; 25 | use crate::CHContext; 26 | 27 | pub struct Cmd { 28 | packet: Packet, 29 | } 30 | 31 | impl Cmd { 32 | pub fn create(packet: Packet) -> Self { 33 | Self { packet } 34 | } 35 | 36 | pub async fn apply(self, connection: &mut Connection, ctx: &mut CHContext) -> Result<()> { 37 | let mut encoder = Encoder::new(); 38 | match self.packet { 39 | Packet::Ping => { 40 | encoder.uvarint(SERVER_PONG); 41 | } 42 | // todo cancel 43 | Packet::Cancel => {} 44 | Packet::Hello(hello) => { 45 | if !connection 46 | .session 47 | .authenticate(&hello.user, &hello.password, &connection.client_addr) 48 | .await 49 | { 50 | let err = Error::Server(ServerError { 51 | code: WRONG_PASSWORD, 52 | name: "AuthenticateException".to_owned(), 53 | message: "Unknown user or wrong password".to_owned(), 54 | stack_trace: "".to_owned(), 55 | }); 56 | connection.write_error(&err).await?; 57 | return Err(err); 58 | } 59 | 60 | let metadata = connection.session.metadata(); 61 | let (dbms_version_major, dbms_version_minor, dbms_version_patch) = 62 | metadata.version(); 63 | let response = HelloResponse { 64 | dbms_name: metadata.name().to_string(), 65 | dbms_version_major, 66 | dbms_version_minor, 67 | dbms_tcp_protocol_version: metadata.tcp_protocol_version(), 68 | timezone: metadata.timezone().to_string(), 69 | server_display_name: metadata.display_name().to_string(), 70 | dbms_version_patch, 71 | }; 72 | 73 | ctx.client_revision = metadata.tcp_protocol_version.min(hello.client_revision); 74 | ctx.hello = Some(hello); 75 | 76 | response.encode(&mut encoder, ctx.client_revision)?; 77 | } 78 | Packet::Query(query) => { 79 | ctx.state.query = query.query.clone(); 80 | ctx.state.compression = query.compression; 81 | 82 | let session = connection.session.clone(); 83 | session.execute_query(ctx, connection).await?; 84 | 85 | if ctx.state.out.is_some() { 86 | ctx.state.stage = Stage::InsertPrepare; 87 | } else { 88 | connection.write_end_of_stream().await?; 89 | } 90 | } 91 | Packet::Data(block) => { 92 | if block.is_empty() { 93 | match ctx.state.stage { 94 | Stage::InsertPrepare => { 95 | ctx.state.stage = Stage::InsertStarted; 96 | } 97 | Stage::InsertStarted => { 98 | // reset will reset the out, so the outer stream will break 99 | ctx.state.reset(); 100 | 101 | ctx.state.sent_all_data.notified().await; 102 | // wait stream finished 103 | connection.write_end_of_stream().await?; 104 | ctx.state.stage = Stage::Default; 105 | } 106 | _ => {} 107 | } 108 | } else if let Some(out) = &ctx.state.out { 109 | // out.block_stream. 110 | out.send(block).await.unwrap(); 111 | } 112 | } 113 | }; 114 | 115 | let bytes = encoder.get_buffer(); 116 | if !bytes.is_empty() { 117 | connection.write_bytes(bytes).await?; 118 | } 119 | Ok(()) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /clickhouse/src/protocols/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod protocol_exception; 16 | mod protocol_hello; 17 | mod protocol_query; 18 | mod protocol_type; 19 | 20 | pub use protocol_exception::*; 21 | pub use protocol_hello::*; 22 | pub use protocol_query::*; 23 | pub use protocol_type::*; 24 | 25 | use crate::types::Block; 26 | 27 | #[derive(Debug)] 28 | pub enum Packet { 29 | Ping, 30 | Cancel, 31 | Hello(HelloRequest), 32 | Query(QueryRequest), 33 | Data(Block), 34 | } 35 | 36 | #[derive(Debug, Default)] 37 | pub enum Stage { 38 | #[default] 39 | Default = 0, 40 | InsertPrepare, 41 | InsertStarted, 42 | EOS, 43 | } 44 | 45 | pub const DBMS_MIN_REVISION_WITH_CLIENT_INFO: u64 = 54032; 46 | pub const DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE: u64 = 54058; 47 | pub const DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO: u64 = 54060; 48 | pub const DBMS_MIN_REVISION_WITH_TABLES_STATUS: u64 = 54226; 49 | pub const DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE: u64 = 54337; 50 | pub const DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME: u64 = 54372; 51 | pub const DBMS_MIN_REVISION_WITH_VERSION_PATCH: u64 = 54401; 52 | pub const DBMS_MIN_REVISION_WITH_SERVER_LOGS: u64 = 54406; 53 | pub const DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA: u64 = 54415; 54 | // Minimum revision with exactly the same set of aggregation methods and rules to select them. 55 | // Two-level (bucketed) aggregation is incompatible if servers are inconsistent in these rules 56 | // (keys will be placed in different buckets and result will not be fully aggregated). 57 | pub const DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD: u64 = 54431; 58 | pub const DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA: u64 = 54410; 59 | 60 | pub const DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE: u64 = 54405; 61 | pub const DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO: u64 = 54420; 62 | 63 | // Minimum revision supporting SettingsBinaryFormat::STRINGS. 64 | pub const DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS: u64 = 54429; 65 | 66 | // Minimum revision supporting OpenTelemetry 67 | pub const DBMS_MIN_REVISION_WITH_OPENTELEMETRY: u64 = 54442; 68 | 69 | // Minimum revision supporting interserver secret. 70 | pub const DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET: u64 = 54441; 71 | 72 | pub const DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO: u64 = 54443; 73 | pub const DBMS_MIN_REVISION_WITH_REFERER_IN_CLIENT_INFO: u64 = 54447; 74 | -------------------------------------------------------------------------------- /clickhouse/src/protocols/protocol_exception.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::binary::Encoder; 16 | use crate::error_codes::UNKNOWN_EXCEPTION; 17 | use crate::errors::Error; 18 | use crate::protocols::*; 19 | 20 | pub struct ExceptionResponse {} 21 | 22 | impl ExceptionResponse { 23 | pub fn write(encoder: &mut Encoder, error: &Error, with_stack_trace: bool) { 24 | let mut code = UNKNOWN_EXCEPTION; 25 | let name = error.exception_name(); 26 | let mut stack_trace = "".to_string(); 27 | let mut message = error.to_string(); 28 | 29 | if let Error::Server(e) = error { 30 | code = e.code; 31 | if with_stack_trace { 32 | stack_trace = e.stack_trace.clone(); 33 | } 34 | message = e.message.clone(); 35 | } 36 | encoder.uvarint(SERVER_EXCEPTION); 37 | 38 | encoder.write(code); 39 | //Name 40 | encoder.string(name); 41 | // Message 42 | encoder.string(message); 43 | // StackTrace 44 | encoder.string(stack_trace); 45 | // Nested. 46 | encoder.write(false); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /clickhouse/src/protocols/protocol_hello.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Read; 16 | 17 | use crate::binary::Encoder; 18 | use crate::binary::ReadEx; 19 | use crate::error_codes; 20 | use crate::errors::Error; 21 | use crate::errors::Result; 22 | use crate::errors::ServerError; 23 | use crate::protocols::*; 24 | 25 | #[derive(Default, Debug, Clone)] 26 | pub struct HelloRequest { 27 | pub client_name: Vec, 28 | pub client_version_major: u64, 29 | pub client_version_minor: u64, 30 | pub client_revision: u64, 31 | pub default_database: Vec, 32 | 33 | pub user: String, 34 | pub password: Vec, 35 | 36 | // Not set currently 37 | pub client_version_patch: u64, 38 | } 39 | 40 | impl HelloRequest { 41 | pub fn read_from(reader: &mut R) -> Result { 42 | let request = HelloRequest { 43 | client_name: reader.read_len_encode_bytes()?, 44 | client_version_major: reader.read_uvarint()?, 45 | client_version_minor: reader.read_uvarint()?, 46 | client_revision: reader.read_uvarint()?, 47 | default_database: reader.read_len_encode_bytes()?, 48 | user: reader.read_string()?, 49 | password: reader.read_len_encode_bytes()?, 50 | 51 | client_version_patch: 0, 52 | }; 53 | 54 | if request.user.is_empty() { 55 | return Err(Error::Server(ServerError { 56 | name: "UNEXPECTED_PACKET_FROM_CLIENT".to_string(), 57 | code: error_codes::UNEXPECTED_PACKET_FROM_CLIENT, 58 | message: "Unexpected packet from client (no user in Hello package)".to_string(), 59 | stack_trace: "".to_string(), 60 | })); 61 | } 62 | 63 | Ok(request) 64 | } 65 | } 66 | 67 | pub struct HelloResponse { 68 | pub dbms_name: String, 69 | pub dbms_version_major: u64, 70 | pub dbms_version_minor: u64, 71 | pub dbms_tcp_protocol_version: u64, 72 | pub timezone: String, 73 | pub server_display_name: String, 74 | pub dbms_version_patch: u64, 75 | } 76 | 77 | impl HelloResponse { 78 | pub fn encode(&self, encoder: &mut Encoder, client_revision: u64) -> Result<()> { 79 | encoder.uvarint(SERVER_HELLO); 80 | 81 | encoder.string(&self.dbms_name); 82 | encoder.uvarint(self.dbms_version_major); 83 | encoder.uvarint(self.dbms_version_minor); 84 | encoder.uvarint(self.dbms_tcp_protocol_version); 85 | 86 | if client_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE { 87 | encoder.string(&self.timezone); 88 | } 89 | 90 | if client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME { 91 | encoder.string(&self.server_display_name); 92 | } 93 | 94 | if client_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH { 95 | encoder.uvarint(self.dbms_version_patch); 96 | } 97 | 98 | Ok(()) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /clickhouse/src/protocols/protocol_query.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Read; 16 | 17 | use super::*; 18 | use crate::binary::ReadEx; 19 | use crate::errors::DriverError::UnknownSetting; 20 | use crate::errors::Error; 21 | use crate::errors::Result; 22 | 23 | const TCP: u8 = 1; 24 | const HTTP: u8 = 2; 25 | 26 | #[derive(Default, Debug)] 27 | pub struct QueryClientInfo { 28 | pub query_kind: u8, 29 | pub initial_user: String, 30 | pub initial_query_id: Vec, 31 | 32 | pub initial_address: Vec, 33 | pub interface: u8, 34 | 35 | // TCP 36 | pub os_user: Vec, 37 | pub client_hostname: Vec, 38 | pub client_name: Vec, 39 | 40 | pub client_version_major: u64, 41 | pub client_version_minor: u64, 42 | pub client_version_patch: u64, 43 | pub client_revision: u64, 44 | 45 | // HTTP 46 | pub http_method: u8, 47 | pub http_user_agent: Vec, 48 | 49 | pub quota_key: Vec, 50 | } 51 | 52 | impl QueryClientInfo { 53 | pub fn read_from(reader: &mut R) -> Result { 54 | let mut client_info = QueryClientInfo { 55 | query_kind: reader.read_scalar()?, 56 | ..Default::default() 57 | }; 58 | 59 | if client_info.query_kind == 0 { 60 | return Ok(client_info); 61 | } 62 | 63 | client_info.initial_user = reader.read_string()?; 64 | client_info.initial_query_id = reader.read_len_encode_bytes()?; 65 | client_info.initial_address = reader.read_len_encode_bytes()?; 66 | 67 | client_info.interface = reader.read_scalar()?; 68 | 69 | match client_info.interface { 70 | TCP => { 71 | client_info.os_user = reader.read_len_encode_bytes()?; 72 | client_info.client_hostname = reader.read_len_encode_bytes()?; 73 | client_info.client_name = reader.read_len_encode_bytes()?; 74 | 75 | client_info.client_version_major = reader.read_uvarint()?; 76 | client_info.client_version_minor = reader.read_uvarint()?; 77 | let client_revision = reader.read_uvarint()?; 78 | 79 | client_info.client_revision = client_revision; 80 | client_info.client_version_patch = client_revision; 81 | } 82 | HTTP => { 83 | client_info.http_method = reader.read_scalar()?; 84 | client_info.http_user_agent = reader.read_len_encode_bytes()?; 85 | } 86 | _ => {} 87 | } 88 | 89 | if client_info.client_revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO { 90 | client_info.quota_key = reader.read_len_encode_bytes()?; 91 | } 92 | 93 | if client_info.interface == TCP 94 | && client_info.client_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH 95 | { 96 | client_info.client_version_patch = reader.read_uvarint()?; 97 | } 98 | 99 | // TODO 100 | // if client_info.client_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY { 101 | // let trace_id: u8 = reader.read_scalar()?; 102 | // if trace_id > 0 { 103 | // } 104 | // } 105 | 106 | Ok(client_info) 107 | } 108 | } 109 | 110 | #[allow(dead_code)] 111 | #[derive(Default, Debug)] 112 | pub struct QueryRequest { 113 | pub(crate) query_id: String, 114 | pub(crate) client_info: QueryClientInfo, 115 | pub(crate) stage: u64, 116 | pub(crate) compression: u64, 117 | pub(crate) query: String, 118 | } 119 | 120 | impl QueryRequest { 121 | pub fn read_from( 122 | reader: &mut R, 123 | hello_request: &HelloRequest, 124 | ) -> Result { 125 | let query_id = reader.read_string()?; 126 | 127 | let mut client_info = Default::default(); 128 | if hello_request.client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO { 129 | client_info = QueryClientInfo::read_from(reader)?; 130 | } 131 | 132 | if client_info.query_kind == 0 { 133 | client_info.query_kind = INITIAL_QUERY; 134 | client_info.client_name = hello_request.client_name.clone(); 135 | client_info.client_version_major = hello_request.client_version_major; 136 | client_info.client_version_minor = hello_request.client_version_minor; 137 | client_info.client_version_patch = hello_request.client_version_patch; 138 | client_info.client_revision = hello_request.client_revision; 139 | } 140 | 141 | client_info.interface = TCP; 142 | 143 | loop { 144 | let name = reader.read_string()?; 145 | 146 | if name.is_empty() { 147 | break; 148 | } 149 | 150 | match name.as_str() { 151 | "max_block_size" | "max_threads" | "readonly" => { 152 | let _ = reader.read_uvarint()?; 153 | } 154 | _ => { 155 | return Err(Error::Driver(UnknownSetting { name })); 156 | } 157 | } 158 | } 159 | 160 | let query_protocol = QueryRequest { 161 | query_id, 162 | client_info, 163 | stage: reader.read_uvarint()?, 164 | compression: reader.read_uvarint()?, 165 | query: reader.read_string()?, 166 | }; 167 | 168 | Ok(query_protocol) 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /clickhouse/src/protocols/protocol_type.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Name, version, revision, default DB 16 | pub const CLIENT_HELLO: u64 = 0; 17 | // Query id, query settings, stage up to which the query must be executed, 18 | // whether the compression must be used, 19 | // query text (without data for INSERTs). 20 | // A block of data (compressed or not). 21 | pub const CLIENT_QUERY: u64 = 1; 22 | // A block of data (compressed or not). 23 | pub const CLIENT_DATA: u64 = 2; 24 | // Cancel the query execution. 25 | pub const CLIENT_CANCEL: u64 = 3; 26 | // Check that connection to the server is alive. 27 | pub const CLIENT_PING: u64 = 4; 28 | // Check status of tables on the server 29 | pub const CLIENT_TABLES_STATUS_REQUEST: u64 = 5; 30 | // Keep the connection alive 31 | pub const CLIENT_KEEP_ALIVE: u64 = 6; 32 | // A block of data (compressed or not) 33 | pub const CLIENT_SCALAR: u64 = 7; 34 | // List of unique parts ids to exclude from query processing 35 | pub const CLIENT_INGORED_PART_UUIDS: u64 = 8; 36 | 37 | pub const SERVER_HELLO: u64 = 0; 38 | pub const SERVER_DATA: u64 = 1; 39 | pub const SERVER_EXCEPTION: u64 = 2; 40 | pub const SERVER_PROGRESS: u64 = 3; 41 | pub const SERVER_PONG: u64 = 4; 42 | pub const SERVER_END_OF_STREAM: u64 = 5; 43 | pub const SERVER_PROFILE_INFO: u64 = 6; 44 | pub const SERVER_TOTALS: u64 = 7; 45 | pub const SERVER_EXTREMES: u64 = 8; 46 | 47 | pub const NO_QUERY: u8 = 0; 48 | pub const INITIAL_QUERY: u8 = 1; 49 | pub const SECONDARY_QUERY: u8 = 2; 50 | -------------------------------------------------------------------------------- /clickhouse/src/types/block/block_info.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::binary::Encoder; 16 | use crate::binary::ReadEx; 17 | use crate::errors::Result; 18 | 19 | #[allow(dead_code)] 20 | #[derive(Copy, Clone)] 21 | pub struct BlockInfo { 22 | num1: u64, 23 | is_overflows: bool, 24 | num2: u64, 25 | bucket_num: i32, 26 | num3: u64, 27 | } 28 | 29 | impl Default for BlockInfo { 30 | fn default() -> Self { 31 | Self { 32 | num1: 0, 33 | is_overflows: false, 34 | num2: 0, 35 | bucket_num: -1, 36 | num3: 0, 37 | } 38 | } 39 | } 40 | 41 | impl BlockInfo { 42 | pub(crate) fn read(reader: &mut R) -> Result { 43 | let block_info = Self { 44 | num1: reader.read_uvarint()?, 45 | is_overflows: reader.read_scalar()?, 46 | num2: reader.read_uvarint()?, 47 | bucket_num: reader.read_scalar()?, 48 | num3: reader.read_uvarint()?, 49 | }; 50 | Ok(block_info) 51 | } 52 | 53 | pub fn write(&self, encoder: &mut Encoder) { 54 | encoder.uvarint(1); 55 | encoder.write(self.is_overflows); 56 | encoder.uvarint(2); 57 | encoder.write(self.bucket_num); 58 | encoder.uvarint(0); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /clickhouse/src/types/block/builder.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::borrow::Cow; 16 | use std::marker; 17 | 18 | use chrono_tz::Tz; 19 | 20 | use crate::errors::Error; 21 | use crate::errors::FromSqlError; 22 | use crate::errors::Result; 23 | use crate::types::block::ColumnIdx; 24 | use crate::types::column::ArcColumnWrapper; 25 | use crate::types::column::ColumnData; 26 | use crate::types::column::Either; 27 | use crate::types::Column; 28 | use crate::types::ColumnType; 29 | use crate::types::SqlType; 30 | use crate::types::Value; 31 | use crate::Block; 32 | 33 | pub trait RowBuilder { 34 | fn apply(self, block: &mut Block) -> Result<()>; 35 | } 36 | 37 | pub struct RNil; 38 | 39 | pub struct RCons 40 | where 41 | T: RowBuilder, 42 | { 43 | key: Cow<'static, str>, 44 | value: Value, 45 | tail: T, 46 | } 47 | 48 | impl RNil { 49 | pub fn put(self, key: Cow<'static, str>, value: Value) -> RCons { 50 | RCons { 51 | key, 52 | value, 53 | tail: RNil, 54 | } 55 | } 56 | } 57 | 58 | impl RCons 59 | where 60 | T: RowBuilder, 61 | { 62 | pub fn put(self, key: Cow<'static, str>, value: Value) -> RCons { 63 | RCons { 64 | key, 65 | value, 66 | tail: self, 67 | } 68 | } 69 | } 70 | 71 | impl RowBuilder for RNil { 72 | #[inline(always)] 73 | fn apply(self, _block: &mut Block) -> Result<()> { 74 | Ok(()) 75 | } 76 | } 77 | 78 | impl RowBuilder for RCons 79 | where 80 | T: RowBuilder, 81 | { 82 | #[inline(always)] 83 | fn apply(self, block: &mut Block) -> Result<()> { 84 | put_param(self.key, self.value, block)?; 85 | self.tail.apply(block) 86 | } 87 | } 88 | 89 | impl RowBuilder for Vec<(String, Value)> { 90 | fn apply(self, block: &mut Block) -> Result<()> { 91 | for (k, v) in self { 92 | put_param(k.into(), v, block)?; 93 | } 94 | Ok(()) 95 | } 96 | } 97 | 98 | fn put_param( 99 | key: Cow<'static, str>, 100 | value: Value, 101 | block: &mut Block, 102 | ) -> Result<()> { 103 | let col_index = match key.as_ref().get_index(&block.columns) { 104 | Ok(col_index) => col_index, 105 | Err(Error::FromSql(FromSqlError::OutOfRange)) => { 106 | if block.row_count() <= 1 { 107 | let sql_type: SqlType = From::from(value.clone()); 108 | 109 | let timezone = extract_timezone(&value); 110 | 111 | let column = Column { 112 | name: key.clone().into(), 113 | data: ::from_type::( 114 | sql_type, 115 | timezone, 116 | block.capacity, 117 | )?, 118 | _marker: marker::PhantomData, 119 | }; 120 | 121 | block.columns.push(column); 122 | return put_param(key, value, block); 123 | } else { 124 | return Err(Error::FromSql(FromSqlError::OutOfRange)); 125 | } 126 | } 127 | Err(err) => return Err(err), 128 | }; 129 | 130 | block.columns[col_index].push(value); 131 | Ok(()) 132 | } 133 | 134 | fn extract_timezone(value: &Value) -> Tz { 135 | match value { 136 | Value::DateTime(_, tz) => *tz, 137 | Value::Nullable(Either::Right(d)) => extract_timezone(d), 138 | Value::Array(_, data) => { 139 | if let Some(v) = data.first() { 140 | extract_timezone(v) 141 | } else { 142 | Tz::Zulu 143 | } 144 | } 145 | _ => Tz::Zulu, 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /clickhouse/src/types/block/chunk_iterator.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cmp; 16 | 17 | use crate::types::Block; 18 | use crate::types::ColumnType; 19 | 20 | pub struct ChunkIterator<'a, K: ColumnType> { 21 | position: usize, 22 | size: usize, 23 | block: &'a Block, 24 | } 25 | 26 | impl<'a, K: ColumnType> Iterator for ChunkIterator<'a, K> { 27 | type Item = Block; 28 | 29 | fn next(&mut self) -> Option { 30 | let m = self.block.row_count(); 31 | 32 | if m == 0 && self.position == 0 { 33 | self.position += 1; 34 | let mut result = Block::default(); 35 | 36 | for column in self.block.columns().iter() { 37 | let data = column.slice(0..0); 38 | result = result.column(column.name(), data); 39 | } 40 | 41 | return Some(result); 42 | } 43 | 44 | if self.position >= m { 45 | return None; 46 | } 47 | 48 | let mut result = Block::new(); 49 | let size = cmp::min(self.size, m - self.position); 50 | 51 | for column in self.block.columns().iter() { 52 | let range = self.position..self.position + size; 53 | let data = column.slice(range); 54 | result = result.column(column.name(), data); 55 | } 56 | 57 | self.position += size; 58 | Some(result) 59 | } 60 | } 61 | 62 | impl<'a, K: ColumnType> ChunkIterator<'a, K> { 63 | pub fn new(size: usize, block: &Block) -> ChunkIterator { 64 | ChunkIterator { 65 | position: 0, 66 | size, 67 | block, 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /clickhouse/src/types/block/compressed.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io; 16 | use std::io::Read; 17 | use std::mem; 18 | use std::os::raw::c_char; 19 | use std::os::raw::c_int; 20 | 21 | use byteorder::LittleEndian; 22 | use byteorder::WriteBytesExt; 23 | use lz4::liblz4::LZ4_decompress_safe; 24 | use naive_cityhash::cityhash128; 25 | use naive_cityhash::U128; 26 | 27 | use crate::binary::ReadEx; 28 | use crate::errors::Error; 29 | use crate::errors::Result; 30 | 31 | const DBMS_MAX_COMPRESSED_SIZE: u32 = 0x4000_0000; // 1GB 32 | 33 | pub(crate) struct CompressedReader<'a, R> { 34 | reader: &'a mut R, 35 | cursor: io::Cursor>, 36 | } 37 | 38 | pub(crate) fn make(reader: &mut R) -> CompressedReader { 39 | CompressedReader { 40 | reader, 41 | cursor: io::Cursor::new(Vec::new()), 42 | } 43 | } 44 | 45 | impl<'a, R> CompressedReader<'a, R> 46 | where 47 | R: Read + ReadEx, 48 | { 49 | fn is_empty(&self) -> bool { 50 | let len = self.cursor.get_ref().len(); 51 | let pos = self.cursor.position() as usize; 52 | len == pos 53 | } 54 | 55 | fn fill(&mut self) -> Result<()> { 56 | let cursor = mem::replace(&mut self.cursor, io::Cursor::new(Vec::new())); 57 | let buffer = cursor.into_inner(); 58 | 59 | let tmp = decompress_buffer(&mut self.reader, buffer)?; 60 | self.cursor = io::Cursor::new(tmp); 61 | Ok(()) 62 | } 63 | } 64 | 65 | impl<'a, R> Read for CompressedReader<'a, R> 66 | where 67 | R: Read + ReadEx, 68 | { 69 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 70 | if self.is_empty() { 71 | self.fill()?; 72 | } 73 | 74 | self.cursor.read(buf) 75 | } 76 | } 77 | 78 | pub fn decompress_buffer(reader: &mut R, mut buffer: Vec) -> Result> 79 | where 80 | R: ReadEx, 81 | { 82 | let h = U128 { 83 | lo: reader.read_scalar()?, 84 | hi: reader.read_scalar()?, 85 | }; 86 | 87 | let method: u8 = reader.read_scalar()?; 88 | if method != 0x82 { 89 | let message: String = format!("unsupported compression method {}", method); 90 | return Err(raise_error(message)); 91 | } 92 | 93 | let compressed: u32 = reader.read_scalar()?; 94 | let original: u32 = reader.read_scalar()?; 95 | 96 | if compressed > DBMS_MAX_COMPRESSED_SIZE { 97 | return Err(raise_error("compressed data too big".to_string())); 98 | } 99 | 100 | buffer.resize(compressed as usize, 0_u8); 101 | { 102 | let mut cursor = io::Cursor::new(&mut buffer); 103 | cursor.write_u8(0x82)?; 104 | cursor.write_u32::(compressed)?; 105 | cursor.write_u32::(original)?; 106 | } 107 | reader.read_bytes(&mut buffer[9..])?; 108 | 109 | if h != cityhash128(&buffer) { 110 | return Err(raise_error("data was corrupted".to_string())); 111 | } 112 | 113 | let data = vec![0_u8; original as usize]; 114 | let status = unsafe { 115 | LZ4_decompress_safe( 116 | (buffer.as_mut_ptr() as *const c_char).add(9), 117 | data.as_ptr() as *mut c_char, 118 | (compressed - 9) as c_int, 119 | original as c_int, 120 | ) 121 | }; 122 | 123 | if status < 0 { 124 | return Err(raise_error("can't decompress data".to_string())); 125 | } 126 | 127 | Ok(data) 128 | } 129 | 130 | fn raise_error(message: String) -> Error { 131 | message.into() 132 | } 133 | -------------------------------------------------------------------------------- /clickhouse/src/types/block/row.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::marker; 16 | use std::sync::Arc; 17 | 18 | use crate::errors::Result; 19 | use crate::types::block::ColumnIdx; 20 | use crate::types::Block; 21 | use crate::types::Column; 22 | use crate::types::ColumnType; 23 | use crate::types::FromSql; 24 | use crate::types::SqlType; 25 | 26 | /// A row from Clickhouse 27 | pub struct Row<'a, K: ColumnType> { 28 | pub(crate) row: usize, 29 | pub(crate) block_ref: BlockRef<'a, K>, 30 | pub(crate) kind: marker::PhantomData, 31 | } 32 | 33 | impl<'a, K: ColumnType> Row<'a, K> { 34 | /// Get the value of a particular cell of the row. 35 | pub fn get(&'a self, col: I) -> Result 36 | where 37 | T: FromSql<'a>, 38 | I: ColumnIdx + Copy, 39 | { 40 | self.block_ref.get(self.row, col) 41 | } 42 | 43 | /// Return the number of cells in the current row. 44 | pub fn len(&self) -> usize { 45 | self.block_ref.column_count() 46 | } 47 | 48 | /// Returns `true` if the row contains no cells. 49 | pub fn is_empty(&self) -> bool { 50 | self.len() == 0 51 | } 52 | 53 | /// Get the name of a particular cell of the row. 54 | pub fn name(&self, col: I) -> Result<&str> { 55 | Ok(self.block_ref.get_column(col)?.name()) 56 | } 57 | 58 | /// Get the type of a particular cell of the row. 59 | pub fn sql_type(&self, col: I) -> Result { 60 | Ok(self.block_ref.get_column(col)?.sql_type()) 61 | } 62 | } 63 | 64 | #[allow(dead_code)] 65 | pub(crate) enum BlockRef<'a, K: ColumnType> { 66 | Borrowed(&'a Block), 67 | Owned(Arc>), 68 | } 69 | 70 | impl<'a, K: ColumnType> Clone for BlockRef<'a, K> { 71 | fn clone(&self) -> Self { 72 | match self { 73 | BlockRef::Borrowed(block_ref) => BlockRef::Borrowed(*block_ref), 74 | BlockRef::Owned(block_ref) => BlockRef::Owned(block_ref.clone()), 75 | } 76 | } 77 | } 78 | 79 | impl<'a, K: ColumnType> BlockRef<'a, K> { 80 | fn row_count(&self) -> usize { 81 | match self { 82 | BlockRef::Borrowed(block) => block.row_count(), 83 | BlockRef::Owned(block) => block.row_count(), 84 | } 85 | } 86 | 87 | fn column_count(&self) -> usize { 88 | match self { 89 | BlockRef::Borrowed(block) => block.column_count(), 90 | BlockRef::Owned(block) => block.column_count(), 91 | } 92 | } 93 | 94 | fn get<'s, T, I>(&'s self, row: usize, col: I) -> Result 95 | where 96 | T: FromSql<'s>, 97 | I: ColumnIdx + Copy, 98 | { 99 | match self { 100 | BlockRef::Borrowed(block) => block.get(row, col), 101 | BlockRef::Owned(block) => block.get(row, col), 102 | } 103 | } 104 | 105 | fn get_column(&self, col: I) -> Result<&Column> { 106 | match self { 107 | BlockRef::Borrowed(block) => { 108 | let column_index = col.get_index(block.columns())?; 109 | Ok(&block.columns[column_index]) 110 | } 111 | BlockRef::Owned(block) => { 112 | let column_index = col.get_index(block.columns())?; 113 | Ok(&block.columns[column_index]) 114 | } 115 | } 116 | } 117 | } 118 | 119 | /// Immutable rows iterator 120 | pub struct Rows<'a, K: ColumnType> { 121 | pub(crate) row: usize, 122 | pub(crate) block_ref: BlockRef<'a, K>, 123 | pub(crate) kind: marker::PhantomData, 124 | } 125 | 126 | impl<'a, K: ColumnType> Iterator for Rows<'a, K> { 127 | type Item = Row<'a, K>; 128 | 129 | fn next(&mut self) -> Option { 130 | if self.row >= self.block_ref.row_count() { 131 | return None; 132 | } 133 | let result = Some(Row { 134 | row: self.row, 135 | block_ref: self.block_ref.clone(), 136 | kind: marker::PhantomData, 137 | }); 138 | self.row += 1; 139 | result 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/array.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use chrono_tz::Tz; 18 | 19 | use crate::binary::Encoder; 20 | use crate::binary::ReadEx; 21 | use crate::errors::Result; 22 | use crate::types::column::column_data::ArcColumnData; 23 | use crate::types::column::column_data::BoxColumnData; 24 | use crate::types::column::list::List; 25 | use crate::types::column::ArcColumnWrapper; 26 | use crate::types::column::ColumnData; 27 | use crate::types::SqlType; 28 | use crate::types::Value; 29 | use crate::types::ValueRef; 30 | 31 | pub struct ArrayColumnData { 32 | pub(crate) inner: ArcColumnData, 33 | pub(crate) offsets: List, 34 | } 35 | 36 | impl ArrayColumnData { 37 | #[allow(dead_code)] 38 | pub fn create(inner: ArcColumnData, offsets: List) -> Self { 39 | ArrayColumnData { inner, offsets } 40 | } 41 | 42 | pub(crate) fn load( 43 | reader: &mut R, 44 | type_name: &str, 45 | rows: usize, 46 | tz: Tz, 47 | ) -> Result { 48 | let mut offsets = List::with_capacity(rows); 49 | offsets.resize(rows, 0_u64); 50 | reader.read_bytes(offsets.as_mut())?; 51 | 52 | let size = match rows { 53 | 0 => 0, 54 | _ => offsets.at(rows - 1) as usize, 55 | }; 56 | let inner = 57 | ::load_data::(reader, type_name, size, tz)?; 58 | 59 | Ok(ArrayColumnData { inner, offsets }) 60 | } 61 | } 62 | 63 | impl ColumnData for ArrayColumnData { 64 | fn sql_type(&self) -> SqlType { 65 | let inner_type = self.inner.sql_type(); 66 | SqlType::Array(inner_type.into()) 67 | } 68 | 69 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize) { 70 | let mut offset = 0_u64; 71 | 72 | for i in start..end { 73 | offset = self.offsets.at(i); 74 | encoder.write(offset); 75 | } 76 | 77 | self.inner.save(encoder, 0, offset as usize); 78 | } 79 | 80 | fn len(&self) -> usize { 81 | self.offsets.len() 82 | } 83 | 84 | fn push(&mut self, value: Value) { 85 | if let Value::Array(_, vs) = value { 86 | let offsets_len = self.offsets.len(); 87 | let prev = if offsets_len == 0 { 88 | 0_usize 89 | } else { 90 | self.offsets.at(offsets_len - 1) as usize 91 | }; 92 | 93 | let inner_column = Arc::get_mut(&mut self.inner).unwrap(); 94 | self.offsets.push((prev + vs.len()) as u64); 95 | for v in vs.iter() { 96 | inner_column.push(v.clone()); 97 | } 98 | } else { 99 | panic!("value should be an array") 100 | } 101 | } 102 | 103 | fn at(&self, index: usize) -> ValueRef { 104 | let sql_type = self.inner.sql_type(); 105 | 106 | let start = if index > 0 { 107 | self.offsets.at(index - 1) as usize 108 | } else { 109 | 0_usize 110 | }; 111 | let end = self.offsets.at(index) as usize; 112 | let mut vs = Vec::with_capacity(end); 113 | for i in start..end { 114 | let v = self.inner.at(i); 115 | vs.push(v); 116 | } 117 | ValueRef::Array(sql_type.into(), Arc::new(vs)) 118 | } 119 | 120 | fn clone_instance(&self) -> BoxColumnData { 121 | Box::new(Self { 122 | inner: self.inner.clone(), 123 | offsets: self.offsets.clone(), 124 | }) 125 | } 126 | 127 | unsafe fn get_internal(&self, pointers: &[*mut *const u8], level: u8) -> Result<()> { 128 | if level == self.sql_type().level() { 129 | *pointers[0] = self.offsets.as_ptr() as *const u8; 130 | *(pointers[1] as *mut usize) = self.offsets.len(); 131 | Ok(()) 132 | } else { 133 | self.inner.get_internal(pointers, level) 134 | } 135 | } 136 | 137 | fn cast_to(&self, _this: &ArcColumnData, target: &SqlType) -> Option { 138 | if let SqlType::Array(inner_target) = target { 139 | if let Some(inner) = self.inner.cast_to(&self.inner, inner_target) { 140 | return Some(Arc::new(ArrayColumnData { 141 | inner, 142 | offsets: self.offsets.clone(), 143 | })); 144 | } 145 | } 146 | None 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/chunk.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::cmp; 16 | use std::ops; 17 | 18 | use super::ColumnData; 19 | use crate::binary::Encoder; 20 | use crate::types::column::column_data::ArcColumnData; 21 | use crate::types::column::column_data::BoxColumnData; 22 | use crate::types::SqlType; 23 | use crate::types::Value; 24 | use crate::types::ValueRef; 25 | 26 | pub struct ChunkColumnData { 27 | data: ArcColumnData, 28 | range: ops::Range, 29 | } 30 | 31 | impl ChunkColumnData { 32 | pub(crate) fn new(data: ArcColumnData, range: ops::Range) -> Self { 33 | Self { data, range } 34 | } 35 | } 36 | 37 | impl ColumnData for ChunkColumnData { 38 | fn sql_type(&self) -> SqlType { 39 | self.data.sql_type() 40 | } 41 | 42 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize) { 43 | self.data.save( 44 | encoder, 45 | self.range.start + start, 46 | cmp::min(self.range.end, self.range.start + end), 47 | ) 48 | } 49 | 50 | fn len(&self) -> usize { 51 | self.range.len() 52 | } 53 | 54 | fn push(&mut self, _value: Value) { 55 | unimplemented!() 56 | } 57 | 58 | fn at(&self, index: usize) -> ValueRef { 59 | if index >= self.range.len() { 60 | panic!("out of range"); 61 | } 62 | 63 | self.data.at(index + self.range.start) 64 | } 65 | 66 | fn clone_instance(&self) -> BoxColumnData { 67 | unimplemented!() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/column_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::convert; 16 | use std::sync::Arc; 17 | 18 | use crate::binary::Encoder; 19 | use crate::errors::Error; 20 | use crate::errors::FromSqlError; 21 | use crate::errors::Result; 22 | use crate::types::SqlType; 23 | use crate::types::Value; 24 | use crate::types::ValueRef; 25 | 26 | pub type ArcColumnData = Arc; 27 | 28 | pub type BoxColumnData = Box; 29 | 30 | pub trait ColumnData { 31 | fn sql_type(&self) -> SqlType; 32 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize); 33 | fn len(&self) -> usize; 34 | fn is_empty(&self) -> bool { 35 | self.len() == 0 36 | } 37 | fn push(&mut self, value: Value); 38 | fn at(&self, index: usize) -> ValueRef; 39 | 40 | fn clone_instance(&self) -> BoxColumnData; 41 | 42 | #[allow(clippy::missing_safety_doc)] 43 | unsafe fn get_internal(&self, _pointers: &[*mut *const u8], _level: u8) -> Result<()> { 44 | Err(Error::FromSql(FromSqlError::UnsupportedOperation)) 45 | } 46 | 47 | fn cast_to(&self, _this: &ArcColumnData, _target: &SqlType) -> Option { 48 | None 49 | } 50 | } 51 | 52 | pub trait ColumnDataExt { 53 | fn append>(&mut self, value: T); 54 | } 55 | 56 | impl ColumnDataExt for C { 57 | fn append>(&mut self, value: T) { 58 | self.push(value.into()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/concat.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::iter; 16 | 17 | use super::column_data::ArcColumnData; 18 | use super::column_data::BoxColumnData; 19 | use super::column_data::ColumnData; 20 | use crate::binary::Encoder; 21 | use crate::errors::Error; 22 | use crate::errors::FromSqlError; 23 | use crate::errors::Result; 24 | use crate::types::SqlType; 25 | use crate::types::Value; 26 | use crate::types::ValueRef; 27 | 28 | pub struct ConcatColumnData { 29 | data: Vec, 30 | index: Vec, 31 | } 32 | 33 | impl ConcatColumnData { 34 | pub fn concat(data: Vec) -> Self { 35 | Self::check_columns(&data); 36 | 37 | let index = build_index(data.iter().map(|x| x.len())); 38 | Self { data, index } 39 | } 40 | 41 | fn check_columns(data: &[ArcColumnData]) { 42 | match data.first() { 43 | None => panic!("data should not be empty."), 44 | Some(first) => { 45 | for column in data.iter().skip(1) { 46 | if first.sql_type() != column.sql_type() { 47 | panic!( 48 | "all columns should have the same type ({:?} != {:?}).", 49 | first.sql_type(), 50 | column.sql_type() 51 | ); 52 | } 53 | } 54 | } 55 | } 56 | } 57 | } 58 | 59 | impl ColumnData for ConcatColumnData { 60 | fn sql_type(&self) -> SqlType { 61 | self.data[0].sql_type() 62 | } 63 | 64 | fn save(&self, _: &mut Encoder, _: usize, _: usize) { 65 | unimplemented!() 66 | } 67 | 68 | fn len(&self) -> usize { 69 | *self.index.last().unwrap() 70 | } 71 | 72 | fn push(&mut self, _value: Value) { 73 | unimplemented!() 74 | } 75 | 76 | fn at(&self, index: usize) -> ValueRef { 77 | let chunk_index = find_chunk(&self.index, index); 78 | let chunk = &self.data[chunk_index]; 79 | chunk.at(index - self.index[chunk_index]) 80 | } 81 | 82 | fn clone_instance(&self) -> BoxColumnData { 83 | unimplemented!() 84 | } 85 | 86 | unsafe fn get_internal(&self, pointers: &[*mut *const u8], level: u8) -> Result<()> { 87 | if level == 0xff { 88 | *pointers[0] = &self.data as *const Vec as *mut u8; 89 | Ok(()) 90 | } else { 91 | Err(Error::FromSql(FromSqlError::UnsupportedOperation)) 92 | } 93 | } 94 | } 95 | 96 | pub fn build_index<'a, I>(sizes: I) -> Vec 97 | where 98 | I: iter::Iterator + 'a, 99 | { 100 | let mut acc = 0; 101 | let mut index = vec![acc]; 102 | 103 | for size in sizes { 104 | acc += size; 105 | index.push(acc); 106 | } 107 | 108 | index 109 | } 110 | 111 | pub fn find_chunk(index: &[usize], ix: usize) -> usize { 112 | let mut lo = 0_usize; 113 | let mut hi = index.len() - 1; 114 | 115 | while lo < hi { 116 | let mid = lo + (hi - lo) / 2; 117 | 118 | if index[lo] == index[lo + 1] { 119 | lo += 1; 120 | continue; 121 | } 122 | 123 | if ix < index[mid] { 124 | hi = mid; 125 | } else if ix >= index[mid + 1] { 126 | lo = mid + 1; 127 | } else { 128 | return mid; 129 | } 130 | } 131 | 132 | 0 133 | } 134 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/datetime64.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use chrono::prelude::*; 16 | use chrono_tz::Tz; 17 | 18 | use crate::binary::Encoder; 19 | use crate::binary::ReadEx; 20 | use crate::errors::Result; 21 | use crate::types::column::column_data::BoxColumnData; 22 | use crate::types::column::column_data::ColumnData; 23 | use crate::types::column::list::List; 24 | use crate::types::DateTimeType; 25 | use crate::types::SqlType; 26 | use crate::types::Value; 27 | use crate::types::ValueRef; 28 | 29 | pub struct DateTime64ColumnData { 30 | data: List, 31 | params: (u32, Tz), 32 | } 33 | 34 | impl DateTime64ColumnData { 35 | pub(crate) fn load( 36 | reader: &mut R, 37 | size: usize, 38 | precision: u32, 39 | tz: Tz, 40 | ) -> Result { 41 | let mut data = List::with_capacity(size); 42 | unsafe { 43 | data.set_len(size); 44 | } 45 | reader.read_bytes(data.as_mut())?; 46 | Ok(DateTime64ColumnData { 47 | data, 48 | params: (precision, tz), 49 | }) 50 | } 51 | 52 | pub(crate) fn with_capacity(capacity: usize, precision: u32, timezone: Tz) -> Self { 53 | DateTime64ColumnData { 54 | data: List::with_capacity(capacity), 55 | params: (precision, timezone), 56 | } 57 | } 58 | } 59 | 60 | impl ColumnData for DateTime64ColumnData { 61 | fn sql_type(&self) -> SqlType { 62 | let (precision, tz) = self.params; 63 | SqlType::DateTime(DateTimeType::DateTime64(precision, tz)) 64 | } 65 | 66 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize) { 67 | for i in start..end { 68 | encoder.write(self.data.at(i)); 69 | } 70 | } 71 | 72 | fn len(&self) -> usize { 73 | self.data.len() 74 | } 75 | 76 | fn push(&mut self, value: Value) { 77 | let (precision, tz) = &self.params; 78 | let time = DateTime::::from(value); 79 | let stamp = from_datetime(time.with_timezone(tz), *precision); 80 | self.data.push(stamp) 81 | } 82 | 83 | fn at(&self, index: usize) -> ValueRef { 84 | let value = self.data.at(index); 85 | ValueRef::DateTime64(value, &self.params) 86 | } 87 | 88 | fn clone_instance(&self) -> BoxColumnData { 89 | Box::new(Self { 90 | data: self.data.clone(), 91 | params: self.params, 92 | }) 93 | } 94 | 95 | unsafe fn get_internal(&self, pointers: &[*mut *const u8], level: u8) -> Result<()> { 96 | assert_eq!(level, 0); 97 | let (precision, tz) = &self.params; 98 | *pointers[0] = self.data.as_ptr() as *const u8; 99 | *pointers[1] = tz as *const Tz as *const u8; 100 | *(pointers[2] as *mut usize) = self.len(); 101 | *(pointers[3] as *mut Option) = Some(*precision); 102 | Ok(()) 103 | } 104 | } 105 | 106 | pub fn from_datetime(time: DateTime, precision: u32) -> i64 { 107 | let base10: i64 = 10; 108 | let timestamp = time.timestamp_nanos_opt().unwrap(); 109 | timestamp / base10.pow(9 - precision) 110 | } 111 | 112 | #[inline(always)] 113 | pub fn to_datetime(value: i64, precision: u32, tz: Tz) -> DateTime { 114 | let base10: i64 = 10; 115 | 116 | let nano = if precision < 19 { 117 | value * base10.pow(9 - precision) 118 | } else { 119 | 0_i64 120 | }; 121 | 122 | let sec = nano / 1_000_000_000; 123 | let nsec = nano - sec * 1_000_000_000; 124 | 125 | tz.timestamp_opt(sec, nsec as u32).unwrap() 126 | } 127 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/list.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt; 16 | use std::mem; 17 | use std::slice; 18 | 19 | use micromarshal::Marshal; 20 | use micromarshal::Unmarshal; 21 | 22 | use crate::types::StatBuffer; 23 | 24 | #[derive(Clone)] 25 | pub struct List 26 | where 27 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static, 28 | { 29 | data: Vec, 30 | } 31 | 32 | impl List 33 | where 34 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static, 35 | { 36 | pub fn len(&self) -> usize { 37 | self.data.len() 38 | } 39 | 40 | pub fn is_empty(&self) -> bool { 41 | self.len() == 0 42 | } 43 | 44 | pub fn at(&self, index: usize) -> T { 45 | self.data[index] 46 | } 47 | 48 | pub fn push(&mut self, value: T) { 49 | self.data.push(value); 50 | } 51 | 52 | pub fn new() -> List { 53 | List { data: Vec::new() } 54 | } 55 | 56 | pub fn with_capacity(capacity: usize) -> List { 57 | Self { 58 | data: Vec::with_capacity(capacity), 59 | } 60 | } 61 | 62 | pub fn resize(&mut self, new_len: usize, value: T) { 63 | self.data.resize(new_len, value); 64 | } 65 | 66 | pub(super) unsafe fn set_len(&mut self, new_len: usize) { 67 | self.data.set_len(new_len); 68 | } 69 | 70 | pub(super) unsafe fn as_ptr(&self) -> *const T { 71 | self.data.as_ptr() 72 | } 73 | } 74 | 75 | impl Default for List 76 | where 77 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static, 78 | { 79 | fn default() -> Self { 80 | Self::new() 81 | } 82 | } 83 | 84 | impl fmt::Debug for List 85 | where 86 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static + fmt::Debug, 87 | { 88 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 89 | write!(f, "{:?}", self.data) 90 | } 91 | } 92 | 93 | impl AsRef<[u8]> for List 94 | where 95 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static, 96 | { 97 | fn as_ref(&self) -> &[u8] { 98 | let ptr = self.data.as_ptr() as *const u8; 99 | let size = self.len() * mem::size_of::(); 100 | unsafe { slice::from_raw_parts(ptr, size) } 101 | } 102 | } 103 | 104 | impl AsMut<[u8]> for List 105 | where 106 | T: StatBuffer + Unmarshal + Marshal + Copy + Sync + 'static, 107 | { 108 | fn as_mut(&mut self) -> &mut [u8] { 109 | let ptr = self.data.as_mut_ptr() as *mut u8; 110 | let size = self.len() * mem::size_of::(); 111 | unsafe { slice::from_raw_parts_mut(ptr, size) } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/nullable.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use chrono_tz::Tz; 18 | 19 | use crate::binary::Encoder; 20 | use crate::binary::ReadEx; 21 | use crate::errors::Result; 22 | use crate::types::column::column_data::ArcColumnData; 23 | use crate::types::column::column_data::BoxColumnData; 24 | use crate::types::column::ArcColumnWrapper; 25 | use crate::types::column::ColumnData; 26 | use crate::types::column::Either; 27 | use crate::types::SqlType; 28 | use crate::types::Value; 29 | use crate::types::ValueRef; 30 | 31 | pub struct NullableColumnData { 32 | pub inner: ArcColumnData, 33 | pub nulls: Vec, 34 | } 35 | 36 | impl NullableColumnData { 37 | pub(crate) fn load( 38 | reader: &mut R, 39 | type_name: &str, 40 | size: usize, 41 | tz: Tz, 42 | ) -> Result { 43 | let mut nulls = vec![0; size]; 44 | reader.read_bytes(nulls.as_mut())?; 45 | let inner = 46 | ::load_data::(reader, type_name, size, tz)?; 47 | Ok(NullableColumnData { inner, nulls }) 48 | } 49 | } 50 | 51 | impl ColumnData for NullableColumnData { 52 | fn sql_type(&self) -> SqlType { 53 | let inner_type = self.inner.sql_type(); 54 | SqlType::Nullable(inner_type.into()) 55 | } 56 | 57 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize) { 58 | let nulls: &[u8] = self.nulls.as_ref(); 59 | encoder.write_bytes(&nulls[start..end]); 60 | self.inner.save(encoder, start, end); 61 | } 62 | 63 | fn len(&self) -> usize { 64 | assert_eq!(self.nulls.len(), self.inner.len()); 65 | self.inner.len() 66 | } 67 | 68 | fn push(&mut self, value: Value) { 69 | let inner_column: &mut dyn ColumnData = Arc::get_mut(&mut self.inner).unwrap(); 70 | 71 | if let Value::Nullable(e) = value { 72 | match e { 73 | Either::Left(sql_type) => { 74 | let default_value = Value::default(sql_type.clone()); 75 | inner_column.push(default_value); 76 | self.nulls.push(true as u8); 77 | } 78 | Either::Right(inner) => { 79 | inner_column.push(*inner); 80 | self.nulls.push(false as u8); 81 | } 82 | } 83 | } else { 84 | inner_column.push(value); 85 | self.nulls.push(false as u8); 86 | } 87 | } 88 | 89 | fn at(&self, index: usize) -> ValueRef { 90 | if self.nulls[index] == 1 { 91 | let sql_type = self.inner.sql_type(); 92 | ValueRef::Nullable(Either::Left(sql_type.into())) 93 | } else { 94 | let inner_value = self.inner.at(index); 95 | ValueRef::Nullable(Either::Right(Box::new(inner_value))) 96 | } 97 | } 98 | 99 | fn clone_instance(&self) -> BoxColumnData { 100 | Box::new(Self { 101 | inner: self.inner.clone(), 102 | nulls: self.nulls.clone(), 103 | }) 104 | } 105 | 106 | unsafe fn get_internal(&self, pointers: &[*mut *const u8], level: u8) -> Result<()> { 107 | if level == self.sql_type().level() { 108 | *pointers[0] = self.nulls.as_ptr(); 109 | *(pointers[1] as *mut usize) = self.len(); 110 | Ok(()) 111 | } else { 112 | self.inner.get_internal(pointers, level) 113 | } 114 | } 115 | 116 | fn cast_to(&self, _this: &ArcColumnData, target: &SqlType) -> Option { 117 | if let SqlType::Nullable(inner_target) = target { 118 | if let Some(inner) = self.inner.cast_to(&self.inner, inner_target) { 119 | return Some(Arc::new(NullableColumnData { 120 | inner, 121 | nulls: self.nulls.clone(), 122 | })); 123 | } 124 | } 125 | None 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/string_pool.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Write; 16 | use std::slice; 17 | 18 | const AVG_STR_SIZE: usize = 80; 19 | 20 | #[derive(Copy, Clone)] 21 | struct StringPtr { 22 | chunk: usize, 23 | shift: usize, 24 | len: usize, 25 | } 26 | 27 | #[derive(Clone)] 28 | pub struct StringPool { 29 | chunks: Vec>, 30 | pointers: Vec, 31 | position: usize, 32 | capacity: usize, 33 | } 34 | 35 | pub(crate) struct StringIter<'a> { 36 | pool: &'a StringPool, 37 | index: usize, 38 | } 39 | 40 | impl<'a> Iterator for StringIter<'a> { 41 | type Item = &'a [u8]; 42 | 43 | fn next(&mut self) -> Option { 44 | if self.index < self.pool.len() { 45 | let result = self.pool.get(self.index); 46 | self.index += 1; 47 | return Some(result); 48 | } 49 | 50 | None 51 | } 52 | } 53 | 54 | impl From> for StringPool 55 | where 56 | T: AsRef<[u8]>, 57 | { 58 | fn from(source: Vec) -> Self { 59 | let mut pool = StringPool::with_capacity(source.len()); 60 | for s in source.iter() { 61 | let mut b = pool.allocate(s.as_ref().len()); 62 | b.write_all(s.as_ref()).unwrap(); 63 | } 64 | pool 65 | } 66 | } 67 | 68 | impl StringPool { 69 | pub fn with_capacity(capacity: usize) -> StringPool { 70 | StringPool { 71 | pointers: Vec::with_capacity(capacity), 72 | chunks: Vec::new(), 73 | position: 0, 74 | capacity, 75 | } 76 | } 77 | 78 | pub fn allocate(&mut self, size: usize) -> &mut [u8] { 79 | if self.free_space() < size || self.chunks.is_empty() { 80 | self.reserve(size); 81 | return self.allocate(size); 82 | } 83 | 84 | self.try_allocate(size).unwrap() 85 | } 86 | 87 | fn free_space(&self) -> usize { 88 | if let Some(buffer) = self.chunks.last() { 89 | return buffer.len() - self.position; 90 | } 91 | 92 | 0 93 | } 94 | 95 | fn try_allocate(&mut self, size: usize) -> Option<&mut [u8]> { 96 | if !self.chunks.is_empty() { 97 | let chunk = self.chunks.len() - 1; 98 | 99 | let position = self.position; 100 | self.position += size; 101 | self.pointers.push(StringPtr { 102 | len: size, 103 | shift: position, 104 | chunk, 105 | }); 106 | 107 | let buffer = &mut self.chunks[chunk]; 108 | return Some(&mut buffer[position..position + size]); 109 | } 110 | 111 | None 112 | } 113 | 114 | fn reserve(&mut self, size: usize) { 115 | use std::cmp::max; 116 | self.position = 0; 117 | self.chunks 118 | .push(vec![0_u8; max(self.capacity * AVG_STR_SIZE, size)]); 119 | } 120 | 121 | #[inline(always)] 122 | pub fn get(&self, index: usize) -> &[u8] { 123 | let pointer = &self.pointers[index]; 124 | unsafe { self.get_by_pointer(pointer) } 125 | } 126 | 127 | #[inline(always)] 128 | pub(crate) unsafe fn get_unchecked(&self, index: usize) -> &[u8] { 129 | let pointer = self.pointers.get_unchecked(index); 130 | self.get_by_pointer(pointer) 131 | } 132 | 133 | #[inline(always)] 134 | unsafe fn get_by_pointer(&self, pointer: &StringPtr) -> &[u8] { 135 | let chunk = &self.chunks.get_unchecked(pointer.chunk); 136 | 137 | let ptr = chunk.as_ptr().add(pointer.shift); 138 | slice::from_raw_parts(ptr, pointer.len) 139 | } 140 | 141 | #[inline(always)] 142 | pub(crate) fn len(&self) -> usize { 143 | self.pointers.len() 144 | } 145 | 146 | pub(crate) fn strings(&self) -> StringIter { 147 | StringIter { 148 | pool: self, 149 | index: 0, 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /clickhouse/src/types/column/tuple.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use chrono_tz::Tz; 18 | 19 | use crate::binary::Encoder; 20 | use crate::binary::ReadEx; 21 | use crate::errors::Result; 22 | use crate::types::column::column_data::ArcColumnData; 23 | use crate::types::column::column_data::BoxColumnData; 24 | use crate::types::column::ArcColumnWrapper; 25 | use crate::types::column::ColumnData; 26 | use crate::types::column::ColumnFrom; 27 | use crate::types::column::ColumnWrapper; 28 | use crate::types::SqlType; 29 | use crate::types::Value; 30 | use crate::types::ValueRef; 31 | 32 | pub struct TupleColumnData { 33 | pub inner: Vec, 34 | } 35 | 36 | impl TupleColumnData { 37 | pub(crate) fn load( 38 | reader: &mut R, 39 | inner_types: Vec<&str>, 40 | rows: usize, 41 | tz: Tz, 42 | ) -> Result { 43 | let inner = inner_types 44 | .into_iter() 45 | .map(|type_name| { 46 | ::load_data::(reader, type_name, rows, tz) 47 | .unwrap() 48 | }) 49 | .collect(); 50 | Ok(Self { inner }) 51 | } 52 | } 53 | 54 | impl ColumnFrom for Vec { 55 | fn column_from(inner: Self) -> W::Wrapper { 56 | W::wrap(TupleColumnData { inner }) 57 | } 58 | } 59 | 60 | impl ColumnData for TupleColumnData { 61 | fn sql_type(&self) -> SqlType { 62 | SqlType::Tuple(self.inner.iter().map(|c| c.sql_type().into()).collect()) 63 | } 64 | 65 | fn save(&self, encoder: &mut Encoder, start: usize, end: usize) { 66 | self.inner.iter().for_each(|c| c.save(encoder, start, end)); 67 | } 68 | 69 | fn len(&self) -> usize { 70 | self.inner[0].len() 71 | } 72 | 73 | fn push(&mut self, value: Value) { 74 | if let Value::Tuple(vs) = value { 75 | vs.iter().zip(self.inner.iter_mut()).for_each(|(v, c)| { 76 | let inner_column = Arc::get_mut(c).unwrap(); 77 | inner_column.push(v.clone()); 78 | }); 79 | } else { 80 | panic!("value should be a tuple"); 81 | } 82 | } 83 | 84 | fn at(&self, index: usize) -> ValueRef { 85 | ValueRef::Tuple(Arc::new(self.inner.iter().map(|c| c.at(index)).collect())) 86 | } 87 | 88 | fn clone_instance(&self) -> BoxColumnData { 89 | Box::new(Self { 90 | inner: self.inner.clone(), 91 | }) 92 | } 93 | 94 | fn cast_to(&self, _this: &ArcColumnData, target: &SqlType) -> Option { 95 | if let SqlType::Tuple(target_types) = target { 96 | if target_types.len() == self.inner.len() { 97 | let mut new_inner = Vec::with_capacity(self.inner.len()); 98 | for (i, c) in self.inner.iter().enumerate() { 99 | let target_type = target_types[i]; 100 | let new_c = c.cast_to(c, target_type); 101 | if let Some(new_c) = new_c { 102 | new_inner.push(new_c); 103 | } else { 104 | return None; 105 | } 106 | } 107 | return Some(Arc::new(TupleColumnData { inner: new_inner })); 108 | } 109 | } 110 | None 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /clickhouse/src/types/date_converter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use chrono::prelude::*; 16 | use chrono_tz::Tz; 17 | 18 | use crate::types::DateTimeType; 19 | use crate::types::SqlType; 20 | use crate::types::Value; 21 | use crate::types::ValueRef; 22 | 23 | pub const UNIX_EPOCH_DAY: i64 = 719_163; 24 | 25 | pub trait DateConverter { 26 | fn to_date(&self, tz: Tz) -> ValueRef<'static>; 27 | fn get_stamp(source: Value) -> Self; 28 | fn date_type() -> SqlType; 29 | 30 | fn get_days(date: NaiveDate) -> u16 { 31 | let gregorian_day = i64::from(date.num_days_from_ce()); 32 | (gregorian_day - UNIX_EPOCH_DAY) as u16 33 | } 34 | } 35 | 36 | impl DateConverter for u16 { 37 | fn to_date(&self, _: Tz) -> ValueRef<'static> { 38 | ValueRef::Date(*self) 39 | } 40 | 41 | fn get_stamp(source: Value) -> Self { 42 | Self::get_days(NaiveDate::from(source)) 43 | } 44 | 45 | fn date_type() -> SqlType { 46 | SqlType::Date 47 | } 48 | } 49 | 50 | impl DateConverter for u32 { 51 | fn to_date(&self, tz: Tz) -> ValueRef<'static> { 52 | ValueRef::DateTime(*self, tz) 53 | } 54 | 55 | fn get_stamp(source: Value) -> Self { 56 | DateTime::::from(source).timestamp() as Self 57 | } 58 | 59 | fn date_type() -> SqlType { 60 | SqlType::DateTime(DateTimeType::DateTime32) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /clickhouse/src/types/enums.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt; 16 | 17 | // TODO Using strings as a keys 18 | #[derive(Clone, Copy, Default)] 19 | pub struct Enum8(pub(crate) i8); 20 | 21 | #[derive(Clone, Copy, Default)] 22 | pub struct Enum16(pub(crate) i16); 23 | 24 | impl PartialEq for Enum16 { 25 | fn eq(&self, other: &Self) -> bool { 26 | self.0 == other.0 27 | } 28 | } 29 | 30 | impl fmt::Display for Enum16 { 31 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 32 | write!(f, "Enum({})", self.0) 33 | } 34 | } 35 | 36 | impl fmt::Debug for Enum16 { 37 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 38 | write!(f, "Enum({})", self.0) 39 | } 40 | } 41 | 42 | impl Enum16 { 43 | pub fn of(source: i16) -> Self { 44 | Self(source) 45 | } 46 | #[inline(always)] 47 | pub fn internal(self) -> i16 { 48 | self.0 49 | } 50 | } 51 | impl PartialEq for Enum8 { 52 | fn eq(&self, other: &Self) -> bool { 53 | self.0 == other.0 54 | } 55 | } 56 | 57 | impl fmt::Display for Enum8 { 58 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 59 | write!(f, "Enum8({})", self.0) 60 | } 61 | } 62 | 63 | impl fmt::Debug for Enum8 { 64 | fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { 65 | write!(f, "Enum8({})", self.0) 66 | } 67 | } 68 | 69 | impl Enum8 { 70 | pub fn of(source: i8) -> Self { 71 | Self(source) 72 | } 73 | #[inline(always)] 74 | pub fn internal(self) -> i8 { 75 | self.0 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /clickhouse/src/types/query.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[derive(Clone, Debug)] 16 | pub struct Query { 17 | sql: String, 18 | id: String, 19 | } 20 | 21 | #[allow(dead_code)] 22 | impl Query { 23 | pub fn new(sql: impl AsRef) -> Self { 24 | Self { 25 | sql: sql.as_ref().to_string(), 26 | id: "".to_string(), 27 | } 28 | } 29 | 30 | #[must_use] 31 | pub fn id(self, id: impl AsRef) -> Self { 32 | Self { 33 | id: id.as_ref().to_string(), 34 | ..self 35 | } 36 | } 37 | 38 | pub(crate) fn get_sql(&self) -> &str { 39 | &self.sql 40 | } 41 | 42 | pub(crate) fn get_id(&self) -> &str { 43 | &self.id 44 | } 45 | 46 | pub(crate) fn map_sql(self, f: F) -> Self 47 | where 48 | F: Fn(&str) -> String, 49 | { 50 | Self { 51 | sql: f(&self.sql), 52 | ..self 53 | } 54 | } 55 | } 56 | 57 | impl From for Query 58 | where 59 | T: AsRef, 60 | { 61 | fn from(source: T) -> Self { 62 | Self::new(source) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /clickhouse/src/types/stat_buffer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::types::SqlType; 16 | 17 | pub trait StatBuffer { 18 | type Buffer: AsMut<[u8]> + AsRef<[u8]> + Copy + Sync; 19 | fn buffer() -> Self::Buffer; 20 | fn sql_type() -> SqlType; 21 | } 22 | 23 | impl StatBuffer for u8 { 24 | type Buffer = [Self; 1]; 25 | 26 | fn buffer() -> Self::Buffer { 27 | [0; 1] 28 | } 29 | 30 | fn sql_type() -> SqlType { 31 | SqlType::UInt8 32 | } 33 | } 34 | 35 | impl StatBuffer for u16 { 36 | type Buffer = [u8; 2]; 37 | 38 | fn buffer() -> Self::Buffer { 39 | [0; 2] 40 | } 41 | 42 | fn sql_type() -> SqlType { 43 | SqlType::UInt16 44 | } 45 | } 46 | 47 | impl StatBuffer for u32 { 48 | type Buffer = [u8; 4]; 49 | 50 | fn buffer() -> Self::Buffer { 51 | [0; 4] 52 | } 53 | 54 | fn sql_type() -> SqlType { 55 | SqlType::UInt32 56 | } 57 | } 58 | 59 | impl StatBuffer for u64 { 60 | type Buffer = [u8; 8]; 61 | 62 | fn buffer() -> Self::Buffer { 63 | [0; 8] 64 | } 65 | 66 | fn sql_type() -> SqlType { 67 | SqlType::UInt64 68 | } 69 | } 70 | 71 | impl StatBuffer for i8 { 72 | type Buffer = [u8; 1]; 73 | 74 | fn buffer() -> Self::Buffer { 75 | [0; 1] 76 | } 77 | 78 | fn sql_type() -> SqlType { 79 | SqlType::Int8 80 | } 81 | } 82 | 83 | impl StatBuffer for i16 { 84 | type Buffer = [u8; 2]; 85 | 86 | fn buffer() -> Self::Buffer { 87 | [0; 2] 88 | } 89 | 90 | fn sql_type() -> SqlType { 91 | SqlType::Int16 92 | } 93 | } 94 | 95 | impl StatBuffer for i32 { 96 | type Buffer = [u8; 4]; 97 | 98 | fn buffer() -> Self::Buffer { 99 | [0; 4] 100 | } 101 | 102 | fn sql_type() -> SqlType { 103 | SqlType::Int32 104 | } 105 | } 106 | 107 | impl StatBuffer for i64 { 108 | type Buffer = [u8; 8]; 109 | 110 | fn buffer() -> Self::Buffer { 111 | [0; 8] 112 | } 113 | 114 | fn sql_type() -> SqlType { 115 | SqlType::Int64 116 | } 117 | } 118 | 119 | impl StatBuffer for f32 { 120 | type Buffer = [u8; 4]; 121 | 122 | fn buffer() -> Self::Buffer { 123 | [0; 4] 124 | } 125 | 126 | fn sql_type() -> SqlType { 127 | SqlType::Float32 128 | } 129 | } 130 | 131 | impl StatBuffer for f64 { 132 | type Buffer = [u8; 8]; 133 | 134 | fn buffer() -> Self::Buffer { 135 | [0; 8] 136 | } 137 | 138 | fn sql_type() -> SqlType { 139 | SqlType::Float64 140 | } 141 | } 142 | 143 | impl StatBuffer for bool { 144 | type Buffer = [u8; 1]; 145 | 146 | fn buffer() -> Self::Buffer { 147 | [0; 1] 148 | } 149 | 150 | fn sql_type() -> SqlType { 151 | unimplemented!() 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /clickhouse/tests/it/binary.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::binary::put_uvarint; 16 | use opensrv_clickhouse::binary::ReadEx; 17 | 18 | #[test] 19 | fn test_read_uvarint() { 20 | use std::io::Cursor; 21 | 22 | let bytes = [194_u8, 10]; 23 | let mut cursor = Cursor::new(bytes); 24 | 25 | let actual = cursor.read_uvarint().unwrap(); 26 | 27 | assert_eq!(actual, 1346) 28 | } 29 | 30 | #[test] 31 | fn test_put_uvarint() { 32 | let expected = [148u8, 145, 6, 0, 0, 0, 0, 0, 0, 0]; 33 | let mut buffer = [0u8; 10]; 34 | 35 | let actual = put_uvarint(&mut buffer[..], 100_500); 36 | 37 | assert_eq!(actual, 3); 38 | assert_eq!(buffer, expected); 39 | } 40 | -------------------------------------------------------------------------------- /clickhouse/tests/it/errors.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::errors::*; 16 | 17 | #[test] 18 | fn to_std_error_without_recursion() { 19 | let src_err: Error = From::from("Somth went wrong."); 20 | let dst_err: Box = src_err.into(); 21 | assert_eq!(dst_err.to_string(), "Other error: `Somth went wrong.`"); 22 | } 23 | 24 | #[test] 25 | fn to_io_error_without_recursion() { 26 | let src_err: Error = From::from("Somth went wrong."); 27 | let dst_err: std::io::Error = src_err.into(); 28 | assert_eq!(dst_err.to_string(), "Other error: `Somth went wrong.`"); 29 | } 30 | -------------------------------------------------------------------------------- /clickhouse/tests/it/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod binary; 16 | mod errors; 17 | mod types; 18 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/block/builder.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use chrono::prelude::*; 18 | use chrono_tz::Tz::UTC; 19 | use chrono_tz::Tz::{self}; 20 | use opensrv_clickhouse::row; 21 | use opensrv_clickhouse::types::*; 22 | 23 | #[test] 24 | fn test_push_row() { 25 | let date_value: NaiveDate = UTC 26 | .with_ymd_and_hms(2016, 10, 22, 0, 0, 0) 27 | .unwrap() 28 | .date_naive(); 29 | let date_time_value: DateTime = UTC.with_ymd_and_hms(2014, 7, 8, 14, 0, 0).unwrap(); 30 | 31 | let decimal = Decimal::of(2.0_f64, 4); 32 | 33 | let tuple = Value::Tuple(Arc::new(vec![ 34 | Value::Int32(1), 35 | Value::String(Arc::new(b"text".to_vec())), 36 | Value::Float64(2.3), 37 | ])); 38 | 39 | let mut block = Block::::new(); 40 | block 41 | .push(row! { 42 | i8_field: 1_i8, 43 | i16_field: 1_i16, 44 | i32_field: 1_i32, 45 | i64_field: 1_i64, 46 | 47 | u8_field: 1_u8, 48 | u16_field: 1_u16, 49 | u32_field: 1_u32, 50 | u64_field: 1_u64, 51 | 52 | f32_field: 4.66_f32, 53 | f64_field: 2.71_f64, 54 | 55 | str_field: "text", 56 | opt_filed: Some("text"), 57 | nil_filed: Option::<&str>::None, 58 | 59 | date_field: date_value, 60 | date_time_field: date_time_value, 61 | 62 | decimal_field: decimal, 63 | 64 | tuple_field: tuple, 65 | }) 66 | .unwrap(); 67 | 68 | assert_eq!(block.row_count(), 1); 69 | 70 | assert_eq!(block.columns()[0].sql_type(), SqlType::Int8); 71 | assert_eq!(block.columns()[1].sql_type(), SqlType::Int16); 72 | assert_eq!(block.columns()[2].sql_type(), SqlType::Int32); 73 | assert_eq!(block.columns()[3].sql_type(), SqlType::Int64); 74 | 75 | assert_eq!(block.columns()[4].sql_type(), SqlType::UInt8); 76 | assert_eq!(block.columns()[5].sql_type(), SqlType::UInt16); 77 | assert_eq!(block.columns()[6].sql_type(), SqlType::UInt32); 78 | assert_eq!(block.columns()[7].sql_type(), SqlType::UInt64); 79 | 80 | assert_eq!(block.columns()[8].sql_type(), SqlType::Float32); 81 | assert_eq!(block.columns()[9].sql_type(), SqlType::Float64); 82 | 83 | assert_eq!(block.columns()[10].sql_type(), SqlType::String); 84 | assert_eq!( 85 | block.columns()[11].sql_type(), 86 | SqlType::Nullable(SqlType::String.into()) 87 | ); 88 | assert_eq!( 89 | block.columns()[12].sql_type(), 90 | SqlType::Nullable(SqlType::String.into()) 91 | ); 92 | 93 | assert_eq!(block.columns()[13].sql_type(), SqlType::Date); 94 | assert_eq!( 95 | block.columns()[14].sql_type(), 96 | SqlType::DateTime(DateTimeType::DateTime32) 97 | ); 98 | assert_eq!(block.columns()[15].sql_type(), SqlType::Decimal(18, 4)); 99 | 100 | assert_eq!( 101 | block.columns()[16].sql_type(), 102 | SqlType::Tuple(vec![ 103 | SqlType::Int32.into(), 104 | SqlType::String.into(), 105 | SqlType::Float64.into() 106 | ]) 107 | ); 108 | } 109 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/block/compressed.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::types::decompress_buffer; 16 | 17 | #[test] 18 | fn test_decompress() { 19 | let expected = vec![ 20 | 1u8, 0, 2, 255, 255, 255, 255, 0, 1, 1, 1, 115, 6, 83, 116, 114, 105, 110, 103, 3, 97, 98, 21 | 99, 22 | ]; 23 | 24 | let source = [ 25 | 245_u8, 5, 222, 235, 225, 158, 59, 108, 225, 31, 65, 215, 66, 66, 36, 92, 130, 34, 0, 0, 0, 26 | 23, 0, 0, 0, 240, 8, 1, 0, 2, 255, 255, 255, 255, 0, 1, 1, 1, 115, 6, 83, 116, 114, 105, 27 | 110, 103, 3, 97, 98, 99, 28 | ]; 29 | 30 | let mut cursor = std::io::Cursor::new(&source[..]); 31 | let actual = decompress_buffer(&mut cursor, Vec::new()).unwrap(); 32 | 33 | assert_eq!(actual, expected); 34 | } 35 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/block/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod builder; 16 | mod compressed; 17 | 18 | use std::io::Cursor; 19 | 20 | use chrono_tz::Tz; 21 | use opensrv_clickhouse::binary::*; 22 | use opensrv_clickhouse::errors::Result; 23 | use opensrv_clickhouse::types::*; 24 | 25 | #[test] 26 | fn test_write_default() { 27 | let expected = [1_u8, 0, 2, 255, 255, 255, 255, 0, 0, 0]; 28 | let mut encoder = Encoder::new(); 29 | Block::::default().write(&mut encoder, false); 30 | assert_eq!(encoder.get_buffer_ref(), &expected) 31 | } 32 | 33 | #[test] 34 | fn test_compress_block() { 35 | let expected = vec![ 36 | 245_u8, 5, 222, 235, 225, 158, 59, 108, 225, 31, 65, 215, 66, 66, 36, 92, 130, 34, 0, 0, 0, 37 | 23, 0, 0, 0, 240, 8, 1, 0, 2, 255, 255, 255, 255, 0, 1, 1, 1, 115, 6, 83, 116, 114, 105, 38 | 110, 103, 3, 97, 98, 99, 39 | ]; 40 | 41 | let block = Block::::new().column("s", vec!["abc"]); 42 | 43 | let mut encoder = Encoder::new(); 44 | block.write(&mut encoder, true); 45 | 46 | let actual = encoder.get_buffer(); 47 | assert_eq!(actual, expected); 48 | } 49 | 50 | #[test] 51 | fn test_decompress_block() { 52 | let expected = Block::::new().column("s", vec!["abc"]); 53 | 54 | let source = [ 55 | 245_u8, 5, 222, 235, 225, 158, 59, 108, 225, 31, 65, 215, 66, 66, 36, 92, 130, 34, 0, 0, 0, 56 | 23, 0, 0, 0, 240, 8, 1, 0, 2, 255, 255, 255, 255, 0, 1, 1, 1, 115, 6, 83, 116, 114, 105, 57 | 110, 103, 3, 97, 98, 99, 58 | ]; 59 | 60 | let mut cursor = Cursor::new(&source[..]); 61 | let actual = Block::load(&mut cursor, Tz::UTC, true).unwrap(); 62 | 63 | assert_eq!(actual, expected); 64 | } 65 | 66 | #[test] 67 | fn test_read_empty_block() { 68 | let source = [1, 0, 2, 255, 255, 255, 255, 0, 0, 0]; 69 | let mut cursor = Cursor::new(&source[..]); 70 | match Block::::load(&mut cursor, Tz::Zulu, false) { 71 | Ok(block) => assert!(block.is_empty()), 72 | Err(_) => unreachable!(), 73 | } 74 | } 75 | 76 | #[test] 77 | fn test_empty() { 78 | assert!(Block::::default().is_empty()) 79 | } 80 | 81 | #[test] 82 | fn test_column_and_rows() { 83 | let block = Block::::new() 84 | .column("hello_id", vec![5_u64, 6]) 85 | .column("value", vec!["lol", "zuz"]); 86 | 87 | assert_eq!(block.column_count(), 2); 88 | assert_eq!(block.row_count(), 2); 89 | } 90 | 91 | #[test] 92 | fn test_from_sql() { 93 | let block = Block::::new() 94 | .column("hello_id", vec![5_u64, 6]) 95 | .column("value", vec!["lol", "zuz"]); 96 | 97 | let v: Result = block.get(0, "hello_id"); 98 | assert_eq!(v.unwrap(), 5); 99 | } 100 | 101 | #[test] 102 | fn test_concat() { 103 | let block_a = make_block(); 104 | let block_b = make_block(); 105 | 106 | let actual = Block::concat(&[block_a, block_b]); 107 | assert_eq!(actual.row_count(), 4); 108 | assert_eq!(actual.column_count(), 1); 109 | 110 | assert_eq!( 111 | "5446d186-4e90-4dd8-8ec1-f9a436834613".to_string(), 112 | actual.get::(0, 0).unwrap() 113 | ); 114 | assert_eq!( 115 | "f7cf31f4-7f37-4e27-91c0-5ac0ad0b145b".to_string(), 116 | actual.get::(1, 0).unwrap() 117 | ); 118 | assert_eq!( 119 | "5446d186-4e90-4dd8-8ec1-f9a436834613".to_string(), 120 | actual.get::(2, 0).unwrap() 121 | ); 122 | assert_eq!( 123 | "f7cf31f4-7f37-4e27-91c0-5ac0ad0b145b".to_string(), 124 | actual.get::(3, 0).unwrap() 125 | ); 126 | } 127 | 128 | fn make_block() -> Block { 129 | Block::new().column( 130 | "9b96ad8b-488a-4fef-8087-8a9ae4800f00", 131 | vec![ 132 | "5446d186-4e90-4dd8-8ec1-f9a436834613".to_string(), 133 | "f7cf31f4-7f37-4e27-91c0-5ac0ad0b145b".to_string(), 134 | ], 135 | ) 136 | } 137 | 138 | #[test] 139 | fn test_chunks() { 140 | let first = Block::new().column("A", vec![1, 2]); 141 | let second = Block::new().column("A", vec![3, 4]); 142 | let third = Block::new().column("A", vec![5]); 143 | 144 | let block = Block::::new().column("A", vec![1, 2, 3, 4, 5]); 145 | let mut iter = block.chunks(2); 146 | 147 | assert_eq!(Some(first), iter.next()); 148 | assert_eq!(Some(second), iter.next()); 149 | assert_eq!(Some(third), iter.next()); 150 | assert_eq!(None, iter.next()); 151 | } 152 | 153 | #[test] 154 | fn test_chunks_of_empty_block() { 155 | let block = Block::default(); 156 | assert_eq!(1, block.chunks(100_500).count()); 157 | assert_eq!(Some(block.clone()), block.chunks(100_500).next()); 158 | } 159 | 160 | #[test] 161 | fn test_rows() { 162 | let expected = vec![1_u8, 2, 3]; 163 | let block = Block::::new().column("A", vec![1_u8, 2, 3]); 164 | let actual: Vec = block.rows().map(|row| row.get("A").unwrap()).collect(); 165 | assert_eq!(expected, actual); 166 | } 167 | 168 | #[test] 169 | fn test_write_and_read() { 170 | let block = Block::::new().column("y", vec![Some(1_u8), None]); 171 | 172 | let mut encoder = Encoder::new(); 173 | block.write(&mut encoder, false); 174 | 175 | let mut reader = Cursor::new(encoder.get_buffer_ref()); 176 | let rblock = Block::load(&mut reader, Tz::Zulu, false).unwrap(); 177 | 178 | assert_eq!(block, rblock); 179 | } 180 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/array.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Cursor; 16 | 17 | use chrono_tz::Tz; 18 | use opensrv_clickhouse::binary::Encoder; 19 | use opensrv_clickhouse::types::*; 20 | 21 | #[test] 22 | fn test_write_and_read() { 23 | let block = Block::::new().column( 24 | "vals", 25 | vec![vec![7_u32, 8], vec![9, 1, 2], vec![3, 4, 5, 6]], 26 | ); 27 | 28 | let mut encoder = Encoder::new(); 29 | block.write(&mut encoder, false); 30 | 31 | let mut reader = Cursor::new(encoder.get_buffer_ref()); 32 | let rblock = Block::load(&mut reader, Tz::Zulu, false).unwrap(); 33 | 34 | assert_eq!(block, rblock); 35 | } 36 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/concat.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use opensrv_clickhouse::types::column::concat::*; 18 | use opensrv_clickhouse::types::column::*; 19 | 20 | #[test] 21 | fn test_build_index() { 22 | let sizes = [2_usize, 3, 4]; 23 | let index = build_index(sizes.iter().cloned()); 24 | assert_eq!(index, vec![0, 2, 5, 9]) 25 | } 26 | 27 | #[test] 28 | fn test_find_chunk() { 29 | let index = vec![0_usize, 2, 5, 9]; 30 | assert_eq!(find_chunk(&index, 0), 0); 31 | assert_eq!(find_chunk(&index, 1), 0); 32 | assert_eq!(find_chunk(&index, 2), 1); 33 | assert_eq!(find_chunk(&index, 3), 1); 34 | assert_eq!(find_chunk(&index, 4), 1); 35 | assert_eq!(find_chunk(&index, 5), 2); 36 | assert_eq!(find_chunk(&index, 6), 2); 37 | 38 | assert_eq!(find_chunk(&index, 7), 2); 39 | assert_eq!(find_chunk(&[0], 7), 0); 40 | } 41 | 42 | #[test] 43 | fn test_find_chunk2() { 44 | let index = vec![0_usize, 0, 5]; 45 | assert_eq!(find_chunk(&index, 0), 1); 46 | assert_eq!(find_chunk(&index, 1), 1); 47 | assert_eq!(find_chunk(&index, 2), 1); 48 | assert_eq!(find_chunk(&index, 3), 1); 49 | assert_eq!(find_chunk(&index, 4), 1); 50 | assert_eq!(find_chunk(&index, 5), 0); 51 | } 52 | 53 | #[test] 54 | fn test_find_chunk5() { 55 | let index = vec![ 56 | 0_usize, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57, 60, 57 | 63, 66, 69, 58 | ]; 59 | for i in 0..69 { 60 | assert_eq!(find_chunk(&index, i), 1 + i / 3); 61 | } 62 | } 63 | 64 | #[test] 65 | fn test_concat_column() { 66 | let xs = vec![make_string_column(), make_string_column()]; 67 | let actual = ConcatColumnData::concat(xs); 68 | 69 | assert_eq!( 70 | actual.at(0).as_str().unwrap(), 71 | "13298a5f-6a10-4fbe-9644-807f7ebf82cc" 72 | ); 73 | assert_eq!( 74 | actual.at(1).as_str().unwrap(), 75 | "df0e62bb-c0db-4728-a558-821f8e8da38c" 76 | ); 77 | assert_eq!( 78 | actual.at(2).as_str().unwrap(), 79 | "13298a5f-6a10-4fbe-9644-807f7ebf82cc" 80 | ); 81 | assert_eq!( 82 | actual.at(3).as_str().unwrap(), 83 | "df0e62bb-c0db-4728-a558-821f8e8da38c" 84 | ); 85 | 86 | assert_eq!(actual.len(), 4); 87 | } 88 | 89 | #[test] 90 | fn test_concat_num_column() { 91 | let xs = vec![make_num_column(), make_num_column()]; 92 | let actual = ConcatColumnData::concat(xs); 93 | 94 | assert_eq!(u32::from(actual.at(0)), 1_u32); 95 | assert_eq!(u32::from(actual.at(1)), 2_u32); 96 | assert_eq!(u32::from(actual.at(2)), 1_u32); 97 | assert_eq!(u32::from(actual.at(3)), 2_u32); 98 | 99 | assert_eq!(actual.len(), 4); 100 | } 101 | 102 | fn make_string_column() -> ArcColumnData { 103 | let mut data = StringColumnData::with_capacity(1); 104 | data.append("13298a5f-6a10-4fbe-9644-807f7ebf82cc".to_string()); 105 | data.append("df0e62bb-c0db-4728-a558-821f8e8da38c".to_string()); 106 | Arc::new(data) 107 | } 108 | 109 | fn make_num_column() -> ArcColumnData { 110 | let mut data = VectorColumnData::::with_capacity(1); 111 | data.append(1_u32); 112 | data.append(2_u32); 113 | Arc::new(data) 114 | } 115 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/date.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use chrono::TimeZone; 16 | use chrono_tz::Tz; 17 | use opensrv_clickhouse::types::column::*; 18 | use opensrv_clickhouse::types::*; 19 | 20 | #[test] 21 | fn test_create_date() { 22 | let tz = Tz::Zulu; 23 | let column = Vec::column_from::(vec![tz 24 | .with_ymd_and_hms(2016, 10, 22, 0, 0, 0) 25 | .unwrap() 26 | .date_naive()]); 27 | assert_eq!("2016-10-22", format!("{:#}", column.at(0))); 28 | assert_eq!(SqlType::Date, column.sql_type()); 29 | } 30 | 31 | #[test] 32 | fn test_create_date_time() { 33 | let tz = Tz::Zulu; 34 | let column = Vec::column_from::(vec![tz 35 | .with_ymd_and_hms(2016, 10, 22, 12, 0, 0) 36 | .unwrap()]); 37 | assert_eq!(format!("{}", column.at(0)), "2016-10-22 12:00:00"); 38 | } 39 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/datetime64.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use chrono::prelude::*; 16 | use chrono_tz::Tz; 17 | use opensrv_clickhouse::types::column::datetime64::*; 18 | 19 | #[test] 20 | fn test_to_datetime() { 21 | let expected = DateTime::parse_from_rfc3339("2019-01-01T00:00:00-00:00").unwrap(); 22 | let actual = to_datetime(1_546_300_800_000, 3, Tz::UTC); 23 | assert_eq!(actual, expected) 24 | } 25 | 26 | #[test] 27 | fn test_from_datetime() { 28 | let origin = DateTime::parse_from_rfc3339("2019-01-01T00:00:00-00:00").unwrap(); 29 | let actual = from_datetime(origin, 3); 30 | assert_eq!(actual, 1_546_300_800_000) 31 | } 32 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/iter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::types::Block; 16 | 17 | #[test] 18 | fn test_complex_iter() { 19 | let lo = Block::new().column("?", vec![1_u32, 2]); 20 | let hi = Block::new().column("?", vec![3_u32, 4, 5]); 21 | 22 | let block = Block::concat(&[lo, hi]); 23 | 24 | let columns = block.columns()[0].iter::().unwrap(); 25 | let actual: Vec<_> = columns.collect(); 26 | assert_eq!(actual, vec![&1_u32, &2, &3, &4, &5]); 27 | 28 | let null_u32 = Block::new().column("?", vec![Some(3_u32), Some(4), None]); 29 | let columns = null_u32.columns()[0].iter::>().unwrap(); 30 | let actual: Vec<_> = columns.collect(); 31 | assert_eq!(actual, vec![Some(&3_u32), Some(&4), None]); 32 | } 33 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/list.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::types::column::*; 16 | use rand::random; 17 | 18 | #[test] 19 | fn test_push_and_len() { 20 | let mut list = List::with_capacity(100_500); 21 | 22 | for i in 0..100_500 { 23 | assert_eq!(list.len(), i as usize); 24 | list.push(i); 25 | } 26 | } 27 | 28 | #[test] 29 | fn test_push_and_get() { 30 | let mut list = List::::new(); 31 | let mut vs = vec![0.0_f64; 100]; 32 | 33 | for (count, _) in (0..100).enumerate() { 34 | assert_eq!(list.len(), count); 35 | 36 | for (i, v) in vs.iter().take(count).enumerate() { 37 | assert!((list.at(i) - *v).abs() < f64::EPSILON); 38 | } 39 | 40 | let k = random(); 41 | list.push(k); 42 | vs[count] = k; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod array; 16 | mod concat; 17 | mod date; 18 | mod datetime64; 19 | mod factory; 20 | mod iter; 21 | mod list; 22 | mod string_pool; 23 | mod tuple; 24 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/string_pool.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Write; 16 | 17 | use opensrv_clickhouse::types::StringPool; 18 | 19 | #[test] 20 | fn test_allocate() { 21 | let mut pool = StringPool::with_capacity(10); 22 | for i in 1..1000 { 23 | let buffer = pool.allocate(i); 24 | assert_eq!(buffer.len(), i); 25 | assert_eq!(buffer[0], 0); 26 | buffer[0] = 1 27 | } 28 | } 29 | 30 | #[test] 31 | fn test_get() { 32 | let mut pool = StringPool::with_capacity(10); 33 | 34 | for i in 0..1000 { 35 | let s = format!("text-{}", i); 36 | let mut buffer = pool.allocate(s.len()); 37 | 38 | assert_eq!(buffer.len(), s.len()); 39 | buffer.write_all(s.as_bytes()).unwrap(); 40 | } 41 | 42 | for i in 0..1000 { 43 | let s = String::from_utf8(Vec::from(pool.get(i))).unwrap(); 44 | assert_eq!(s, format!("text-{}", i)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/column/tuple.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use chrono::TimeZone; 18 | use chrono_tz::Tz; 19 | use opensrv_clickhouse::types::column::*; 20 | use opensrv_clickhouse::types::*; 21 | 22 | #[test] 23 | fn test_tuple_type() { 24 | let tz = Tz::Zulu; 25 | let inner = vec![ 26 | Vec::column_from::(vec![ 27 | tz.with_ymd_and_hms(2016, 10, 22, 0, 0, 0) 28 | .unwrap() 29 | .date_naive(), 30 | tz.with_ymd_and_hms(2017, 11, 23, 0, 0, 0) 31 | .unwrap() 32 | .date_naive(), 33 | ]), 34 | Vec::column_from::(vec![1_i32, 2]), 35 | Vec::column_from::(vec!["hello", "data"]), 36 | ]; 37 | 38 | let mut column = Vec::column_from::(inner); 39 | assert_eq!( 40 | SqlType::Tuple(vec![&SqlType::Date, &SqlType::Int32, &SqlType::String]), 41 | column.sql_type() 42 | ); 43 | assert_eq!(2, column.len()); 44 | 45 | let value = Value::Tuple(Arc::new(vec![ 46 | Value::Date(u16::get_days( 47 | tz.with_ymd_and_hms(2018, 12, 24, 0, 0, 0) 48 | .unwrap() 49 | .date_naive(), 50 | )), 51 | Value::Int32(3), 52 | Value::String(Arc::new(b"bend".to_vec())), 53 | ])); 54 | let column = Arc::get_mut(&mut column).unwrap(); 55 | column.push(value); 56 | assert_eq!(3, column.len()); 57 | assert_eq!(format!("{}", column.at(2)), "(2018-12-24, 3, bend)"); 58 | } 59 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/decimal.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use opensrv_clickhouse::types::*; 16 | 17 | #[test] 18 | fn test_new() { 19 | assert_eq!(Decimal::new(2, 1), Decimal::of(0.2_f64, 1)); 20 | assert_eq!(Decimal::new(2, 5), Decimal::of(0.00002_f64, 5)); 21 | } 22 | 23 | #[test] 24 | fn test_display() { 25 | assert_eq!(format!("{}", Decimal::of(2.1_f32, 4)), "2.1000"); 26 | assert_eq!(format!("{}", Decimal::of(0.2_f64, 4)), "0.2000"); 27 | assert_eq!(format!("{}", Decimal::of(2, 4)), "2.0000"); 28 | assert_eq!(format!("{:?}", Decimal::of(2, 4)), "2.0000"); 29 | } 30 | 31 | #[test] 32 | fn test_eq() { 33 | assert_eq!(Decimal::of(2.0_f64, 4), Decimal::of(2.0_f64, 4)); 34 | assert_ne!(Decimal::of(3.0_f64, 4), Decimal::of(2.0_f64, 4)); 35 | 36 | assert_eq!(Decimal::of(2.0_f64, 4), Decimal::of(2.0_f64, 2)); 37 | assert_ne!(Decimal::of(2.0_f64, 4), Decimal::of(3.0_f64, 2)); 38 | 39 | assert_eq!(Decimal::of(2.0_f64, 2), Decimal::of(2.0_f64, 4)); 40 | assert_ne!(Decimal::of(3.0_f64, 2), Decimal::of(2.0_f64, 4)); 41 | } 42 | 43 | #[test] 44 | fn test_internal32() { 45 | let internal: i32 = Decimal::of(2, 4).internal(); 46 | assert_eq!(internal, 20000_i32); 47 | } 48 | 49 | #[test] 50 | fn test_internal64() { 51 | let internal: i64 = Decimal::of(2, 4).internal(); 52 | assert_eq!(internal, 20000_i64); 53 | } 54 | 55 | #[test] 56 | fn test_scale() { 57 | assert_eq!(Decimal::of(2, 4).scale(), 4); 58 | } 59 | 60 | #[test] 61 | fn test_from_f32() { 62 | let value: f32 = Decimal::of(2, 4).into(); 63 | assert!((value - 2.0_f32).abs() <= f32::EPSILON); 64 | } 65 | 66 | #[test] 67 | fn test_from_f64() { 68 | let value: f64 = Decimal::of(2, 4).into(); 69 | assert!((value - 2.0_f64).abs() < f64::EPSILON); 70 | } 71 | 72 | #[test] 73 | fn set_scale1() { 74 | let a = Decimal::of(12, 3); 75 | let b = a.set_scale(2); 76 | 77 | assert_eq!(2, b.scale()); 78 | assert_eq!(1200, b.internal::()); 79 | } 80 | 81 | #[test] 82 | fn set_scale2() { 83 | let a = Decimal::of(12, 3); 84 | let b = a.set_scale(4); 85 | 86 | assert_eq!(4, b.scale()); 87 | assert_eq!(120_000, b.internal::()); 88 | } 89 | 90 | #[test] 91 | fn test_decimal2str() { 92 | let d = Decimal::of(0.00001, 5); 93 | let actual = decimal2str(&d); 94 | assert_eq!(actual, "0.00001".to_string()); 95 | } 96 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/from_sql.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use chrono::prelude::*; 16 | use chrono_tz::Tz; 17 | use opensrv_clickhouse::types::*; 18 | 19 | #[test] 20 | fn test_u8() { 21 | let v = ValueRef::from(42_u8); 22 | let actual = u8::from_sql(v).unwrap(); 23 | assert_eq!(actual, 42_u8); 24 | } 25 | 26 | #[test] 27 | fn test_bad_convert() { 28 | let v = ValueRef::from(42_u16); 29 | match u32::from_sql(v) { 30 | Ok(_) => panic!("should fail"), 31 | Err(e) => assert_eq!( 32 | "From SQL error: `SqlType::UInt16 cannot be cast to u32.`".to_string(), 33 | format!("{}", e) 34 | ), 35 | } 36 | } 37 | 38 | #[test] 39 | fn null_to_datetime() { 40 | let null_value = ValueRef::Nullable(Either::Left( 41 | SqlType::DateTime(DateTimeType::DateTime32).into(), 42 | )); 43 | let date = Option::>::from_sql(null_value); 44 | assert_eq!(date.unwrap(), None); 45 | } 46 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod block; 16 | mod column; 17 | mod decimal; 18 | mod from_sql; 19 | mod opions; 20 | mod value; 21 | mod value_ref; 22 | -------------------------------------------------------------------------------- /clickhouse/tests/it/types/opions.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::str::FromStr; 16 | use std::time::Duration; 17 | 18 | use opensrv_clickhouse::types::*; 19 | use url::Url; 20 | 21 | #[test] 22 | fn test_parse_hosts() { 23 | let source = "host2:9000,host3:9000"; 24 | let expected = vec![ 25 | Url::from_str("tcp://host2:9000").unwrap(), 26 | Url::from_str("tcp://host3:9000").unwrap(), 27 | ]; 28 | let actual = parse_hosts(source).unwrap(); 29 | assert_eq!(actual, expected) 30 | } 31 | 32 | #[test] 33 | fn test_parse_default() { 34 | let url = "tcp://host1"; 35 | let options = from_url(url).unwrap(); 36 | assert_eq!(options.database, "default"); 37 | assert_eq!(options.username, "default"); 38 | assert_eq!(options.password, ""); 39 | } 40 | 41 | #[test] 42 | #[cfg(feature = "tls")] 43 | fn test_parse_secure_options() { 44 | let url = "tcp://username:password@host1:9001/database?ping_timeout=42ms&keepalive=99s&compression=lz4&connection_timeout=10s&secure=true&skip_verify=true"; 45 | assert_eq!( 46 | Options { 47 | username: "username".into(), 48 | password: "password".into(), 49 | addr: Url::parse("tcp://username:password@host1:9001").unwrap(), 50 | database: "database".into(), 51 | keepalive: Some(Duration::from_secs(99)), 52 | ping_timeout: Duration::from_millis(42), 53 | connection_timeout: Duration::from_secs(10), 54 | compression: true, 55 | secure: true, 56 | skip_verify: true, 57 | ..Options::default() 58 | }, 59 | from_url(url).unwrap(), 60 | ); 61 | } 62 | 63 | #[test] 64 | fn test_parse_options() { 65 | let url = "tcp://username:password@host1:9001/database?ping_timeout=42ms&keepalive=99s&compression=lz4&connection_timeout=10s"; 66 | assert_eq!( 67 | Options { 68 | username: "username".into(), 69 | password: "password".into(), 70 | addr: Url::parse("tcp://username:password@host1:9001").unwrap(), 71 | database: "database".into(), 72 | keepalive: Some(Duration::from_secs(99)), 73 | ping_timeout: Duration::from_millis(42), 74 | connection_timeout: Duration::from_secs(10), 75 | compression: true, 76 | ..Options::default() 77 | }, 78 | from_url(url).unwrap(), 79 | ); 80 | } 81 | 82 | #[test] 83 | #[should_panic] 84 | fn test_parse_invalid_url() { 85 | let url = "ʘ_ʘ"; 86 | from_url(url).unwrap(); 87 | } 88 | 89 | #[test] 90 | #[should_panic] 91 | fn test_parse_with_unknown_url() { 92 | let url = "tcp://localhost:9000/foo?bar=baz"; 93 | from_url(url).unwrap(); 94 | } 95 | 96 | #[test] 97 | #[should_panic] 98 | fn test_parse_with_multi_databases() { 99 | let url = "tcp://localhost:9000/foo/bar"; 100 | from_url(url).unwrap(); 101 | } 102 | 103 | #[test] 104 | fn test_parse_duration() { 105 | assert_eq!(parse_duration("3s").unwrap(), Duration::from_secs(3)); 106 | assert_eq!(parse_duration("123ms").unwrap(), Duration::from_millis(123)); 107 | 108 | parse_duration("ms").unwrap_err(); 109 | parse_duration("1ss").unwrap_err(); 110 | } 111 | 112 | #[test] 113 | fn test_parse_opt_duration() { 114 | assert_eq!( 115 | parse_opt_duration("3s").unwrap(), 116 | Some(Duration::from_secs(3)) 117 | ); 118 | assert_eq!(parse_opt_duration("none").unwrap(), None::); 119 | } 120 | 121 | #[test] 122 | fn test_parse_compression() { 123 | assert!(!parse_compression("none").unwrap()); 124 | assert!(parse_compression("lz4").unwrap()); 125 | parse_compression("?").unwrap_err(); 126 | } 127 | -------------------------------------------------------------------------------- /components/micromarshal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "micromarshal" 3 | version = "0.7.0" 4 | authors = ["Databend Authors "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | description = "(De)Serialisation between Rust values and binary byte objects. " 8 | readme = "README.md" 9 | repository = "https://github.com/datafuselabs/opensrv" 10 | categories = ["emulators", "encoding", "parser-implementations", "parsing"] 11 | keywords = ["marshal", "read", "decode"] 12 | 13 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 14 | 15 | [lib] 16 | doctest = false 17 | 18 | [dependencies] 19 | ordered-float = "4.1.0" 20 | ethnum = "1.3" 21 | 22 | [dev-dependencies] 23 | rand = "0.8.5" 24 | -------------------------------------------------------------------------------- /components/micromarshal/README.md: -------------------------------------------------------------------------------- 1 | # Micro Marshal 2 | 3 | **(De)serialisation between Rust values and binary byte objects.** 4 | 5 | ## Getting help 6 | 7 | Submit [issues](https://github.com/datafuselabs/opensrv/issues/new/choose) for bug report or asking questions in [discussion](https://github.com/datafuselabs/opensrv/discussions/new?category=q-a). 8 | 9 | ## Tips 10 | 11 | As the functionality of this crate is largely stable, its version will mostly just change as other crates change. 12 | 13 | ## Credits 14 | 15 | This project is extracted from the Databend's common-io module. Now released as a separate crate. 16 | 17 | ## License 18 | 19 | Licensed under Apache License, Version 2.0. -------------------------------------------------------------------------------- /components/micromarshal/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod marshal; 16 | mod unmarshal; 17 | 18 | pub use marshal::Marshal; 19 | pub use unmarshal::Unmarshal; 20 | -------------------------------------------------------------------------------- /components/micromarshal/src/marshal.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use ethnum::{I256, U256}; 16 | use ordered_float::OrderedFloat; 17 | 18 | pub trait Marshal { 19 | fn marshal(&self, scratch: &mut [u8]); 20 | } 21 | 22 | impl Marshal for u8 { 23 | fn marshal(&self, scratch: &mut [u8]) { 24 | scratch[0] = *self; 25 | } 26 | } 27 | 28 | impl Marshal for u16 { 29 | fn marshal(&self, scratch: &mut [u8]) { 30 | let bytes = self.to_le_bytes(); 31 | scratch.copy_from_slice(&bytes); 32 | } 33 | } 34 | 35 | impl Marshal for u32 { 36 | fn marshal(&self, scratch: &mut [u8]) { 37 | let bytes = self.to_le_bytes(); 38 | scratch.copy_from_slice(&bytes); 39 | } 40 | } 41 | 42 | impl Marshal for u64 { 43 | fn marshal(&self, scratch: &mut [u8]) { 44 | let bytes = self.to_le_bytes(); 45 | scratch.copy_from_slice(&bytes); 46 | } 47 | } 48 | 49 | impl Marshal for u128 { 50 | fn marshal(&self, scratch: &mut [u8]) { 51 | let bytes = self.to_le_bytes(); 52 | scratch.copy_from_slice(&bytes); 53 | } 54 | } 55 | 56 | impl Marshal for U256 { 57 | fn marshal(&self, scratch: &mut [u8]) { 58 | let bytes = self.to_le_bytes(); 59 | scratch.copy_from_slice(&bytes); 60 | } 61 | } 62 | 63 | impl Marshal for i8 { 64 | fn marshal(&self, scratch: &mut [u8]) { 65 | scratch[0] = *self as u8; 66 | } 67 | } 68 | 69 | impl Marshal for i16 { 70 | fn marshal(&self, scratch: &mut [u8]) { 71 | let bytes = self.to_le_bytes(); 72 | scratch.copy_from_slice(&bytes); 73 | } 74 | } 75 | 76 | impl Marshal for i32 { 77 | fn marshal(&self, scratch: &mut [u8]) { 78 | let bytes = self.to_le_bytes(); 79 | scratch.copy_from_slice(&bytes); 80 | } 81 | } 82 | 83 | impl Marshal for i64 { 84 | fn marshal(&self, scratch: &mut [u8]) { 85 | let bytes = self.to_le_bytes(); 86 | scratch.copy_from_slice(&bytes); 87 | } 88 | } 89 | 90 | impl Marshal for i128 { 91 | fn marshal(&self, scratch: &mut [u8]) { 92 | let bytes = self.to_le_bytes(); 93 | scratch.copy_from_slice(&bytes); 94 | } 95 | } 96 | 97 | impl Marshal for I256 { 98 | fn marshal(&self, scratch: &mut [u8]) { 99 | let bytes = self.to_le_bytes(); 100 | scratch.copy_from_slice(&bytes); 101 | } 102 | } 103 | 104 | impl Marshal for f32 { 105 | fn marshal(&self, scratch: &mut [u8]) { 106 | let bits = self.to_bits(); 107 | let bytes = bits.to_le_bytes(); 108 | scratch.copy_from_slice(&bytes); 109 | } 110 | } 111 | 112 | impl Marshal for f64 { 113 | fn marshal(&self, scratch: &mut [u8]) { 114 | let bits = self.to_bits(); 115 | let bytes = bits.to_le_bytes(); 116 | scratch.copy_from_slice(&bytes); 117 | } 118 | } 119 | 120 | impl Marshal for OrderedFloat { 121 | fn marshal(&self, scratch: &mut [u8]) { 122 | let bits = self.to_bits(); 123 | let bytes = bits.to_le_bytes(); 124 | scratch.copy_from_slice(&bytes); 125 | } 126 | } 127 | 128 | impl Marshal for OrderedFloat { 129 | fn marshal(&self, scratch: &mut [u8]) { 130 | let bits = self.to_bits(); 131 | let bytes = bits.to_le_bytes(); 132 | scratch.copy_from_slice(&bytes); 133 | } 134 | } 135 | 136 | impl Marshal for bool { 137 | fn marshal(&self, scratch: &mut [u8]) { 138 | scratch[0] = *self as u8; 139 | } 140 | } 141 | 142 | impl Marshal for char { 143 | fn marshal(&self, scratch: &mut [u8]) { 144 | let bits = *self as u32; 145 | let bytes = bits.to_le_bytes(); 146 | scratch.copy_from_slice(&bytes); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /components/micromarshal/src/unmarshal.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io; 16 | 17 | use std::io::Error; 18 | use std::io::Result; 19 | 20 | use ethnum::I256; 21 | use ethnum::U256; 22 | use ordered_float::OrderedFloat; 23 | 24 | pub trait Unmarshal { 25 | fn unmarshal(scratch: &[u8]) -> T; 26 | fn try_unmarshal(scratch: &[u8]) -> Result { 27 | Ok(Self::unmarshal(scratch)) 28 | } 29 | } 30 | 31 | impl Unmarshal for u8 { 32 | fn unmarshal(scratch: &[u8]) -> Self { 33 | scratch[0] 34 | } 35 | } 36 | 37 | impl Unmarshal for u16 { 38 | fn unmarshal(scratch: &[u8]) -> Self { 39 | u16::from_le_bytes(scratch.try_into().unwrap()) 40 | } 41 | } 42 | 43 | impl Unmarshal for u32 { 44 | fn unmarshal(scratch: &[u8]) -> Self { 45 | u32::from_le_bytes(scratch.try_into().unwrap()) 46 | } 47 | } 48 | 49 | impl Unmarshal for u64 { 50 | fn unmarshal(scratch: &[u8]) -> Self { 51 | u64::from_le_bytes(scratch.try_into().unwrap()) 52 | } 53 | } 54 | 55 | impl Unmarshal for u128 { 56 | fn unmarshal(scratch: &[u8]) -> Self { 57 | u128::from_le_bytes(scratch.try_into().unwrap()) 58 | } 59 | } 60 | 61 | impl Unmarshal for U256 { 62 | fn unmarshal(scratch: &[u8]) -> Self { 63 | U256::from_le_bytes(scratch.try_into().unwrap()) 64 | } 65 | } 66 | 67 | impl Unmarshal for i8 { 68 | fn unmarshal(scratch: &[u8]) -> Self { 69 | i8::from_le_bytes(scratch.try_into().unwrap()) 70 | } 71 | } 72 | 73 | impl Unmarshal for i16 { 74 | fn unmarshal(scratch: &[u8]) -> Self { 75 | i16::from_le_bytes(scratch.try_into().unwrap()) 76 | } 77 | } 78 | 79 | impl Unmarshal for i32 { 80 | fn unmarshal(scratch: &[u8]) -> Self { 81 | i32::from_le_bytes(scratch.try_into().unwrap()) 82 | } 83 | } 84 | 85 | impl Unmarshal for i64 { 86 | fn unmarshal(scratch: &[u8]) -> Self { 87 | i64::from_le_bytes(scratch.try_into().unwrap()) 88 | } 89 | } 90 | 91 | impl Unmarshal for i128 { 92 | fn unmarshal(scratch: &[u8]) -> Self { 93 | i128::from_le_bytes(scratch.try_into().unwrap()) 94 | } 95 | } 96 | 97 | impl Unmarshal for I256 { 98 | fn unmarshal(scratch: &[u8]) -> Self { 99 | I256::from_le_bytes(scratch.try_into().unwrap()) 100 | } 101 | } 102 | 103 | impl Unmarshal for f32 { 104 | fn unmarshal(scratch: &[u8]) -> Self { 105 | let bits = u32::from_le_bytes(scratch.try_into().unwrap()); 106 | Self::from_bits(bits) 107 | } 108 | } 109 | 110 | impl Unmarshal for f64 { 111 | fn unmarshal(scratch: &[u8]) -> Self { 112 | let bits = u64::from_le_bytes(scratch.try_into().unwrap()); 113 | Self::from_bits(bits) 114 | } 115 | } 116 | 117 | impl Unmarshal> for OrderedFloat { 118 | fn unmarshal(scratch: &[u8]) -> Self { 119 | let bits = u32::from_le_bytes(scratch.try_into().unwrap()); 120 | f32::from_bits(bits).into() 121 | } 122 | } 123 | 124 | impl Unmarshal> for OrderedFloat { 125 | fn unmarshal(scratch: &[u8]) -> Self { 126 | let bits = u64::from_le_bytes(scratch.try_into().unwrap()); 127 | f64::from_bits(bits).into() 128 | } 129 | } 130 | 131 | impl Unmarshal for bool { 132 | fn unmarshal(scratch: &[u8]) -> Self { 133 | scratch[0] != 0 134 | } 135 | } 136 | 137 | impl Unmarshal for char { 138 | fn unmarshal(_: &[u8]) -> char { 139 | unimplemented!() 140 | } 141 | 142 | fn try_unmarshal(scratch: &[u8]) -> Result { 143 | let bits = u32::from_le_bytes(scratch.try_into().unwrap()); 144 | match char::from_u32(bits) { 145 | Some(c) => Ok(c), 146 | None => Err(Error::new( 147 | io::ErrorKind::Other, 148 | format!("try unmarshal u32 to char failed: {}", bits), 149 | )), 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /components/micromarshal/tests/it/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt; 16 | 17 | use ethnum::I256; 18 | use ethnum::U256; 19 | use micromarshal::*; 20 | use rand::distributions::Distribution; 21 | use rand::distributions::Standard; 22 | use rand::random; 23 | 24 | mod statbuffer; 25 | 26 | use statbuffer::StatBuffer; 27 | 28 | trait Random { 29 | fn random() -> Self; 30 | } 31 | 32 | impl Random for U256 { 33 | fn random() -> Self { 34 | U256::from_words(random(), random()) 35 | } 36 | } 37 | 38 | impl Random for I256 { 39 | fn random() -> Self { 40 | I256::from_words(random(), random()) 41 | } 42 | } 43 | 44 | fn test_some() 45 | where 46 | T: Copy + fmt::Debug + StatBuffer + Marshal + Unmarshal + PartialEq, 47 | Standard: Distribution, 48 | { 49 | for _ in 0..100 { 50 | let mut buffer = T::buffer(); 51 | let v = random::(); 52 | 53 | v.marshal(buffer.as_mut()); 54 | let u = T::unmarshal(buffer.as_ref()); 55 | 56 | assert_eq!(v, u); 57 | } 58 | } 59 | 60 | fn test_some_large() 61 | where 62 | T: Copy + fmt::Debug + StatBuffer + Marshal + Unmarshal + PartialEq + Random, 63 | { 64 | for _ in 0..100 { 65 | let mut buffer = T::buffer(); 66 | let v = T::random(); 67 | 68 | v.marshal(buffer.as_mut()); 69 | let u = T::unmarshal(buffer.as_ref()); 70 | 71 | assert_eq!(v, u); 72 | } 73 | } 74 | 75 | #[test] 76 | fn test_u8() { 77 | test_some::() 78 | } 79 | 80 | #[test] 81 | fn test_u16() { 82 | test_some::() 83 | } 84 | 85 | #[test] 86 | fn test_u32() { 87 | test_some::() 88 | } 89 | 90 | #[test] 91 | fn test_u64() { 92 | test_some::() 93 | } 94 | 95 | #[test] 96 | fn test_u128() { 97 | test_some::() 98 | } 99 | 100 | #[test] 101 | fn test_u256() { 102 | test_some_large::(); 103 | } 104 | 105 | #[test] 106 | fn test_i8() { 107 | test_some::() 108 | } 109 | 110 | #[test] 111 | fn test_i16() { 112 | test_some::() 113 | } 114 | 115 | #[test] 116 | fn test_i32() { 117 | test_some::() 118 | } 119 | 120 | #[test] 121 | fn test_i64() { 122 | test_some::() 123 | } 124 | 125 | #[test] 126 | fn test_i128() { 127 | test_some::() 128 | } 129 | 130 | #[test] 131 | fn test_i256() { 132 | test_some_large::() 133 | } 134 | 135 | #[test] 136 | fn test_f32() { 137 | test_some::() 138 | } 139 | 140 | #[test] 141 | fn test_f64() { 142 | test_some::() 143 | } 144 | 145 | #[test] 146 | fn test_bool() { 147 | test_some::() 148 | } 149 | -------------------------------------------------------------------------------- /components/micromarshal/tests/it/statbuffer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt::Debug; 16 | 17 | use ethnum::{I256, U256}; 18 | 19 | pub trait StatBuffer { 20 | type Buffer: AsMut<[u8]> + AsRef<[u8]> + Copy + Sync + Debug; 21 | fn buffer() -> Self::Buffer; 22 | } 23 | 24 | impl StatBuffer for u8 { 25 | type Buffer = [Self; 1]; 26 | 27 | fn buffer() -> Self::Buffer { 28 | [0; 1] 29 | } 30 | } 31 | 32 | impl StatBuffer for u16 { 33 | type Buffer = [u8; 2]; 34 | 35 | fn buffer() -> Self::Buffer { 36 | [0; 2] 37 | } 38 | } 39 | 40 | impl StatBuffer for u32 { 41 | type Buffer = [u8; 4]; 42 | 43 | fn buffer() -> Self::Buffer { 44 | [0; 4] 45 | } 46 | } 47 | 48 | impl StatBuffer for u64 { 49 | type Buffer = [u8; 8]; 50 | 51 | fn buffer() -> Self::Buffer { 52 | [0; 8] 53 | } 54 | } 55 | 56 | impl StatBuffer for u128 { 57 | type Buffer = [u8; 16]; 58 | 59 | fn buffer() -> Self::Buffer { 60 | [0; 16] 61 | } 62 | } 63 | 64 | impl StatBuffer for U256 { 65 | type Buffer = [u8; 32]; 66 | 67 | fn buffer() -> Self::Buffer { 68 | [0; 32] 69 | } 70 | } 71 | 72 | impl StatBuffer for i8 { 73 | type Buffer = [u8; 1]; 74 | 75 | fn buffer() -> Self::Buffer { 76 | [0; 1] 77 | } 78 | } 79 | 80 | impl StatBuffer for i16 { 81 | type Buffer = [u8; 2]; 82 | 83 | fn buffer() -> Self::Buffer { 84 | [0; 2] 85 | } 86 | } 87 | 88 | impl StatBuffer for i32 { 89 | type Buffer = [u8; 4]; 90 | 91 | fn buffer() -> Self::Buffer { 92 | [0; 4] 93 | } 94 | } 95 | 96 | impl StatBuffer for i64 { 97 | type Buffer = [u8; 8]; 98 | 99 | fn buffer() -> Self::Buffer { 100 | [0; 8] 101 | } 102 | } 103 | 104 | impl StatBuffer for i128 { 105 | type Buffer = [u8; 16]; 106 | 107 | fn buffer() -> Self::Buffer { 108 | [0; 16] 109 | } 110 | } 111 | 112 | impl StatBuffer for I256 { 113 | type Buffer = [u8; 32]; 114 | 115 | fn buffer() -> Self::Buffer { 116 | [0; 32] 117 | } 118 | } 119 | 120 | impl StatBuffer for f32 { 121 | type Buffer = [u8; 4]; 122 | 123 | fn buffer() -> Self::Buffer { 124 | [0; 4] 125 | } 126 | } 127 | 128 | impl StatBuffer for f64 { 129 | type Buffer = [u8; 8]; 130 | 131 | fn buffer() -> Self::Buffer { 132 | [0; 8] 133 | } 134 | } 135 | 136 | impl StatBuffer for bool { 137 | type Buffer = [u8; 1]; 138 | 139 | fn buffer() -> Self::Buffer { 140 | [0; 1] 141 | } 142 | } 143 | 144 | impl StatBuffer for char { 145 | type Buffer = [u8; 4]; 146 | 147 | fn buffer() -> Self::Buffer { 148 | [0; 4] 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /mysql/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "opensrv-mysql" 3 | version = "0.8.0" 4 | authors = ["Databend Authors "] 5 | edition = "2021" 6 | license = "Apache-2.0" 7 | description = "Bindings for emulating a MySQL/MariaDB server." 8 | readme = "README.md" 9 | repository = "https://github.com/datafuselabs/opensrv" 10 | keywords = ["api-bindings", "database", "sql", "mock"] 11 | categories = ["api-bindings", "network-programming", "database-implementations"] 12 | 13 | [lib] 14 | doctest = false 15 | 16 | [features] 17 | default = ["tls"] 18 | tls = ["tokio-rustls", "pin-project-lite"] 19 | 20 | [dependencies] 21 | async-trait = "0.1.52" 22 | byteorder = "1.4.3" 23 | bytes = "1.7.0" 24 | chrono = "0.4.20" 25 | mysql_common = { version = "0.32.0", features = ["chrono"] } 26 | nom = "7.1.0" 27 | tokio = { version = "1.17.0", features = ["io-util", "io-std"] } 28 | tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["ring"] } 29 | pin-project-lite = { version = "0.2.9", optional = true } 30 | 31 | [dev-dependencies] 32 | mysql = "=24.0.0" 33 | mysql_async = "=0.31.0" 34 | tokio = { version = "1.0", features = ["full"] } 35 | futures = "0.3" 36 | rcgen = "0.12" 37 | tempfile = "3.3.0" 38 | native-tls = "0.2.8" 39 | rustls-pemfile = { version = "2.0" } 40 | rustls-pki-types = "1.0" 41 | 42 | [[example]] 43 | name = "serve_auth" 44 | path = "examples/serve_auth.rs" 45 | 46 | [[example]] 47 | name = "serve_one" 48 | path = "examples/serve_one.rs" 49 | 50 | [[example]] 51 | name = "serve_secure" 52 | path = "examples/serve_secure.rs" 53 | -------------------------------------------------------------------------------- /mysql/README.md: -------------------------------------------------------------------------------- 1 | # OpenSrv - MySQL 2 | 3 | **Bindings for emulating a MySQL/MariaDB server.** 4 | 5 | When developing new databases or caching layers, it can be immensely useful to test your system 6 | using existing applications. However, this often requires significant work modifying 7 | applications to use your database over the existing ones. This crate solves that problem by 8 | acting as a MySQL server, and delegating operations such as querying and query execution to 9 | user-defined logic. 10 | 11 | ## Usage 12 | 13 | To start, implement `AsyncMysqlShim` for your backend, and create a `AsyncMysqlIntermediary` over an 14 | instance of your backend and a connection stream. The appropriate methods will be called on 15 | your backend whenever a client issues a `QUERY`, `PREPARE`, or `EXECUTE` command, and you will 16 | have a chance to respond appropriately. For example, to write a shim that always responds to 17 | all commands with a "no results" reply: 18 | 19 | ```rust 20 | use std::io; 21 | use tokio::io::AsyncWrite; 22 | 23 | use opensrv_mysql::*; 24 | use tokio::net::TcpListener; 25 | 26 | struct Backend; 27 | 28 | #[async_trait::async_trait] 29 | impl AsyncMysqlShim for Backend { 30 | type Error = io::Error; 31 | 32 | async fn on_prepare<'a>( 33 | &'a mut self, 34 | _: &'a str, 35 | info: StatementMetaWriter<'a, W>, 36 | ) -> io::Result<()> { 37 | info.reply(42, &[], &[]).await 38 | } 39 | 40 | async fn on_execute<'a>( 41 | &'a mut self, 42 | _: u32, 43 | _: opensrv_mysql::ParamParser<'a>, 44 | results: QueryResultWriter<'a, W>, 45 | ) -> io::Result<()> { 46 | results.completed(OkResponse::default()).await 47 | } 48 | 49 | async fn on_close(&mut self, _: u32) {} 50 | 51 | async fn on_query<'a>( 52 | &'a mut self, 53 | sql: &'a str, 54 | results: QueryResultWriter<'a, W>, 55 | ) -> io::Result<()> { 56 | println!("execute sql {:?}", sql); 57 | results.start(&[]).await?.finish().await 58 | } 59 | } 60 | 61 | #[tokio::main] 62 | async fn main() -> Result<(), Box> { 63 | let listener = TcpListener::bind("0.0.0.0:3306").await?; 64 | 65 | loop { 66 | let (stream, _) = listener.accept().await?; 67 | let (r, w) = stream.into_split(); 68 | tokio::spawn(async move { AsyncMysqlIntermediary::run_on(Backend, r, w).await }); 69 | } 70 | } 71 | ``` 72 | 73 | This example can be exected with: 74 | 75 | ``` 76 | cargo run --example=serve_one 77 | ``` 78 | 79 | More examples can be found [here](examples). 80 | 81 | ## Getting help 82 | 83 | Submit [issues](https://github.com/datafuselabs/opensrv/issues/new/choose) for bug report or asking questions in [discussion](https://github.com/datafuselabs/opensrv/discussions/new?category=q-a). 84 | 85 | ## Credits 86 | 87 | This project is a branch of [jonhoo/msql-srv](https://github.com/jonhoo/msql-srv) and focuses on providing asynchronous support. 88 | 89 | ## License 90 | 91 | Licensed under Apache License, Version 2.0. 92 | -------------------------------------------------------------------------------- /mysql/examples/serve_auth.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! After running this, you should be able to run: 16 | //! 17 | //! ```console 18 | //! $ echo "SELECT * FROM foo" | mysql -h 127.0.0.1 -u default --table 19 | //! $ 20 | //! ``` 21 | 22 | use std::io; 23 | use std::iter; 24 | 25 | use mysql_common as myc; 26 | use opensrv_mysql::*; 27 | use tokio::io::AsyncWrite; 28 | use tokio::net::TcpListener; 29 | 30 | struct Backend; 31 | 32 | #[async_trait::async_trait] 33 | impl AsyncMysqlShim for Backend { 34 | type Error = io::Error; 35 | 36 | async fn on_prepare<'a>( 37 | &'a mut self, 38 | _: &'a str, 39 | info: StatementMetaWriter<'a, W>, 40 | ) -> io::Result<()> { 41 | info.reply(42, &[], &[]).await 42 | } 43 | 44 | async fn on_execute<'a>( 45 | &'a mut self, 46 | _: u32, 47 | _: opensrv_mysql::ParamParser<'a>, 48 | results: QueryResultWriter<'a, W>, 49 | ) -> io::Result<()> { 50 | let resp = OkResponse::default(); 51 | results.completed(resp).await 52 | } 53 | 54 | async fn on_close(&mut self, _: u32) {} 55 | 56 | async fn on_query<'a>( 57 | &'a mut self, 58 | sql: &'a str, 59 | results: QueryResultWriter<'a, W>, 60 | ) -> io::Result<()> { 61 | println!("execute sql {:?}", sql); 62 | 63 | let cols = &[Column { 64 | table: String::new(), 65 | column: "abc".to_string(), 66 | coltype: myc::constants::ColumnType::MYSQL_TYPE_LONG, 67 | colflags: myc::constants::ColumnFlags::UNSIGNED_FLAG, 68 | }]; 69 | 70 | let mut w = results.start(cols).await?; 71 | w.write_row(iter::once(67108864u32)).await?; 72 | w.write_row(iter::once(167108864u32)).await?; 73 | 74 | w.finish_with_info("ExtraInfo").await 75 | } 76 | 77 | /// authenticate method for the specified plugin 78 | async fn authenticate( 79 | &self, 80 | _auth_plugin: &str, 81 | username: &[u8], 82 | _salt: &[u8], 83 | _auth_data: &[u8], 84 | ) -> bool { 85 | username == "default".as_bytes() 86 | } 87 | 88 | fn version(&self) -> String { 89 | // 5.1.10 because that's what Ruby's ActiveRecord requires 90 | "5.1.10-alpha-msql-proxy".to_string() 91 | } 92 | 93 | fn connect_id(&self) -> u32 { 94 | u32::from_le_bytes([0x08, 0x00, 0x00, 0x00]) 95 | } 96 | 97 | fn default_auth_plugin(&self) -> &str { 98 | "mysql_native_password" 99 | } 100 | 101 | async fn auth_plugin_for_username(&self, _user: &[u8]) -> &str { 102 | "mysql_native_password" 103 | } 104 | 105 | fn salt(&self) -> [u8; 20] { 106 | let bs = ";X,po_k}>o6^Wz!/kM}N".as_bytes(); 107 | let mut scramble: [u8; 20] = [0; 20]; 108 | for i in 0..20 { 109 | scramble[i] = bs[i]; 110 | if scramble[i] == b'\0' || scramble[i] == b'$' { 111 | scramble[i] += 1; 112 | } 113 | } 114 | scramble 115 | } 116 | } 117 | 118 | #[tokio::main] 119 | async fn main() -> Result<(), Box> { 120 | let listener = TcpListener::bind("0.0.0.0:3306").await?; 121 | 122 | loop { 123 | let (stream, _) = listener.accept().await?; 124 | let (r, w) = stream.into_split(); 125 | tokio::spawn(async move { AsyncMysqlIntermediary::run_on(Backend, r, w).await }); 126 | } 127 | } 128 | 129 | #[test] 130 | fn it_works() { 131 | let c: u8 = b'\0'; 132 | let d: u8 = 0 as u8; 133 | let e: u8 = 0x00; 134 | 135 | assert_eq!(c, d); 136 | assert_eq!(e, d); 137 | } 138 | -------------------------------------------------------------------------------- /mysql/examples/serve_one.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! After running this, you should be able to run: 16 | //! 17 | //! ```console 18 | //! $ echo "SELECT * FROM foo" | mysql -h 127.0.0.1 --table 19 | //! ``` 20 | 21 | use std::io; 22 | use tokio::io::AsyncWrite; 23 | 24 | use opensrv_mysql::*; 25 | use tokio::net::TcpListener; 26 | 27 | struct Backend; 28 | 29 | #[async_trait::async_trait] 30 | impl AsyncMysqlShim for Backend { 31 | type Error = io::Error; 32 | 33 | async fn on_prepare<'a>( 34 | &'a mut self, 35 | _: &'a str, 36 | info: StatementMetaWriter<'a, W>, 37 | ) -> io::Result<()> { 38 | info.reply(42, &[], &[]).await 39 | } 40 | 41 | async fn on_execute<'a>( 42 | &'a mut self, 43 | _: u32, 44 | _: opensrv_mysql::ParamParser<'a>, 45 | results: QueryResultWriter<'a, W>, 46 | ) -> io::Result<()> { 47 | results.completed(OkResponse::default()).await 48 | } 49 | 50 | async fn on_close(&mut self, _: u32) {} 51 | 52 | async fn on_query<'a>( 53 | &'a mut self, 54 | sql: &'a str, 55 | results: QueryResultWriter<'a, W>, 56 | ) -> io::Result<()> { 57 | println!("execute sql {:?}", sql); 58 | results.start(&[]).await?.finish().await 59 | } 60 | } 61 | 62 | #[tokio::main] 63 | async fn main() -> Result<(), Box> { 64 | let listener = TcpListener::bind("0.0.0.0:3306").await?; 65 | 66 | loop { 67 | let (stream, _) = listener.accept().await?; 68 | let (r, w) = stream.into_split(); 69 | tokio::spawn(async move { AsyncMysqlIntermediary::run_on(Backend, r, w).await }); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /mysql/examples/serve_secure.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! After running this, you should be able to run: 16 | //! 17 | //! ```console 18 | //! $ echo "SELECT * FROM foo" | mysql -h 127.0.0.1 --table --ssl-mode=REQUIRED 19 | //! ``` 20 | 21 | #[cfg(feature = "tls")] 22 | mod tls { 23 | 24 | use rustls_pemfile::{certs, pkcs8_private_keys}; 25 | use rustls_pki_types::{CertificateDer, PrivateKeyDer}; 26 | use std::{ 27 | fs::File, 28 | io::{self, BufReader, ErrorKind}, 29 | sync::Arc, 30 | }; 31 | use tokio::io::AsyncWrite; 32 | use tokio_rustls::rustls::ServerConfig; 33 | 34 | use opensrv_mysql::*; 35 | use tokio::net::TcpListener; 36 | 37 | struct Backend; 38 | 39 | #[async_trait::async_trait] 40 | impl AsyncMysqlShim for Backend { 41 | type Error = io::Error; 42 | 43 | async fn on_prepare<'a>( 44 | &'a mut self, 45 | _: &'a str, 46 | info: StatementMetaWriter<'a, W>, 47 | ) -> io::Result<()> { 48 | info.reply(42, &[], &[]).await 49 | } 50 | 51 | async fn on_execute<'a>( 52 | &'a mut self, 53 | _: u32, 54 | _: opensrv_mysql::ParamParser<'a>, 55 | results: QueryResultWriter<'a, W>, 56 | ) -> io::Result<()> { 57 | results.completed(OkResponse::default()).await 58 | } 59 | 60 | async fn on_close(&mut self, _: u32) {} 61 | 62 | async fn on_query<'a>( 63 | &'a mut self, 64 | sql: &'a str, 65 | results: QueryResultWriter<'a, W>, 66 | ) -> io::Result<()> { 67 | println!("execute sql {:?}", sql); 68 | results.start(&[]).await?.finish().await 69 | } 70 | } 71 | 72 | fn setup_tls() -> Result { 73 | let cert = certs(&mut BufReader::new(File::open( 74 | "mysql/tests/ssl/server.crt", 75 | )?)) 76 | .collect::, io::Error>>()?; 77 | 78 | let key = pkcs8_private_keys(&mut BufReader::new(File::open( 79 | "mysql/tests/ssl/server.key", 80 | )?)) 81 | .map(|key| key.map(PrivateKeyDer::from)) 82 | .collect::, io::Error>>()? 83 | .remove(0); 84 | 85 | let config = ServerConfig::builder() 86 | .with_no_client_auth() 87 | .with_single_cert(cert, key) 88 | .map_err(|err| io::Error::new(ErrorKind::InvalidInput, err))?; 89 | 90 | Ok(config) 91 | } 92 | 93 | pub async fn main() -> Result<(), Box> { 94 | let listener = TcpListener::bind("0.0.0.0:3306").await?; 95 | 96 | loop { 97 | let (stream, _) = listener.accept().await?; 98 | let (mut r, mut w) = stream.into_split(); 99 | 100 | tokio::spawn(async move { 101 | let tls_config = setup_tls().unwrap(); 102 | let tls_config = Arc::new(tls_config); 103 | let mut shim = Backend; 104 | let ops = IntermediaryOptions::default(); 105 | 106 | let (is_ssl, init_params) = opensrv_mysql::AsyncMysqlIntermediary::init_before_ssl( 107 | &mut shim, 108 | &mut r, 109 | &mut w, 110 | &Some(tls_config.clone()), 111 | ) 112 | .await 113 | .unwrap(); 114 | 115 | if is_ssl { 116 | opensrv_mysql::secure_run_with_options(shim, w, ops, tls_config, init_params) 117 | .await 118 | } else { 119 | opensrv_mysql::plain_run_with_options(shim, w, ops, init_params).await 120 | } 121 | }); 122 | } 123 | } 124 | } 125 | 126 | #[tokio::main] 127 | async fn main() -> Result<(), Box> { 128 | #[cfg(feature = "tls")] 129 | tls::main().await?; 130 | println!(" cargo run --example serve_secure --features tls "); 131 | Ok(()) 132 | } 133 | -------------------------------------------------------------------------------- /mysql/src/packet_writer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use byteorder::{ByteOrder, LittleEndian}; 16 | use std::io; 17 | use std::io::prelude::*; 18 | use std::io::IoSlice; 19 | 20 | use crate::U24_MAX; 21 | use tokio::io::{AsyncWrite, AsyncWriteExt}; 22 | 23 | /// The writer of mysql packet. 24 | /// - behaves as a sync writer, while build the packet 25 | /// so that trivial async writes could be avoided 26 | /// - behaves like a async writer, while writing data to the output stream 27 | pub struct PacketWriter { 28 | packet_builder: PacketBuilder, 29 | output_stream: W, 30 | } 31 | 32 | // exports the internal builder as sync Write 33 | impl Write for PacketWriter { 34 | fn write(&mut self, buf: &[u8]) -> io::Result { 35 | self.packet_builder.write(buf) 36 | } 37 | 38 | fn flush(&mut self) -> io::Result<()> { 39 | self.packet_builder.flush() 40 | } 41 | } 42 | 43 | impl PacketWriter { 44 | pub fn new(output_stream: W) -> Self { 45 | Self { 46 | packet_builder: PacketBuilder::new(), 47 | output_stream, 48 | } 49 | } 50 | pub fn set_seq(&mut self, seq: u8) { 51 | self.packet_builder.set_seq(seq) 52 | } 53 | } 54 | 55 | const PACKET_HEADER_SIZE: usize = 4; 56 | impl PacketWriter { 57 | /// Build packet(s) and write them to the output stream 58 | pub async fn end_packet(&mut self) -> io::Result<()> { 59 | let builder = &mut self.packet_builder; 60 | if !builder.is_empty() { 61 | let raw_packet = builder.take_buffer(); 62 | 63 | // split the rww buffer at the boundary of size U24_MAX 64 | let chunks = raw_packet.chunks(U24_MAX); 65 | let mut header = [0; PACKET_HEADER_SIZE]; 66 | for chunk in chunks { 67 | // prepare the header 68 | LittleEndian::write_u24(&mut header, chunk.len() as u32); 69 | header[3] = builder.seq(); 70 | builder.increase_seq(); 71 | 72 | // write out the header and payload. 73 | // 74 | // depends on the AsyncWrite provided, this may trigger 75 | // real system call or not (for example, if AsyncWrite is buffered stream) 76 | let written = self 77 | .output_stream 78 | .write_vectored(&[IoSlice::new(&header), IoSlice::new(chunk)]) 79 | .await?; 80 | 81 | // if write buffer is not drained, fall back to write_all 82 | if written != PACKET_HEADER_SIZE + chunk.len() { 83 | let remaining: Vec = header 84 | .iter() 85 | .chain(chunk.iter()) 86 | .skip(written) 87 | .cloned() 88 | .collect(); 89 | self.output_stream.write_all(&remaining).await? 90 | } 91 | } 92 | Ok(()) 93 | } else { 94 | Ok(()) 95 | } 96 | } 97 | 98 | pub async fn flush_all(&mut self) -> io::Result<()> { 99 | self.output_stream.flush().await 100 | } 101 | } 102 | 103 | // Builder that exports as sync `Write`, so that trivial scattered async writes 104 | // could be avoided during constructing the packet, especially the writes in mod [writers] 105 | struct PacketBuilder { 106 | buffer: Vec, 107 | seq: u8, 108 | } 109 | 110 | impl Write for PacketBuilder { 111 | fn write(&mut self, buf: &[u8]) -> io::Result { 112 | // Here we take them all, and split them into raw packets later in `end_packet` if the size 113 | // of buffer is larger than max payload size (16MB) 114 | self.buffer.extend(buf); 115 | Ok(buf.len()) 116 | } 117 | 118 | fn flush(&mut self) -> io::Result<()> { 119 | Ok(()) 120 | } 121 | } 122 | 123 | impl PacketBuilder { 124 | pub fn new() -> Self { 125 | PacketBuilder { 126 | buffer: vec![], 127 | seq: 0, 128 | } 129 | } 130 | 131 | fn is_empty(&self) -> bool { 132 | self.buffer.is_empty() 133 | } 134 | 135 | fn take_buffer(&mut self) -> Vec { 136 | std::mem::take(&mut self.buffer) 137 | } 138 | 139 | fn set_seq(&mut self, seq: u8) { 140 | self.seq = seq; 141 | } 142 | 143 | fn increase_seq(&mut self) { 144 | self.seq = self.seq.wrapping_add(1); 145 | } 146 | 147 | fn seq(&self) -> u8 { 148 | self.seq 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /mysql/src/params.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::collections::HashMap; 16 | 17 | use crate::myc; 18 | use crate::{StatementData, Value}; 19 | 20 | /// A `ParamParser` decodes query parameters included in a client's `EXECUTE` command given 21 | /// type information for the expected parameters. 22 | /// 23 | /// Users should invoke [`iter`](struct.ParamParser.html#method.iter) method to iterate over the 24 | /// provided parameters. 25 | pub struct ParamParser<'a> { 26 | pub(crate) params: u16, 27 | pub(crate) bytes: &'a [u8], 28 | pub(crate) long_data: &'a HashMap>, 29 | pub(crate) bound_types: &'a mut Vec<(myc::constants::ColumnType, bool)>, 30 | } 31 | 32 | impl<'a> ParamParser<'a> { 33 | pub(crate) fn new(input: &'a [u8], stmt: &'a mut StatementData) -> Self { 34 | ParamParser { 35 | params: stmt.params, 36 | bytes: input, 37 | long_data: &stmt.long_data, 38 | bound_types: &mut stmt.bound_types, 39 | } 40 | } 41 | } 42 | 43 | impl<'a> IntoIterator for ParamParser<'a> { 44 | type IntoIter = Params<'a>; 45 | type Item = ParamValue<'a>; 46 | fn into_iter(self) -> Params<'a> { 47 | Params { 48 | params: self.params, 49 | input: self.bytes, 50 | nullmap: None, 51 | col: 0, 52 | long_data: self.long_data, 53 | bound_types: self.bound_types, 54 | } 55 | } 56 | } 57 | 58 | /// An iterator over parameters provided by a client in an `EXECUTE` command. 59 | pub struct Params<'a> { 60 | params: u16, 61 | input: &'a [u8], 62 | nullmap: Option<&'a [u8]>, 63 | col: u16, 64 | long_data: &'a HashMap>, 65 | bound_types: &'a mut Vec<(myc::constants::ColumnType, bool)>, 66 | } 67 | 68 | /// A single parameter value provided by a client when issuing an `EXECUTE` command. 69 | pub struct ParamValue<'a> { 70 | /// The value provided for this parameter. 71 | pub value: Value<'a>, 72 | /// The column type assigned to this parameter. 73 | pub coltype: myc::constants::ColumnType, 74 | } 75 | 76 | impl<'a> Iterator for Params<'a> { 77 | type Item = ParamValue<'a>; 78 | fn next(&mut self) -> Option { 79 | if self.nullmap.is_none() { 80 | let nullmap_len = (self.params as usize + 7) / 8; 81 | let (nullmap, rest) = self.input.split_at(nullmap_len); 82 | self.nullmap = Some(nullmap); 83 | self.input = rest; 84 | 85 | if !rest.is_empty() && rest[0] != 0x00 { 86 | let (typmap, rest) = rest[1..].split_at(2 * self.params as usize); 87 | self.bound_types.clear(); 88 | for i in 0..self.params as usize { 89 | self.bound_types.push(( 90 | myc::constants::ColumnType::try_from(typmap[2 * i]).unwrap_or_else(|e| { 91 | panic!("bad column type 0x{:x}: {}", typmap[2 * i], e) 92 | }), 93 | (typmap[2 * i + 1] & 128) != 0, 94 | )); 95 | } 96 | self.input = rest; 97 | } 98 | } 99 | 100 | if self.col >= self.params { 101 | return None; 102 | } 103 | let pt = &self.bound_types[self.col as usize]; 104 | 105 | // https://web.archive.org/web/20170404144156/https://dev.mysql.com/doc/internals/en/null-bitmap.html 106 | // NULL-bitmap-byte = ((field-pos + offset) / 8) 107 | // NULL-bitmap-bit = ((field-pos + offset) % 8) 108 | if let Some(nullmap) = self.nullmap { 109 | let byte = self.col as usize / 8; 110 | if byte >= nullmap.len() { 111 | return None; 112 | } 113 | if (nullmap[byte] & 1u8 << (self.col % 8)) != 0 { 114 | self.col += 1; 115 | return Some(ParamValue { 116 | value: Value::null(), 117 | coltype: pt.0, 118 | }); 119 | } 120 | } else { 121 | unreachable!(); 122 | } 123 | 124 | let v = if let Some(data) = self.long_data.get(&self.col) { 125 | Value::bytes(&data[..]) 126 | } else { 127 | Value::parse_from(&mut self.input, pt.0, pt.1).unwrap() 128 | }; 129 | self.col += 1; 130 | Some(ParamValue { 131 | value: v, 132 | coltype: pt.0, 133 | }) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /mysql/src/tests/commands.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::Cursor; 16 | 17 | use crate::commands::*; 18 | use crate::myc::constants::{CapabilityFlags, UTF8_GENERAL_CI}; 19 | use crate::packet_reader::PacketReader; 20 | 21 | #[test] 22 | fn it_parses_handshake() { 23 | let data = &[ 24 | 0x5b, 0x00, 0x00, 0x01, 0x8d, 0xa6, 0xff, 0x09, 0x00, 0x00, 0x00, 0x01, 0x21, 0x00, 0x00, 25 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 26 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x00, 0x14, 27 | 0xf7, 0xd1, 0x6c, 0xe9, 0x0d, 0x2f, 0x34, 0xb0, 0x2f, 0xd8, 0x1d, 0x18, 0xc7, 0xa4, 0xe8, 28 | 0x98, 0x97, 0x67, 0xeb, 0xad, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x00, 0x6d, 0x79, 29 | 0x73, 0x71, 0x6c, 0x5f, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x73, 0x73, 30 | 0x77, 0x6f, 0x72, 0x64, 0x00, 31 | ]; 32 | 33 | let r = Cursor::new(&data[..]); 34 | let mut pr = PacketReader::new(r); 35 | let (_, p) = pr.next().unwrap().unwrap(); 36 | let (_, handshake) = client_handshake(&p, false).unwrap(); 37 | println!("{:?}", handshake); 38 | assert!(handshake 39 | .capabilities 40 | .contains(CapabilityFlags::CLIENT_LONG_PASSWORD)); 41 | assert!(handshake 42 | .capabilities 43 | .contains(CapabilityFlags::CLIENT_MULTI_RESULTS)); 44 | assert!(handshake 45 | .capabilities 46 | .contains(CapabilityFlags::CLIENT_CONNECT_WITH_DB)); 47 | assert!(handshake 48 | .capabilities 49 | .contains(CapabilityFlags::CLIENT_DEPRECATE_EOF)); 50 | assert_eq!(handshake.collation, UTF8_GENERAL_CI); 51 | assert_eq!(handshake.username, Some(b"default"[..].to_vec())); 52 | assert_eq!(handshake.maxps, 16777216); 53 | } 54 | 55 | #[test] 56 | fn it_parses_request() { 57 | let data = &[ 58 | 0x21, 0x00, 0x00, 0x00, 0x03, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x40, 0x40, 0x76, 59 | 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x20, 60 | 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x20, 0x31, 61 | ]; 62 | let r = Cursor::new(&data[..]); 63 | let mut pr = PacketReader::new(r); 64 | let (_, p) = pr.next().unwrap().unwrap(); 65 | let (_, cmd) = parse(&p).unwrap(); 66 | assert_eq!( 67 | cmd, 68 | Command::Query(&b"select @@version_comment limit 1"[..]) 69 | ); 70 | } 71 | 72 | #[test] 73 | fn it_handles_list_fields() { 74 | // mysql_list_fields (CommandByte::COM_FIELD_LIST / 0x04) has been deprecated in mysql 5.7 and will be removed 75 | // in a future version. The mysql command line tool issues one of these commands after 76 | // switching databases with USE . 77 | let data = &[ 78 | 0x21, 0x00, 0x00, 0x00, 0x04, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x20, 0x40, 0x40, 0x76, 79 | 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x20, 80 | 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x20, 0x31, 81 | ]; 82 | let r = Cursor::new(&data[..]); 83 | let mut pr = PacketReader::new(r); 84 | let (_, p) = pr.next().unwrap().unwrap(); 85 | let (_, cmd) = parse(&p).unwrap(); 86 | assert_eq!( 87 | cmd, 88 | Command::ListFields(&b"select @@version_comment limit 1"[..]) 89 | ); 90 | } 91 | -------------------------------------------------------------------------------- /mysql/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod commands; 16 | mod packet; 17 | mod value; 18 | -------------------------------------------------------------------------------- /mysql/src/tests/packet.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::packet_reader::*; 16 | use crate::U24_MAX; 17 | 18 | #[test] 19 | fn test_one_ping() { 20 | assert_eq!( 21 | onepacket((&[0x01, 0, 0, 0, 0x10][..]).into()).unwrap().1, 22 | (0, (&[0x10][..]).into()) 23 | ); 24 | } 25 | 26 | #[test] 27 | fn test_ping() { 28 | let p = packet((&[0x01, 0, 0, 0, 0x10][..]).into()).unwrap().1; 29 | assert_eq!(p.0, 0); 30 | assert_eq!(&*p.1, &[0x10][..]); 31 | } 32 | 33 | #[test] 34 | fn test_long_exact() { 35 | let mut data = vec![0xff, 0xff, 0xff, 0]; 36 | data.extend(&[0; U24_MAX][..]); 37 | data.push(0x00); 38 | data.push(0x00); 39 | data.push(0x00); 40 | data.push(1); 41 | 42 | let (rest, p) = packet((&data[..]).into()).unwrap(); 43 | assert!(rest.is_empty()); 44 | assert_eq!(p.0, 1); 45 | assert_eq!(p.1.len(), U24_MAX); 46 | assert_eq!(&*p.1, &[0; U24_MAX][..]); 47 | } 48 | 49 | #[test] 50 | fn test_long_more() { 51 | let mut data = vec![0xff, 0xff, 0xff, 0]; 52 | data.extend(&[0; U24_MAX][..]); 53 | data.push(0x01); 54 | data.push(0x00); 55 | data.push(0x00); 56 | data.push(1); 57 | data.push(0x10); 58 | 59 | let (rest, p) = packet((&data[..]).into()).unwrap(); 60 | assert!(rest.is_empty()); 61 | assert_eq!(p.0, 1); 62 | assert_eq!(p.1.len(), U24_MAX + 1); 63 | assert_eq!(&p.1[..U24_MAX], &[0; U24_MAX][..]); 64 | assert_eq!(&p.1[U24_MAX..], &[0x10]); 65 | } 66 | -------------------------------------------------------------------------------- /mysql/src/tests/value/decode.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::myc; 16 | use crate::value::Value; 17 | use crate::{Column, ColumnFlags, ColumnType}; 18 | use chrono::{self, TimeZone}; 19 | use myc::proto::MySerialize; 20 | use std::time; 21 | 22 | macro_rules! rt { 23 | ($name:ident, $t:ty, $v:expr, $ct:expr) => { 24 | rt!($name, $t, $v, $ct, false); 25 | }; 26 | ($name:ident, $t:ty, $v:expr, $ct:expr, $sig:expr) => { 27 | #[test] 28 | fn $name() { 29 | let mut data = Vec::new(); 30 | let mut col = Column { 31 | table: String::new(), 32 | column: String::new(), 33 | coltype: $ct, 34 | colflags: ColumnFlags::empty(), 35 | }; 36 | 37 | if !$sig { 38 | col.colflags.insert(ColumnFlags::UNSIGNED_FLAG); 39 | } 40 | 41 | let v: $t = $v; 42 | myc::value::Value::from(v).serialize(&mut data); 43 | assert_eq!( 44 | Into::<$t>::into(Value::parse_from(&mut &data[..], $ct, !$sig).unwrap()), 45 | v 46 | ); 47 | } 48 | }; 49 | } 50 | 51 | rt!(u8_one, u8, 1, ColumnType::MYSQL_TYPE_TINY, false); 52 | rt!(i8_one, i8, 1, ColumnType::MYSQL_TYPE_TINY, true); 53 | rt!(u8_one_short, u8, 1, ColumnType::MYSQL_TYPE_SHORT, false); 54 | rt!(i8_one_short, i8, 1, ColumnType::MYSQL_TYPE_SHORT, true); 55 | rt!(u8_one_long, u8, 1, ColumnType::MYSQL_TYPE_LONG, false); 56 | rt!(i8_one_long, i8, 1, ColumnType::MYSQL_TYPE_LONG, true); 57 | rt!( 58 | u8_one_longlong, 59 | u8, 60 | 1, 61 | ColumnType::MYSQL_TYPE_LONGLONG, 62 | false 63 | ); 64 | rt!( 65 | i8_one_longlong, 66 | i8, 67 | 1, 68 | ColumnType::MYSQL_TYPE_LONGLONG, 69 | true 70 | ); 71 | rt!(u16_one, u16, 1, ColumnType::MYSQL_TYPE_SHORT, false); 72 | rt!(i16_one, i16, 1, ColumnType::MYSQL_TYPE_SHORT, true); 73 | rt!(u16_one_long, u16, 1, ColumnType::MYSQL_TYPE_LONG, false); 74 | rt!(i16_one_long, i16, 1, ColumnType::MYSQL_TYPE_LONG, true); 75 | rt!( 76 | u16_one_longlong, 77 | u16, 78 | 1, 79 | ColumnType::MYSQL_TYPE_LONGLONG, 80 | false 81 | ); 82 | rt!( 83 | i16_one_longlong, 84 | i16, 85 | 1, 86 | ColumnType::MYSQL_TYPE_LONGLONG, 87 | true 88 | ); 89 | rt!(u32_one_long, u32, 1, ColumnType::MYSQL_TYPE_LONG, false); 90 | rt!(i32_one_long, i32, 1, ColumnType::MYSQL_TYPE_LONG, true); 91 | rt!( 92 | u32_one_longlong, 93 | u32, 94 | 1, 95 | ColumnType::MYSQL_TYPE_LONGLONG, 96 | false 97 | ); 98 | rt!( 99 | i32_one_longlong, 100 | i32, 101 | 1, 102 | ColumnType::MYSQL_TYPE_LONGLONG, 103 | true 104 | ); 105 | rt!(u64_one, u64, 1, ColumnType::MYSQL_TYPE_LONGLONG, false); 106 | rt!(i64_one, i64, 1, ColumnType::MYSQL_TYPE_LONGLONG, true); 107 | 108 | rt!(f32_one_float, f32, 1.0, ColumnType::MYSQL_TYPE_FLOAT, false); 109 | rt!(f64_one, f64, 1.0, ColumnType::MYSQL_TYPE_DOUBLE, false); 110 | 111 | rt!(u8_max, u8, u8::MAX, ColumnType::MYSQL_TYPE_TINY, false); 112 | rt!(i8_max, i8, i8::MAX, ColumnType::MYSQL_TYPE_TINY, true); 113 | rt!(u16_max, u16, u16::MAX, ColumnType::MYSQL_TYPE_SHORT, false); 114 | rt!(i16_max, i16, i16::MAX, ColumnType::MYSQL_TYPE_SHORT, true); 115 | rt!(u32_max, u32, u32::MAX, ColumnType::MYSQL_TYPE_LONG, false); 116 | rt!(i32_max, i32, i32::MAX, ColumnType::MYSQL_TYPE_LONG, true); 117 | rt!( 118 | u64_max, 119 | u64, 120 | u64::MAX, 121 | ColumnType::MYSQL_TYPE_LONGLONG, 122 | false 123 | ); 124 | rt!( 125 | i64_max, 126 | i64, 127 | i64::MAX, 128 | ColumnType::MYSQL_TYPE_LONGLONG, 129 | true 130 | ); 131 | 132 | rt!( 133 | time, 134 | chrono::NaiveDate, 135 | chrono::Local::now().date_naive(), 136 | ColumnType::MYSQL_TYPE_DATE 137 | ); 138 | rt!( 139 | datetime, 140 | chrono::NaiveDateTime, 141 | chrono::Utc 142 | .with_ymd_and_hms(1989, 12, 7, 8, 0, 4) 143 | .unwrap() 144 | .naive_utc(), 145 | ColumnType::MYSQL_TYPE_DATETIME 146 | ); 147 | rt!( 148 | dur, 149 | time::Duration, 150 | time::Duration::from_secs(1893), 151 | ColumnType::MYSQL_TYPE_TIME 152 | ); 153 | rt!( 154 | dur_zero, 155 | time::Duration, 156 | time::Duration::from_secs(0), 157 | ColumnType::MYSQL_TYPE_TIME 158 | ); 159 | rt!( 160 | bytes, 161 | &[u8], 162 | &[0x42, 0x00, 0x1a], 163 | ColumnType::MYSQL_TYPE_BLOB 164 | ); 165 | rt!(string, &str, "foobar", ColumnType::MYSQL_TYPE_STRING); 166 | -------------------------------------------------------------------------------- /mysql/src/tests/value/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod decode; 16 | mod encode; 17 | -------------------------------------------------------------------------------- /mysql/src/tls.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::io::IoSlice; 16 | use std::pin::Pin; 17 | use std::sync::Arc; 18 | use std::task::{Context, Poll}; 19 | 20 | use pin_project_lite::pin_project; 21 | use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf, ReadHalf, WriteHalf}; 22 | use tokio_rustls::server::TlsStream; 23 | use tokio_rustls::{rustls::ServerConfig, TlsAcceptor}; 24 | 25 | use crate::commands::ClientHandshake; 26 | use crate::myc::constants::CapabilityFlags; 27 | use crate::packet_reader::PacketReader; 28 | use crate::packet_writer::PacketWriter; 29 | use crate::{AsyncMysqlIntermediary, AsyncMysqlShim, IntermediaryOptions}; 30 | 31 | pub async fn plain_run_with_options( 32 | shim: B, 33 | writer: W, 34 | opts: IntermediaryOptions, 35 | init_params: (ClientHandshake, u8, CapabilityFlags, PacketReader), 36 | ) -> Result<(), B::Error> 37 | where 38 | B: AsyncMysqlShim + Send + Sync, 39 | R: AsyncRead + Send + Unpin, 40 | W: AsyncWrite + Send + Unpin, 41 | { 42 | let (handshake, seq, client_capabilities, reader) = init_params; 43 | let reader = PacketReader::new(reader); 44 | let writer = PacketWriter::new(writer); 45 | 46 | let process_use_statement_on_query = opts.process_use_statement_on_query; 47 | let reject_connection_on_dbname_absence = opts.reject_connection_on_dbname_absence; 48 | let mut mi = AsyncMysqlIntermediary { 49 | client_capabilities, 50 | process_use_statement_on_query, 51 | reject_connection_on_dbname_absence, 52 | shim, 53 | reader, 54 | writer, 55 | }; 56 | mi.init_after_ssl(handshake, seq).await?; 57 | mi.run().await 58 | } 59 | 60 | pub async fn secure_run_with_options( 61 | shim: B, 62 | writer: W, 63 | opts: IntermediaryOptions, 64 | tls_config: Arc, 65 | init_params: (ClientHandshake, u8, CapabilityFlags, PacketReader), 66 | ) -> Result<(), B::Error> 67 | where 68 | B: AsyncMysqlShim, W>>>> + Send + Sync, 69 | R: AsyncRead + Send + Unpin, 70 | W: AsyncWrite + Send + Unpin, 71 | { 72 | let (handshake, seq, client_capabilities, reader) = init_params; 73 | let (reader, writer) = switch_to_tls(tls_config, reader, writer).await?; 74 | let reader = PacketReader::new(reader); 75 | let writer = PacketWriter::new(writer); 76 | 77 | let process_use_statement_on_query = opts.process_use_statement_on_query; 78 | let reject_connection_on_dbname_absence = opts.reject_connection_on_dbname_absence; 79 | let mut mi = AsyncMysqlIntermediary { 80 | client_capabilities, 81 | process_use_statement_on_query, 82 | reject_connection_on_dbname_absence, 83 | shim, 84 | reader, 85 | writer, 86 | }; 87 | mi.init_after_ssl(handshake, seq).await?; 88 | mi.run().await 89 | } 90 | 91 | pub async fn switch_to_tls( 92 | config: Arc, 93 | reader: R, 94 | writer: W, 95 | ) -> std::io::Result<( 96 | ReadHalf>>, 97 | WriteHalf>>, 98 | )> { 99 | let stream = Duplex::new(reader, writer); 100 | let acceptor = TlsAcceptor::from(config); 101 | let stream = acceptor.accept(stream).await?; 102 | let (r, w) = tokio::io::split(stream); 103 | Ok((r, w)) 104 | } 105 | 106 | pin_project! { 107 | #[derive(Clone, Debug)] 108 | pub struct Duplex { 109 | #[pin] 110 | reader: R, 111 | #[pin] 112 | writer: W, 113 | } 114 | } 115 | 116 | impl Duplex { 117 | pub fn new(reader: R, writer: W) -> Self { 118 | Self { reader, writer } 119 | } 120 | } 121 | 122 | impl AsyncRead for Duplex { 123 | fn poll_read( 124 | self: Pin<&mut Self>, 125 | cx: &mut Context<'_>, 126 | buf: &mut ReadBuf<'_>, 127 | ) -> Poll> { 128 | AsyncRead::poll_read(self.project().reader, cx, buf) 129 | } 130 | } 131 | 132 | impl AsyncWrite for Duplex { 133 | fn poll_write( 134 | self: Pin<&mut Self>, 135 | cx: &mut Context<'_>, 136 | buf: &[u8], 137 | ) -> Poll> { 138 | AsyncWrite::poll_write(self.project().writer, cx, buf) 139 | } 140 | 141 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 142 | AsyncWrite::poll_flush(self.project().writer, cx) 143 | } 144 | 145 | fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 146 | AsyncWrite::poll_shutdown(self.project().writer, cx) 147 | } 148 | 149 | fn poll_write_vectored( 150 | self: Pin<&mut Self>, 151 | cx: &mut Context<'_>, 152 | bufs: &[IoSlice<'_>], 153 | ) -> Poll> { 154 | AsyncWrite::poll_write_vectored(self.project().writer, cx, bufs) 155 | } 156 | 157 | fn is_write_vectored(&self) -> bool { 158 | AsyncWrite::is_write_vectored(&self.writer) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /mysql/src/value/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub(crate) mod decode; 16 | mod encode; 17 | 18 | pub use self::decode::{Value, ValueInner}; 19 | pub use self::encode::ToMysqlValue; 20 | -------------------------------------------------------------------------------- /mysql/tests/it/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod r#async; 16 | mod secure; 17 | -------------------------------------------------------------------------------- /mysql/tests/it/secure.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Datafuse Labs. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! After running this, you should be able to run: 16 | //! 17 | //! ```console 18 | //! $ echo "SELECT * FROM foo" | mysql -h 127.0.0.1 --table --ssl-mode=REQUIRED 19 | //! ``` 20 | 21 | #[cfg(feature = "tls")] 22 | mod tls { 23 | 24 | use rustls_pemfile::{certs, pkcs8_private_keys}; 25 | use rustls_pki_types::{CertificateDer, PrivateKeyDer}; 26 | use std::{ 27 | fs::File, 28 | io::{self, BufReader, ErrorKind}, 29 | sync::Arc, 30 | }; 31 | use tokio::io::AsyncWrite; 32 | use tokio_rustls::rustls::ServerConfig; 33 | 34 | use opensrv_mysql::*; 35 | use tokio::net::TcpListener; 36 | 37 | struct Backend; 38 | 39 | #[async_trait::async_trait] 40 | impl AsyncMysqlShim for Backend { 41 | type Error = io::Error; 42 | 43 | async fn on_prepare<'a>( 44 | &'a mut self, 45 | _: &'a str, 46 | info: StatementMetaWriter<'a, W>, 47 | ) -> io::Result<()> { 48 | info.reply(42, &[], &[]).await 49 | } 50 | 51 | async fn on_execute<'a>( 52 | &'a mut self, 53 | _: u32, 54 | _: opensrv_mysql::ParamParser<'a>, 55 | results: QueryResultWriter<'a, W>, 56 | ) -> io::Result<()> { 57 | results.completed(OkResponse::default()).await 58 | } 59 | 60 | async fn on_close(&mut self, _: u32) {} 61 | 62 | async fn on_query<'a>( 63 | &'a mut self, 64 | sql: &'a str, 65 | results: QueryResultWriter<'a, W>, 66 | ) -> io::Result<()> { 67 | println!("execute sql {:?}", sql); 68 | results.start(&[]).await?.finish().await 69 | } 70 | } 71 | 72 | fn setup_tls() -> Result { 73 | let cert = certs(&mut BufReader::new(File::open("tests/ssl/server.crt")?)) 74 | .collect::, io::Error>>()?; 75 | 76 | let key = pkcs8_private_keys(&mut BufReader::new(File::open("tests/ssl/server.key")?)) 77 | .map(|key| key.map(PrivateKeyDer::from)) 78 | .collect::, io::Error>>()? 79 | .remove(0); 80 | 81 | let config = ServerConfig::builder() 82 | .with_no_client_auth() 83 | .with_single_cert(cert, key) 84 | .map_err(|err| io::Error::new(ErrorKind::InvalidInput, err))?; 85 | 86 | Ok(config) 87 | } 88 | 89 | pub async fn start_server() -> Result<(), Box> { 90 | let listener = TcpListener::bind("0.0.0.0:3306").await?; 91 | 92 | loop { 93 | let (stream, _) = listener.accept().await?; 94 | let (mut r, mut w) = stream.into_split(); 95 | 96 | tokio::spawn(async move { 97 | let tls_config = setup_tls().unwrap(); 98 | let tls_config = Arc::new(tls_config); 99 | let mut shim = Backend; 100 | let ops = IntermediaryOptions::default(); 101 | 102 | let (is_ssl, init_params) = opensrv_mysql::AsyncMysqlIntermediary::init_before_ssl( 103 | &mut shim, 104 | &mut r, 105 | &mut w, 106 | &Some(tls_config.clone()), 107 | ) 108 | .await 109 | .unwrap(); 110 | 111 | if is_ssl { 112 | opensrv_mysql::secure_run_with_options(shim, w, ops, tls_config, init_params) 113 | .await 114 | } else { 115 | opensrv_mysql::plain_run_with_options(shim, w, ops, init_params).await 116 | } 117 | }); 118 | } 119 | } 120 | } 121 | 122 | #[cfg(feature = "tls")] 123 | #[tokio::test(flavor = "multi_thread", worker_threads = 3)] 124 | async fn test_secure() -> Result<(), Box> { 125 | use std::{ 126 | process::{Command, Stdio}, 127 | time::Duration, 128 | }; 129 | 130 | use std::os::unix::process::ExitStatusExt; 131 | 132 | tokio::spawn(async { 133 | let _ = tls::start_server().await; 134 | }); 135 | 136 | tokio::time::sleep(Duration::from_secs(1)).await; 137 | 138 | let mut echo_output = Command::new("echo") 139 | .arg("\"SELECT * FROM foo\"") 140 | .stdout(Stdio::piped()) 141 | .spawn()?; 142 | 143 | let echo_output = echo_output.stdout.take().unwrap(); 144 | let mut mysql_ssl = Command::new("mysql") 145 | .args(["-h", "127.0.0.1", "--table", "--ssl-mode=REQUIRED"]) 146 | .stdin(echo_output) 147 | .stdout(Stdio::piped()) 148 | .spawn()?; 149 | 150 | let status = mysql_ssl.wait()?; 151 | assert_eq!(status.into_raw(), 0); 152 | 153 | Ok(()) 154 | } 155 | -------------------------------------------------------------------------------- /mysql/tests/ssl/server.crt: -------------------------------------------------------------------------------- 1 | Certificate: 2 | Data: 3 | Version: 3 (0x2) 4 | Serial Number: 5 | 1e:a1:44:88:27:3d:5c:c8:ff:ef:06:2e:da:21:05:29:30:a5:ce:2c 6 | Signature Algorithm: sha256WithRSAEncryption 7 | Issuer: CN = localhost 8 | Validity 9 | Not Before: Oct 11 07:36:01 2022 GMT 10 | Not After : Oct 8 07:36:01 2032 GMT 11 | Subject: CN = localhost 12 | Subject Public Key Info: 13 | Public Key Algorithm: rsaEncryption 14 | RSA Public-Key: (2048 bit) 15 | Modulus: 16 | 00:d5:b0:29:38:63:13:5e:1e:1d:ae:1f:47:88:b4: 17 | 44:96:21:d8:d7:03:a3:d8:f9:03:2f:4e:79:66:e6: 18 | db:19:55:1d:85:9b:f1:78:2d:87:f3:72:91:13:dc: 19 | ff:00:cb:ab:fd:a1:c8:3a:56:26:e3:88:1d:ec:98: 20 | 4a:af:eb:f9:60:80:27:e1:06:ba:c0:0d:c3:09:0e: 21 | fe:d8:86:1e:25:b4:04:62:a5:75:46:8e:11:e8:61: 22 | 59:aa:97:17:ea:c7:4c:c6:13:8c:6d:54:2a:b9:78: 23 | 86:54:a9:6f:d6:31:96:c6:41:76:a3:c7:67:40:6f: 24 | f2:1a:4c:0d:77:05:bb:3d:0b:16:f8:c7:de:6c:de: 25 | 7b:2e:b6:29:85:4b:a8:36:d3:f2:84:75:e0:85:17: 26 | ce:22:84:4b:94:02:17:8a:36:2b:13:ee:2f:aa:55: 27 | 6b:ff:8b:df:d3:e0:23:8d:fd:c3:f8:e2:c8:a7:d5: 28 | 76:a6:73:7d:a8:5f:6a:49:02:78:a2:c5:66:14:ee: 29 | 86:50:3b:d1:67:7f:1b:0c:27:0d:84:ec:44:0d:39: 30 | 08:ba:69:65:e0:35:a4:67:aa:19:e7:fe:0e:4b:9f: 31 | 23:1e:4e:38:ed:d7:93:57:6e:94:31:05:d3:ae:f7: 32 | 6c:01:3c:30:69:19:f4:7b:b5:48:95:71:c9:9c:30: 33 | 43:9d 34 | Exponent: 65537 (0x10001) 35 | X509v3 extensions: 36 | X509v3 Subject Key Identifier: 37 | 8E:81:0B:60:B1:F9:7D:D8:64:91:BB:30:86:E5:3D:CD:B7:82:D8:31 38 | X509v3 Authority Key Identifier: 39 | keyid:8E:81:0B:60:B1:F9:7D:D8:64:91:BB:30:86:E5:3D:CD:B7:82:D8:31 40 | 41 | X509v3 Basic Constraints: critical 42 | CA:TRUE 43 | Signature Algorithm: sha256WithRSAEncryption 44 | 6c:ae:ee:3e:e3:d4:5d:29:37:62:b0:32:ce:a4:36:c7:25:b4: 45 | 6a:9f:ba:b4:f0:2f:0a:96:2f:dc:6d:df:7d:92:e7:f0:ee:f7: 46 | de:44:9d:52:36:ff:0c:98:ef:8b:7f:27:df:6e:fe:64:11:7c: 47 | 01:5d:7f:c8:73:a3:24:24:ba:81:fd:a8:ae:28:4f:93:bb:92: 48 | ff:86:d6:48:a2:ca:a5:1f:ea:1c:0d:02:22:e8:71:23:27:22: 49 | 4f:0f:37:58:9a:d9:fd:70:c5:4c:93:7d:47:1c:b6:ea:1b:4f: 50 | 4e:7c:eb:9d:9a:d3:28:78:67:27:e9:b1:ea:f6:93:68:76:e5: 51 | 2e:52:c6:29:91:ba:0a:96:2e:14:33:69:35:d7:b5:e0:c0:ef: 52 | 05:77:09:9b:a1:cc:7b:b2:f0:6a:cb:5c:5f:a1:27:69:b0:2c: 53 | 6e:93:eb:37:98:cd:97:8d:9e:78:a8:f5:99:12:66:86:48:cf: 54 | b2:e0:68:6f:77:98:06:13:24:55:d1:c3:80:1d:59:53:1f:44: 55 | 85:bc:5d:29:aa:2a:a1:06:17:6b:e7:2b:11:0b:fd:e3:f8:88: 56 | 89:32:57:a3:70:f7:1b:6c:c1:66:c7:3c:a4:2d:e8:5f:00:1c: 57 | 55:2f:72:ed:d4:3a:3f:d0:95:de:6c:a4:96:6e:b4:63:0e:80: 58 | 08:b2:25:d5 59 | -----BEGIN CERTIFICATE----- 60 | MIIDCTCCAfGgAwIBAgIUHqFEiCc9XMj/7wYu2iEFKTClziwwDQYJKoZIhvcNAQEL 61 | BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMTAxMTA3MzYwMVoXDTMyMTAw 62 | ODA3MzYwMVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF 63 | AAOCAQ8AMIIBCgKCAQEA1bApOGMTXh4drh9HiLREliHY1wOj2PkDL055ZubbGVUd 64 | hZvxeC2H83KRE9z/AMur/aHIOlYm44gd7JhKr+v5YIAn4Qa6wA3DCQ7+2IYeJbQE 65 | YqV1Ro4R6GFZqpcX6sdMxhOMbVQquXiGVKlv1jGWxkF2o8dnQG/yGkwNdwW7PQsW 66 | +MfebN57LrYphUuoNtPyhHXghRfOIoRLlAIXijYrE+4vqlVr/4vf0+Ajjf3D+OLI 67 | p9V2pnN9qF9qSQJ4osVmFO6GUDvRZ38bDCcNhOxEDTkIumll4DWkZ6oZ5/4OS58j 68 | Hk447deTV26UMQXTrvdsATwwaRn0e7VIlXHJnDBDnQIDAQABo1MwUTAdBgNVHQ4E 69 | FgQUjoELYLH5fdhkkbswhuU9zbeC2DEwHwYDVR0jBBgwFoAUjoELYLH5fdhkkbsw 70 | huU9zbeC2DEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAbK7u 71 | PuPUXSk3YrAyzqQ2xyW0ap+6tPAvCpYv3G3ffZLn8O733kSdUjb/DJjvi38n327+ 72 | ZBF8AV1/yHOjJCS6gf2orihPk7uS/4bWSKLKpR/qHA0CIuhxIyciTw83WJrZ/XDF 73 | TJN9Rxy26htPTnzrnZrTKHhnJ+mx6vaTaHblLlLGKZG6CpYuFDNpNde14MDvBXcJ 74 | m6HMe7LwastcX6EnabAsbpPrN5jNl42eeKj1mRJmhkjPsuBob3eYBhMkVdHDgB1Z 75 | Ux9EhbxdKaoqoQYXa+crEQv94/iIiTJXo3D3G2zBZsc8pC3oXwAcVS9y7dQ6P9CV 76 | 3myklm60Yw6ACLIl1Q== 77 | -----END CERTIFICATE----- 78 | -------------------------------------------------------------------------------- /mysql/tests/ssl/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDVsCk4YxNeHh2u 3 | H0eItESWIdjXA6PY+QMvTnlm5tsZVR2Fm/F4LYfzcpET3P8Ay6v9ocg6VibjiB3s 4 | mEqv6/lggCfhBrrADcMJDv7Yhh4ltARipXVGjhHoYVmqlxfqx0zGE4xtVCq5eIZU 5 | qW/WMZbGQXajx2dAb/IaTA13Bbs9Cxb4x95s3nsutimFS6g20/KEdeCFF84ihEuU 6 | AheKNisT7i+qVWv/i9/T4CON/cP44sin1Xamc32oX2pJAniixWYU7oZQO9FnfxsM 7 | Jw2E7EQNOQi6aWXgNaRnqhnn/g5LnyMeTjjt15NXbpQxBdOu92wBPDBpGfR7tUiV 8 | ccmcMEOdAgMBAAECggEBAMMCIJv0zpf1o+Bja0S2PmFEQj72c3Buzxk85E2kIA7e 9 | PjLQPW0PICJrSzp1U8HGHQ85tSCHvrWmYqin0oD5OHt4eOxC1+qspHB/3tJ6ksiV 10 | n+rmVEAvJuiK7ulfOdRoTQf2jxC23saj1vMsLYOrfY0v8LVGJFQJ1UdqYF9eO6FX 11 | 8i6eQekV0n8u+DMUysYXfePDXEwpunKrlZwZtThgBY31gAIOdNo/FOAFe1yBJdPl 12 | rUFZes1IrE0c4CNxodajuRNCjtNWoX8TK1cXQVUpPprdFLBcYG2P9mPZ7SkZWJc7 13 | rkyPX6Wkb7q3laUCBxuKL1iOJIwaVBYaKfv4HS7VuYECgYEA9H7VB8+whWx2cTFb 14 | 9oYbcaU3HtbKRh6KQP8eB4IWeKV/c/ceWVAxtU9Hx2QU1zZ2fLl+KkaOGeECNNqD 15 | BP1O5qk2qmkjJcP4kzh1K+p7zkqAkrhHqB36y/gwptB8v7JbCchQq9cnBeYsXNIa 16 | j13KvteprRSnanKu18d2aC43cNMCgYEA3746ITtqy1g6AQ0Q/MXN/axsXixKfVjf 17 | kgN/lpjy6oeoEIWKqiNrOQpwy4NeBo6ZN+cwjUUr9SY/BKsZqMGErO8Xuu+QtJYD 18 | ioW/My9rTrTElbpsLpSvZDLc9IRepV4k+5PpXTIRBqp7Q3BZnTjbRMc8x/owG23G 19 | eXnfVKlWM88CgYEA5HBQuMCrzK3/qFkW9Kpun+tfKfhD++nzATGcrCU2u7jd8cr1 20 | 1zsfhqkxhrIS6tYfNP/XSsarZLCgcCOuAQ5wFwIJaoVbaqDE80Dv8X1f+eoQYYW+ 21 | peyE9OjLBEGOHUoW13gLL9ORyWg7EOraGBPpKBC2n1nJ5qKKjF/4WPS9pjMCgYEA 22 | 3UuUyxGtivn0RN3bk2dBWkmT1YERG/EvD4gORbF5caZDADRU9fqaLoy5C1EfSnT3 23 | 7mbnipKD67CsW72vX04oH7NLUUVpZnOJhRTMC6A3Dl2UolMEdP3yi7QS/nV99ymq 24 | gnnFMrw2QtWTnRweRnbZyKkW4OP/eOGWkMeNsHrcG9kCgYEAz/09cKumk349AIXV 25 | g6Jw64gCTjWh157wnD3ZSPPEcr/09/fZwf1W0gkY/tbCVrVPJHWb3K5t2nRXjLlz 26 | HMnQXmcMxMlY3Ufvm2H3ov1ODPKwpcBWUZqnpFTZX7rC58lO/wvgiKpgtHA3pDdw 27 | oYDaaozVP4EnnByxhmHaM7ce07U= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "stable" 3 | components = ["rustfmt", "clippy"] 4 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | edition = "2021" 2 | reorder_imports = true 3 | --------------------------------------------------------------------------------