├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── clickhouse-data-type ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── src │ ├── array.rs │ ├── date_time.rs │ ├── date_time64.rs │ ├── decimal.rs │ ├── enum.rs │ ├── fixed_string.rs │ ├── grammars │ │ └── type_name.pest │ ├── lib.rs │ ├── low_cardinality.rs │ ├── map.rs │ ├── nullable.rs │ ├── type_name.rs │ └── type_name_parser.rs └── tests │ ├── files │ ├── array.txt │ ├── date.txt │ ├── datetime.txt │ ├── datetime64.txt │ ├── decimal.txt │ ├── enum.txt │ ├── fixedstring.txt │ ├── float.txt │ ├── int_uint.txt │ ├── ipv4.txt │ ├── ipv6.txt │ ├── lowcardinality.txt │ ├── map.txt │ ├── nullable.txt │ ├── string.txt │ ├── tuple.txt │ └── uuid.txt │ └── gen_files.sh ├── clickhouse-data-value ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── src │ ├── date.rs │ ├── date_and_time_parser.rs │ ├── datetime.rs │ ├── grammars │ │ └── date_and_time.pest │ └── lib.rs └── tests │ ├── files │ ├── date.txt │ ├── datetime64_iso.txt │ ├── datetime64_simple.txt │ ├── datetime64_unix_timestamp.txt │ ├── datetime_iso.txt │ ├── datetime_simple.txt │ └── datetime_unix_timestamp.txt │ └── gen_files.sh ├── clickhouse-format ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── src │ ├── format_name.rs │ ├── input │ │ ├── json_compact_each_row.rs │ │ └── mod.rs │ ├── lib.rs │ ├── output │ │ ├── json.rs │ │ ├── json_compact.rs │ │ ├── json_compact_each_row.rs │ │ ├── json_compact_each_row_with_names_and_types.rs │ │ ├── json_compact_strings.rs │ │ ├── json_compact_strings_each_row.rs │ │ ├── json_compact_strings_each_row_with_names_and_types.rs │ │ ├── json_each_row.rs │ │ ├── json_each_row_with_progress.rs │ │ ├── json_strings.rs │ │ ├── json_strings_each_row.rs │ │ ├── json_strings_each_row_with_progress.rs │ │ ├── mod.rs │ │ ├── tsv.rs │ │ ├── tsv_raw.rs │ │ ├── tsv_with_names.rs │ │ └── tsv_with_names_and_types.rs │ └── test_helpers.rs └── tests │ ├── files │ ├── JSON.json │ ├── JSONCompact.json │ ├── JSONCompactEachRow.txt │ ├── JSONCompactEachRowWithNamesAndTypes.txt │ ├── JSONCompactStrings.json │ ├── JSONCompactStringsEachRow.txt │ ├── JSONCompactStringsEachRowWithNamesAndTypes.txt │ ├── JSONEachRow.txt │ ├── JSONEachRowWithProgress.txt │ ├── JSONStrings.json │ ├── JSONStringsEachRow.txt │ ├── JSONStringsEachRowWithProgress.txt │ ├── TSV.tsv │ ├── TSVRaw.tsv │ ├── TSVWithNames.tsv │ └── TSVWithNamesAndTypes.tsv │ └── gen_files.sh ├── clickhouse-http-client ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── src │ ├── client.rs │ ├── client_config.rs │ ├── error.rs │ └── lib.rs └── tests │ ├── integration_tests.rs │ ├── integration_tests │ ├── curd.rs │ ├── helpers.rs │ └── ping.rs │ └── run_integration_tests.sh ├── clickhouse-postgres-client ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── src │ ├── connect_options.rs │ ├── executor.rs │ ├── lib.rs │ ├── row.rs │ └── type_info.rs └── tests │ ├── integration_tests.rs │ ├── integration_tests │ ├── data_types.rs │ └── helpers.rs │ └── run_integration_tests.sh ├── clickhouse_sqls ├── data_types │ ├── array │ │ ├── select_with_Nullable.sql │ │ ├── select_with_String.sql │ │ └── select_with_UInt8.sql │ ├── boolean │ │ ├── create_table.sql │ │ ├── drop_table.sql │ │ ├── insert.sql │ │ └── select.sql │ ├── date │ │ └── select.sql │ ├── datetime │ │ └── select_with_UTC.sql │ ├── datetime64 │ │ ├── select_with_micro_and_UTC.sql │ │ ├── select_with_milli_and_UTC.sql │ │ └── select_with_nano_and_UTC.sql │ ├── decimal │ │ ├── select_Decimal128.sql │ │ ├── select_Decimal256.sql │ │ ├── select_Decimal32.sql │ │ └── select_Decimal64.sql │ ├── domains_ipv4 │ │ ├── select_f_IPv4NumToString.sql │ │ ├── select_f_isIPv4String.sql │ │ ├── select_f_toIPv4.sql │ │ └── select_f_toUInt32.sql │ ├── domains_ipv6 │ │ └── select_f_hex.sql │ ├── enum │ │ ├── select_Enum16.sql │ │ ├── select_Enum8.sql │ │ ├── select_cast_Enum16.sql │ │ └── select_cast_Enum8.sql │ ├── fixedstring │ │ └── select.sql │ ├── float │ │ ├── select_-Inf.sql │ │ ├── select_Float32.sql │ │ ├── select_Float64.sql │ │ ├── select_Inf.sql │ │ └── select_NaN.sql │ ├── geo │ │ ├── create_table_Point.sql │ │ ├── drop_table_Point.sql │ │ ├── insert_Point.sql │ │ └── select_Point.sql │ ├── int-uint │ │ ├── select_Int128.sql │ │ ├── select_Int16.sql │ │ ├── select_Int256.sql │ │ ├── select_Int32.sql │ │ ├── select_Int64.sql │ │ ├── select_Int8.sql │ │ ├── select_Int8_0-9.sql │ │ ├── select_UInt16.sql │ │ ├── select_UInt256.sql │ │ ├── select_UInt32.sql │ │ ├── select_UInt64.sql │ │ └── select_UInt8.sql │ ├── map │ │ └── select_with_UInt8_and_String.sql │ ├── nullable │ │ ├── create_table.sql │ │ ├── drop_table.sql │ │ ├── insert.sql │ │ ├── select_f_ifNull.sql │ │ └── select_f_toNullable.sql │ ├── string │ │ └── select_f_toString_date.sql │ ├── tuple │ │ ├── select_with_UInt8_and_Nullable.sql │ │ └── select_with_UInt8_and_String.sql │ └── uuid │ │ └── select.sql └── settings │ ├── enable_allow_experimental_geo_types.sql │ ├── enable_allow_experimental_map_type.sql │ ├── set_date_time_output_format_to_iso.sql │ ├── set_date_time_output_format_to_simple.sql │ └── set_date_time_output_format_to_unix_timestamp.sql ├── clickhouse_tgz_archive ├── .gitignore ├── README.md └── download.sh ├── demos ├── http_client │ ├── Cargo.toml │ └── src │ │ └── main.rs └── postgres_client │ ├── Cargo.toml │ └── src │ ├── conn.rs │ └── pool.rs └── sqlx-clickhouse-ext ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md └── src ├── error.rs ├── executor.rs ├── lib.rs └── value.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | .vscode 4 | .idea 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "clickhouse-data-type", 4 | "clickhouse-data-value", 5 | "clickhouse-format", 6 | "clickhouse-http-client", 7 | "clickhouse-postgres-client", 8 | "demos/postgres_client", 9 | "demos/http_client", 10 | "sqlx-clickhouse-ext", 11 | ] 12 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Examples 2 | 3 | * [ClickHouse Postgres Client - Connection](demos/postgres_client/src/conn.rs) 4 | * [ClickHouse Postgres Client - Pool](demos/postgres_client/src/pool.rs) 5 | 6 | * [ClickHouse HTTP Client - Simple](demos/http_client/src/main.rs) 7 | * [ClickHouse HTTP Client - CURD](clickhouse-http-client/tests/integration_tests/curd.rs) 8 | 9 | ## Dev 10 | 11 | ``` 12 | cargo clippy -p clickhouse-data-type -p clickhouse-data-value -p clickhouse-format -p clickhouse-http-client --all-features --tests -- -D clippy::all 13 | cargo +nightly clippy -p clickhouse-data-type -p clickhouse-data-value -p clickhouse-format -p clickhouse-http-client --all-features --tests -- -D clippy::all 14 | 15 | cargo clippy -p clickhouse-postgres-client --features _integration_tests --tests -- -D clippy::all 16 | cargo +nightly clippy -p clickhouse-postgres-client --features _integration_tests --tests -- -D clippy::all 17 | 18 | cargo clippy -p sqlx-clickhouse-ext --features postgres,all-types,runtime-tokio-native-tls --tests -- -D clippy::all 19 | cargo +nightly clippy -p sqlx-clickhouse-ext --features postgres,all-types,runtime-tokio-native-tls --tests -- -D clippy::all 20 | 21 | 22 | 23 | cargo fmt -- --check 24 | 25 | 26 | 27 | cargo test -p clickhouse-data-type -p clickhouse-data-value -- --nocapture 28 | cargo test -p clickhouse-format --features with-all -- --nocapture 29 | cargo test -p clickhouse-http-client --features with-format-all -- --nocapture 30 | cargo test -p clickhouse-postgres-client --features all-types,runtime-tokio-native-tls,num-bigint -- --nocapture 31 | cargo test -p sqlx-clickhouse-ext --features postgres,all-types,runtime-tokio-native-tls -- --nocapture 32 | 33 | 34 | 35 | RUST_BACKTRACE=1 RUST_LOG=trace ./clickhouse-http-client/tests/run_integration_tests.sh 36 | RUST_BACKTRACE=1 RUST_LOG=trace ./clickhouse-postgres-client/tests/run_integration_tests.sh 37 | ``` 38 | 39 | ## Publish order 40 | 41 | clickhouse-data-type clickhouse-data-value clickhouse-format 42 | 43 | clickhouse-http-client 44 | 45 | sqlx-clickhouse-ext 46 | 47 | clickhouse-postgres-client 48 | -------------------------------------------------------------------------------- /clickhouse-data-type/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-data-type" 3 | version = "0.2.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "ClickHouse Data Types" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/clickhouse-data-type" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [dependencies] 16 | pest = { version = "2.5", default-features = false, features = ["std"] } 17 | pest_derive = { version = "2.5", default-features = false, features = ["std"] } 18 | 19 | thiserror = { version = "1", default-features = false } 20 | 21 | chrono-tz = { version = "0.8", default-features = false } 22 | 23 | [dev-dependencies] 24 | serde_json = { version = "1" } 25 | -------------------------------------------------------------------------------- /clickhouse-data-type/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /clickhouse-data-type/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /clickhouse-data-type/README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-data-type 2 | 3 | * [ClickHouse Doc](https://clickhouse.tech/docs/en/sql-reference/data-types/) 4 | * [Cargo package](https://crates.io/crates/clickhouse-data-type) 5 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/array.rs: -------------------------------------------------------------------------------- 1 | use pest::iterators::Pairs; 2 | 3 | use crate::{type_name::TypeName, type_name_parser::Rule, ParseError}; 4 | 5 | pub(crate) fn get_data_type(mut array_pairs: Pairs<'_, Rule>) -> Result { 6 | let data_type_pair = array_pairs 7 | .next() 8 | .ok_or(ParseError::Unknown)? 9 | .into_inner() 10 | .next() 11 | .ok_or(ParseError::Unknown)?; 12 | 13 | let data_type = TypeName::from_pair(data_type_pair)?; 14 | 15 | Ok(data_type) 16 | } 17 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/date_time.rs: -------------------------------------------------------------------------------- 1 | use chrono_tz::Tz; 2 | use pest::iterators::Pairs; 3 | 4 | use crate::{type_name_parser::Rule, ParseError}; 5 | 6 | pub(crate) fn get_timezone(mut date_time_pairs: Pairs<'_, Rule>) -> Result, ParseError> { 7 | let timezone = if let Some(pair_timezone) = date_time_pairs.next() { 8 | Some( 9 | pair_timezone 10 | .as_str() 11 | .parse::() 12 | .map_err(|err: &str| ParseError::ValueInvalid(err.to_string()))?, 13 | ) 14 | } else { 15 | None 16 | }; 17 | 18 | Ok(timezone) 19 | } 20 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/date_time64.rs: -------------------------------------------------------------------------------- 1 | use core::num::ParseIntError; 2 | 3 | use chrono_tz::Tz; 4 | use pest::iterators::Pairs; 5 | 6 | use crate::{type_name_parser::Rule, ParseError}; 7 | 8 | const PRECISION_MAX: usize = 9; 9 | 10 | #[derive(PartialEq, Eq, Debug, Clone)] 11 | pub struct DateTime64Precision(pub usize); 12 | impl TryFrom<&str> for DateTime64Precision { 13 | type Error = ParseError; 14 | fn try_from(s: &str) -> Result { 15 | let n: usize = s 16 | .parse() 17 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 18 | 19 | if n > PRECISION_MAX { 20 | return Err(ParseError::ValueInvalid( 21 | "invalid datetime64 precision".to_string(), 22 | )); 23 | } 24 | 25 | Ok(Self(n)) 26 | } 27 | } 28 | 29 | pub(crate) fn get_precision_and_timezone( 30 | mut date_time64_pairs: Pairs<'_, Rule>, 31 | ) -> Result<(DateTime64Precision, Option), ParseError> { 32 | let precision_pair = date_time64_pairs.next().ok_or(ParseError::Unknown)?; 33 | let precision = DateTime64Precision::try_from(precision_pair.as_str())?; 34 | 35 | let timezone = if let Some(pair_timezone) = date_time64_pairs.next() { 36 | Some( 37 | pair_timezone 38 | .as_str() 39 | .parse::() 40 | .map_err(|err: &str| ParseError::ValueInvalid(err.to_string()))?, 41 | ) 42 | } else { 43 | None 44 | }; 45 | 46 | Ok((precision, timezone)) 47 | } 48 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/decimal.rs: -------------------------------------------------------------------------------- 1 | use core::num::ParseIntError; 2 | 3 | use pest::iterators::Pairs; 4 | 5 | use crate::{type_name_parser::Rule, ParseError}; 6 | 7 | const PRECISION_MIN: usize = 1; 8 | const PRECISION_MAX: usize = 76; 9 | 10 | #[derive(PartialEq, Eq, Debug, Clone)] 11 | pub struct DecimalPrecision(pub usize); 12 | impl TryFrom<&str> for DecimalPrecision { 13 | type Error = ParseError; 14 | fn try_from(s: &str) -> Result { 15 | let precision: usize = s 16 | .parse() 17 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 18 | 19 | if precision < PRECISION_MIN { 20 | return Err(ParseError::ValueInvalid( 21 | "invalid decimal precision".to_string(), 22 | )); 23 | } 24 | if precision > PRECISION_MAX { 25 | return Err(ParseError::ValueInvalid( 26 | "invalid decimal precision".to_string(), 27 | )); 28 | } 29 | 30 | Ok(Self(precision)) 31 | } 32 | } 33 | 34 | #[derive(PartialEq, Eq, Debug, Clone)] 35 | pub struct DecimalScale(pub usize); 36 | impl TryFrom<(&str, &DecimalPrecision)> for DecimalScale { 37 | type Error = ParseError; 38 | fn try_from(t: (&str, &DecimalPrecision)) -> Result { 39 | let (s, precision) = t; 40 | 41 | let scale: usize = s 42 | .parse() 43 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 44 | 45 | if scale > precision.0 { 46 | return Err(ParseError::ValueInvalid( 47 | "invalid decimal scale".to_string(), 48 | )); 49 | } 50 | 51 | Ok(Self(scale)) 52 | } 53 | } 54 | 55 | pub(crate) fn get_precision_and_scale( 56 | mut decimal_pairs: Pairs<'_, Rule>, 57 | ) -> Result<(DecimalPrecision, DecimalScale), ParseError> { 58 | let precision_pair = decimal_pairs.next().ok_or(ParseError::Unknown)?; 59 | let scale_pair = decimal_pairs.next().ok_or(ParseError::Unknown)?; 60 | 61 | let precision = DecimalPrecision::try_from(precision_pair.as_str())?; 62 | 63 | let scale = DecimalScale::try_from((scale_pair.as_str(), &precision))?; 64 | 65 | Ok((precision, scale)) 66 | } 67 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/enum.rs: -------------------------------------------------------------------------------- 1 | use core::num::ParseIntError; 2 | use std::collections::HashMap; 3 | 4 | use pest::iterators::Pairs; 5 | 6 | use crate::{type_name_parser::Rule, ParseError}; 7 | 8 | pub type Enum8 = HashMap; 9 | pub type Enum16 = HashMap; 10 | 11 | pub(crate) fn get_enum8(enum_pairs: Pairs<'_, Rule>) -> Result { 12 | let mut map = HashMap::new(); 13 | for pair in enum_pairs { 14 | let mut pair_inner = pair.into_inner(); 15 | let key = pair_inner 16 | .next() 17 | .ok_or(ParseError::Unknown)? 18 | .as_str() 19 | .to_string(); 20 | let value = pair_inner 21 | .next() 22 | .ok_or(ParseError::Unknown)? 23 | .as_str() 24 | .parse() 25 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 26 | 27 | map.insert(key, value); 28 | } 29 | Ok(map) 30 | } 31 | 32 | pub(crate) fn get_enum16(enum_pairs: Pairs<'_, Rule>) -> Result { 33 | let mut map = HashMap::new(); 34 | for pair in enum_pairs { 35 | let mut pair_inner = pair.into_inner(); 36 | let key = pair_inner 37 | .next() 38 | .ok_or(ParseError::Unknown)? 39 | .as_str() 40 | .to_string(); 41 | let value: i16 = pair_inner 42 | .next() 43 | .ok_or(ParseError::Unknown)? 44 | .as_str() 45 | .parse() 46 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 47 | 48 | map.insert(key, value); 49 | } 50 | Ok(map) 51 | } 52 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/fixed_string.rs: -------------------------------------------------------------------------------- 1 | use core::num::ParseIntError; 2 | 3 | use pest::iterators::Pairs; 4 | 5 | use crate::{type_name_parser::Rule, ParseError}; 6 | 7 | const N_MIN: usize = 1; 8 | 9 | #[derive(PartialEq, Eq, Debug, Clone)] 10 | pub struct FixedStringN(pub usize); 11 | impl TryFrom<&str> for FixedStringN { 12 | type Error = ParseError; 13 | fn try_from(s: &str) -> Result { 14 | let n: usize = s 15 | .parse() 16 | .map_err(|err: ParseIntError| ParseError::ValueInvalid(err.to_string()))?; 17 | 18 | if n < N_MIN { 19 | return Err(ParseError::ValueInvalid( 20 | "invalid fixedstring n".to_string(), 21 | )); 22 | } 23 | 24 | Ok(Self(n)) 25 | } 26 | } 27 | 28 | pub(crate) fn get_n(mut fixed_string_pairs: Pairs<'_, Rule>) -> Result { 29 | let n_pair = fixed_string_pairs.next().ok_or(ParseError::Unknown)?; 30 | 31 | let n = FixedStringN::try_from(n_pair.as_str())?; 32 | 33 | Ok(n) 34 | } 35 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/grammars/type_name.pest: -------------------------------------------------------------------------------- 1 | type_name = { 2 | UInt256 | UInt64 | UInt32 | UInt16 | UInt8 | 3 | Int256 | Int128 | Int64 | Int32 | Int16 | Int8 | 4 | Float32 | Float64 | 5 | Decimal | 6 | String | 7 | FixedString | 8 | UUID | 9 | DateTime64 | DateTime | Date | 10 | Enum16 | Enum8 | 11 | IPv4 | IPv6 | 12 | // 13 | // 14 | // 15 | LowCardinality | 16 | Nullable | 17 | Point | Ring | Polygon | MultiPolygon | 18 | // 19 | // 20 | // 21 | Array | 22 | Tuple | 23 | Map 24 | } 25 | 26 | 27 | UInt8 = { "UInt8" } 28 | UInt16 = { "UInt16" } 29 | UInt32 = { "UInt32" } 30 | UInt64 = { "UInt64" } 31 | UInt256 = { "UInt256" } 32 | Int8 = { "Int8" } 33 | Int16 = { "Int16" } 34 | Int32 = { "Int32" } 35 | Int64 = { "Int64" } 36 | Int128 = { "Int128" } 37 | Int256 = { "Int256" } 38 | 39 | Float32 = { "Float32" } 40 | Float64 = { "Float64" } 41 | 42 | Decimal = { "Decimal" ~ "(" ~ Decimal_precision ~ "," ~ Decimal_scale ~ ")" } 43 | Decimal_precision = @{ ASCII_NONZERO_DIGIT{1, 2} } 44 | Decimal_scale = @{ ASCII_DIGIT{1, 2} } 45 | 46 | String = { "String" } 47 | 48 | FixedString = { "FixedString" ~ "(" ~ FixedString_n ~ ")" } 49 | FixedString_n = @{ ASCII_NONZERO_DIGIT+ } 50 | 51 | UUID = { "UUID" } 52 | 53 | Date = { "Date" } 54 | DateTime = { "DateTime" ~ "(" ~ "'" ~ DateTime_timezone ~ "'" ~ ")" | "DateTime" } 55 | DateTime64 = { "DateTime64" ~ "(" ~ DateTime64_precision ~ ("," ~ "'" ~ DateTime_timezone ~ "'" )? ~ ")" } 56 | DateTime64_precision = @{ ASCII_DIGIT } 57 | DateTime_timezone = @{ DateTime_timezone_char+ ~ "/" ~ DateTime_timezone_char+ | DateTime_timezone_char+ } 58 | DateTime_timezone_char = @{ ASCII_ALPHA | "-" | "_" } 59 | 60 | Enum8 = { "Enum8" ~ "(" ~ Enum_pair ~ ("," ~ Enum_pair)* ~ ")" } 61 | Enum16 = { "Enum16" ~ "(" ~ Enum_pair ~ ("," ~ Enum_pair)* ~ ")" } 62 | Enum_pair = { "'" ~ Enum_key ~ "'" ~ " "* ~ "=" ~ " "* ~ Enum_value } 63 | Enum_key = @{ ASCII_ALPHANUMERIC+ } 64 | Enum_value = @{ "-"? ~ ASCII_DIGIT+ } 65 | 66 | IPv4 = { "IPv4" } 67 | 68 | IPv6 = { "IPv6" } 69 | 70 | LowCardinality = { "LowCardinality" ~ "(" ~ LowCardinality_data_type ~ ")" } 71 | LowCardinality_data_type = { 72 | UInt64 | UInt32 | UInt16 | UInt8 | 73 | Int64 | Int32 | Int16 | Int8 | 74 | Float32 | Float64 | 75 | String | 76 | FixedString | 77 | DateTime | Date | 78 | Enum16 | Enum8 | 79 | IPv4 | IPv6 | 80 | // 81 | // 82 | // 83 | Nullable 84 | } 85 | 86 | Nullable = { "Nullable" ~ "(" ~ Nullable_type_name ~ ")" } 87 | Nullable_type_name = { 88 | Nullable_Nothing | 89 | UInt256 | UInt64 | UInt32 | UInt16 | UInt8 | 90 | Int256 | Int128 | Int64 | Int32 | Int16 | Int8 | 91 | Float32 | Float64 | 92 | Decimal | 93 | String | 94 | FixedString | 95 | UUID | 96 | DateTime64 | DateTime | Date | 97 | Enum16 | Enum8 | 98 | IPv4 | IPv6 99 | } 100 | Nullable_Nothing = { "Nothing" } 101 | 102 | Array = { "Array" ~ "(" ~ type_name ~ ")" } 103 | 104 | Tuple = { "Tuple" ~ "(" ~ type_name ~ ("," ~ type_name)* ~ ")" } 105 | 106 | Map = { "Map" ~ "(" ~ Map_key ~ "," ~ Map_value ~ ")" } 107 | Map_key = { 108 | UInt256 | UInt64 | UInt32 | UInt16 | UInt8 | 109 | Int256 | Int128 | Int64 | Int32 | Int16 | Int8 | 110 | Float32 | Float64 | 111 | Decimal | 112 | String | 113 | FixedString 114 | } 115 | Map_value = { 116 | UInt256 | UInt64 | UInt32 | UInt16 | UInt8 | 117 | Int256 | Int128 | Int64 | Int32 | Int16 | Int8 | 118 | Float32 | Float64 | 119 | Decimal | 120 | String | 121 | FixedString | 122 | // 123 | // 124 | // 125 | Array 126 | } 127 | 128 | // geo 129 | Point = { "Point" } 130 | Ring = { "Ring" } 131 | Polygon = { "Polygon" } 132 | MultiPolygon = { "MultiPolygon" } 133 | 134 | WHITESPACE = _{ " " } 135 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod array; 2 | pub mod date_time; 3 | pub mod date_time64; 4 | pub mod decimal; 5 | pub mod r#enum; 6 | pub mod fixed_string; 7 | pub mod low_cardinality; 8 | pub mod map; 9 | pub mod nullable; 10 | 11 | pub mod type_name; 12 | 13 | // https://github.com/pest-parser/pest/issues/490#issuecomment-808942497 14 | #[allow(clippy::upper_case_acronyms)] 15 | pub(crate) mod type_name_parser; 16 | 17 | #[derive(thiserror::Error, Debug)] 18 | pub enum ParseError { 19 | #[error("FormatMismatch {0}")] 20 | FormatMismatch(String), 21 | #[error("ValueInvalid {0}")] 22 | ValueInvalid(String), 23 | #[error("Unknown")] 24 | Unknown, 25 | } 26 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/low_cardinality.rs: -------------------------------------------------------------------------------- 1 | use chrono_tz::Tz; 2 | use pest::iterators::{Pair, Pairs}; 3 | 4 | use crate::{ 5 | date_time, 6 | fixed_string::{self, FixedStringN}, 7 | nullable::{self, NullableTypeName}, 8 | type_name_parser::Rule, 9 | ParseError, 10 | }; 11 | 12 | // https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ 13 | #[derive(PartialEq, Eq, Debug, Clone)] 14 | pub enum LowCardinalityDataType { 15 | UInt8, 16 | UInt16, 17 | UInt32, 18 | UInt64, 19 | Int8, 20 | Int16, 21 | Int32, 22 | Int64, 23 | Float32, 24 | Float64, 25 | String, 26 | FixedString(FixedStringN), 27 | Date, 28 | DateTime(Option), 29 | Ipv4, 30 | Ipv6, 31 | // 32 | // 33 | // 34 | Nullable(NullableTypeName), 35 | } 36 | 37 | impl TryFrom> for LowCardinalityDataType { 38 | type Error = ParseError; 39 | 40 | fn try_from(pair: Pair<'_, Rule>) -> Result { 41 | match pair.as_rule() { 42 | Rule::UInt8 => Ok(Self::UInt8), 43 | Rule::UInt16 => Ok(Self::UInt16), 44 | Rule::UInt32 => Ok(Self::UInt32), 45 | Rule::UInt64 => Ok(Self::UInt64), 46 | Rule::Int8 => Ok(Self::Int8), 47 | Rule::Int16 => Ok(Self::Int16), 48 | Rule::Int32 => Ok(Self::Int32), 49 | Rule::Int64 => Ok(Self::Int64), 50 | Rule::Float32 => Ok(Self::Float32), 51 | Rule::Float64 => Ok(Self::Float64), 52 | Rule::String => Ok(Self::String), 53 | Rule::FixedString => { 54 | let n = fixed_string::get_n(pair.into_inner())?; 55 | 56 | Ok(Self::FixedString(n)) 57 | } 58 | Rule::Date => Ok(Self::Date), 59 | Rule::DateTime => { 60 | let timezone = date_time::get_timezone(pair.into_inner())?; 61 | 62 | Ok(Self::DateTime(timezone)) 63 | } 64 | Rule::IPv4 => Ok(Self::Ipv4), 65 | Rule::IPv6 => Ok(Self::Ipv6), 66 | // 67 | // 68 | // 69 | Rule::Nullable => { 70 | let data_type = nullable::get_type_name(pair.into_inner())?; 71 | 72 | Ok(Self::Nullable(data_type)) 73 | } 74 | _ => Err(ParseError::Unknown), 75 | } 76 | } 77 | } 78 | 79 | pub(crate) fn get_data_type( 80 | mut low_cardinality_pairs: Pairs<'_, Rule>, 81 | ) -> Result { 82 | let low_cardinality_pair = low_cardinality_pairs.next().ok_or(ParseError::Unknown)?; 83 | 84 | let mut data_type_pairs = low_cardinality_pair.into_inner(); 85 | let data_type_pair = data_type_pairs.next().ok_or(ParseError::Unknown)?; 86 | 87 | let data_type = LowCardinalityDataType::try_from(data_type_pair)?; 88 | 89 | Ok(data_type) 90 | } 91 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/map.rs: -------------------------------------------------------------------------------- 1 | use pest::iterators::{Pair, Pairs}; 2 | 3 | use crate::{ 4 | array, 5 | decimal::{self, DecimalPrecision, DecimalScale}, 6 | fixed_string::{self, FixedStringN}, 7 | type_name::TypeName, 8 | type_name_parser::Rule, 9 | ParseError, 10 | }; 11 | 12 | #[derive(PartialEq, Eq, Debug, Clone)] 13 | pub enum MapKey { 14 | UInt8, 15 | UInt16, 16 | UInt32, 17 | UInt64, 18 | UInt256, 19 | Int8, 20 | Int16, 21 | Int32, 22 | Int64, 23 | Int128, 24 | Int256, 25 | Float32, 26 | Float64, 27 | Decimal(DecimalPrecision, DecimalScale), 28 | String, 29 | FixedString(FixedStringN), 30 | } 31 | 32 | impl TryFrom> for MapKey { 33 | type Error = ParseError; 34 | 35 | fn try_from(pair: Pair<'_, Rule>) -> Result { 36 | match pair.as_rule() { 37 | Rule::UInt8 => Ok(Self::UInt8), 38 | Rule::UInt16 => Ok(Self::UInt16), 39 | Rule::UInt32 => Ok(Self::UInt32), 40 | Rule::UInt64 => Ok(Self::UInt64), 41 | Rule::UInt256 => Ok(Self::UInt256), 42 | Rule::Int8 => Ok(Self::Int8), 43 | Rule::Int16 => Ok(Self::Int16), 44 | Rule::Int32 => Ok(Self::Int32), 45 | Rule::Int64 => Ok(Self::Int64), 46 | Rule::Int128 => Ok(Self::Int128), 47 | Rule::Int256 => Ok(Self::Int256), 48 | Rule::Float32 => Ok(Self::Float32), 49 | Rule::Float64 => Ok(Self::Float64), 50 | Rule::Decimal => { 51 | let (precision, scale) = decimal::get_precision_and_scale(pair.into_inner())?; 52 | 53 | Ok(Self::Decimal(precision, scale)) 54 | } 55 | Rule::String => Ok(Self::String), 56 | Rule::FixedString => { 57 | let n = fixed_string::get_n(pair.into_inner())?; 58 | 59 | Ok(Self::FixedString(n)) 60 | } 61 | _ => Err(ParseError::Unknown), 62 | } 63 | } 64 | } 65 | 66 | #[derive(PartialEq, Eq, Debug, Clone)] 67 | pub enum MapValue { 68 | UInt8, 69 | UInt16, 70 | UInt32, 71 | UInt64, 72 | UInt256, 73 | Int8, 74 | Int16, 75 | Int32, 76 | Int64, 77 | Int128, 78 | Int256, 79 | Float32, 80 | Float64, 81 | Decimal(DecimalPrecision, DecimalScale), 82 | String, 83 | FixedString(FixedStringN), 84 | // 85 | // 86 | // 87 | Array(Box), 88 | } 89 | 90 | impl TryFrom> for MapValue { 91 | type Error = ParseError; 92 | 93 | fn try_from(pair: Pair<'_, Rule>) -> Result { 94 | match pair.as_rule() { 95 | Rule::UInt8 => Ok(Self::UInt8), 96 | Rule::UInt16 => Ok(Self::UInt16), 97 | Rule::UInt32 => Ok(Self::UInt32), 98 | Rule::UInt64 => Ok(Self::UInt64), 99 | Rule::UInt256 => Ok(Self::UInt256), 100 | Rule::Int8 => Ok(Self::Int8), 101 | Rule::Int16 => Ok(Self::Int16), 102 | Rule::Int32 => Ok(Self::Int32), 103 | Rule::Int64 => Ok(Self::Int64), 104 | Rule::Int128 => Ok(Self::Int128), 105 | Rule::Int256 => Ok(Self::Int256), 106 | Rule::Float32 => Ok(Self::Float32), 107 | Rule::Float64 => Ok(Self::Float64), 108 | Rule::Decimal => { 109 | let (precision, scale) = decimal::get_precision_and_scale(pair.into_inner())?; 110 | 111 | Ok(Self::Decimal(precision, scale)) 112 | } 113 | Rule::String => Ok(Self::String), 114 | Rule::FixedString => { 115 | let n = fixed_string::get_n(pair.into_inner())?; 116 | 117 | Ok(Self::FixedString(n)) 118 | } 119 | // 120 | // 121 | // 122 | Rule::Array => { 123 | let data_type = self::array::get_data_type(pair.into_inner())?; 124 | 125 | Ok(Self::Array(data_type.into())) 126 | } 127 | _ => Err(ParseError::Unknown), 128 | } 129 | } 130 | } 131 | 132 | pub(crate) fn get_map_key_and_map_value( 133 | mut map_pairs: Pairs<'_, Rule>, 134 | ) -> Result<(MapKey, MapValue), ParseError> { 135 | let key = MapKey::try_from( 136 | map_pairs 137 | .next() 138 | .ok_or(ParseError::Unknown)? 139 | .into_inner() 140 | .next() 141 | .ok_or(ParseError::Unknown)?, 142 | )?; 143 | let value = MapValue::try_from( 144 | map_pairs 145 | .next() 146 | .ok_or(ParseError::Unknown)? 147 | .into_inner() 148 | .next() 149 | .ok_or(ParseError::Unknown)?, 150 | )?; 151 | 152 | Ok((key, value)) 153 | } 154 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/nullable.rs: -------------------------------------------------------------------------------- 1 | use chrono_tz::Tz; 2 | use pest::iterators::{Pair, Pairs}; 3 | 4 | use crate::{ 5 | date_time, 6 | date_time64::{self, DateTime64Precision}, 7 | decimal::{self, DecimalPrecision, DecimalScale}, 8 | fixed_string::{self, FixedStringN}, 9 | r#enum::{self, Enum16, Enum8}, 10 | type_name_parser::Rule, 11 | ParseError, 12 | }; 13 | 14 | #[derive(PartialEq, Eq, Debug, Clone)] 15 | pub enum NullableTypeName { 16 | Nothing, 17 | // 18 | UInt8, 19 | UInt16, 20 | UInt32, 21 | UInt64, 22 | UInt256, 23 | Int8, 24 | Int16, 25 | Int32, 26 | Int64, 27 | Int128, 28 | Int256, 29 | Float32, 30 | Float64, 31 | Decimal(DecimalPrecision, DecimalScale), 32 | String, 33 | FixedString(FixedStringN), 34 | Uuid, 35 | Date, 36 | DateTime(Option), 37 | DateTime64(DateTime64Precision, Option), 38 | Enum8(Enum8), 39 | Enum16(Enum16), 40 | Ipv4, 41 | Ipv6, 42 | } 43 | 44 | impl TryFrom> for NullableTypeName { 45 | type Error = ParseError; 46 | 47 | fn try_from(pair: Pair<'_, Rule>) -> Result { 48 | match pair.as_rule() { 49 | Rule::Nullable_Nothing => Ok(Self::Nothing), 50 | // 51 | Rule::UInt8 => Ok(Self::UInt8), 52 | Rule::UInt16 => Ok(Self::UInt16), 53 | Rule::UInt32 => Ok(Self::UInt32), 54 | Rule::UInt64 => Ok(Self::UInt64), 55 | Rule::UInt256 => Ok(Self::UInt256), 56 | Rule::Int8 => Ok(Self::Int8), 57 | Rule::Int16 => Ok(Self::Int16), 58 | Rule::Int32 => Ok(Self::Int32), 59 | Rule::Int64 => Ok(Self::Int64), 60 | Rule::Int128 => Ok(Self::Int128), 61 | Rule::Int256 => Ok(Self::Int256), 62 | Rule::Float32 => Ok(Self::Float32), 63 | Rule::Float64 => Ok(Self::Float64), 64 | Rule::Decimal => { 65 | let (precision, scale) = decimal::get_precision_and_scale(pair.into_inner())?; 66 | 67 | Ok(Self::Decimal(precision, scale)) 68 | } 69 | Rule::String => Ok(Self::String), 70 | Rule::FixedString => { 71 | let n = fixed_string::get_n(pair.into_inner())?; 72 | 73 | Ok(Self::FixedString(n)) 74 | } 75 | Rule::UUID => Ok(Self::Uuid), 76 | Rule::Date => Ok(Self::Date), 77 | Rule::DateTime => { 78 | let timezone = date_time::get_timezone(pair.into_inner())?; 79 | 80 | Ok(Self::DateTime(timezone)) 81 | } 82 | Rule::DateTime64 => { 83 | let (precision, timezone) = 84 | date_time64::get_precision_and_timezone(pair.into_inner())?; 85 | 86 | Ok(Self::DateTime64(precision, timezone)) 87 | } 88 | Rule::Enum8 => { 89 | let inner = r#enum::get_enum8(pair.into_inner())?; 90 | 91 | Ok(Self::Enum8(inner)) 92 | } 93 | Rule::Enum16 => { 94 | let inner = r#enum::get_enum16(pair.into_inner())?; 95 | 96 | Ok(Self::Enum16(inner)) 97 | } 98 | Rule::IPv4 => Ok(Self::Ipv4), 99 | Rule::IPv6 => Ok(Self::Ipv6), 100 | _ => Err(ParseError::Unknown), 101 | } 102 | } 103 | } 104 | 105 | pub(crate) fn get_type_name( 106 | mut nullable_pairs: Pairs<'_, Rule>, 107 | ) -> Result { 108 | let nullable_pair = nullable_pairs.next().ok_or(ParseError::Unknown)?; 109 | 110 | let mut type_name_pairs = nullable_pair.into_inner(); 111 | let type_name_pair = type_name_pairs.next().ok_or(ParseError::Unknown)?; 112 | 113 | let data_type = NullableTypeName::try_from(type_name_pair)?; 114 | 115 | Ok(data_type) 116 | } 117 | -------------------------------------------------------------------------------- /clickhouse-data-type/src/type_name_parser.rs: -------------------------------------------------------------------------------- 1 | use pest_derive::Parser; 2 | 3 | #[derive(Parser)] 4 | #[grammar = "grammars/type_name.pest"] 5 | pub(crate) struct TypeNameParser; 6 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/array.txt: -------------------------------------------------------------------------------- 1 | ["array_uint8", "array_lowcardinality_uint8", "array_nullable_uint8", "array_array_uint8", "array_tuple_uint8_nullable_nothing"] 2 | ["String", "String", "String", "String", "String"] 3 | ["Array(UInt8)", "Array(UInt8)", "Array(Nullable(UInt8))", "Array(Array(UInt8))", "Array(Tuple(UInt8, Nullable(Nothing)))"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/date.txt: -------------------------------------------------------------------------------- 1 | ["date"] 2 | ["String"] 3 | ["Date"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/datetime.txt: -------------------------------------------------------------------------------- 1 | ["datetime", "datetime_utc", "datetime_shanghai"] 2 | ["String", "String", "String"] 3 | ["DateTime", "DateTime('UTC')", "DateTime('Asia\/Shanghai')"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/datetime64.txt: -------------------------------------------------------------------------------- 1 | ["datetime64", "datetime64_utc", "datetime64_shanghai"] 2 | ["String", "String", "String"] 3 | ["DateTime64(0)", "DateTime64(3, 'UTC')", "DateTime64(9, 'Asia\/Shanghai')"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/decimal.txt: -------------------------------------------------------------------------------- 1 | ["decimal32_ct", "decimal32", "decimal64_ct", "decimal64", "decimal128_ct", "decimal128", "decimal256_ct", "decimal256"] 2 | ["String", "String", "String", "String", "String", "String", "String", "String"] 3 | ["Decimal(9, 9)", "Decimal(9, 1)", "Decimal(18, 18)", "Decimal(18, 2)", "Decimal(38, 38)", "Decimal(38, 3)", "Decimal(76, 76)", "Decimal(76, 4)"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/enum.txt: -------------------------------------------------------------------------------- 1 | ["enum8", "enum16", "enum8_2"] 2 | ["String", "String", "String"] 3 | ["Enum8('a' = -128, 'b' = 127)", "Enum16('a' = -32768, 'b' = 32767)", "Enum8('0' = 0, '1' = 1)"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/fixedstring.txt: -------------------------------------------------------------------------------- 1 | ["fixedstring"] 2 | ["String"] 3 | ["FixedString(8)"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/float.txt: -------------------------------------------------------------------------------- 1 | ["float32", "float64"] 2 | ["String", "String"] 3 | ["Float32", "Float64"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/int_uint.txt: -------------------------------------------------------------------------------- 1 | ["uint8", "uint16", "uint32", "uint64", "uint256", "int8", "int16", "int32", "int64", "int128", "int256"] 2 | ["String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String"] 3 | ["UInt8", "UInt16", "UInt32", "UInt64", "UInt256", "Int8", "Int16", "Int32", "Int64", "Int128", "Int256"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/ipv4.txt: -------------------------------------------------------------------------------- 1 | ["ipv4"] 2 | ["String"] 3 | ["IPv4"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/ipv6.txt: -------------------------------------------------------------------------------- 1 | ["ipv4"] 2 | ["String"] 3 | ["IPv6"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/lowcardinality.txt: -------------------------------------------------------------------------------- 1 | ["lowcardinality_string", "lowcardinality_fixedstring", "lowcardinality_date", "lowcardinality_datetime", "lowcardinality_uint8", "lowcardinality_uint16", "lowcardinality_uint32", "lowcardinality_uint64", "lowcardinality_int8", "lowcardinality_int16", "lowcardinality_int32", "lowcardinality_int64", "lowcardinality_float32", "lowcardinality_float64", "lowcardinality_nullable_string", "lowcardinality_nullable_string_2", "lowcardinality_ipv4", "lowcardinality_ipv6"] 2 | ["String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String"] 3 | ["LowCardinality(String)", "LowCardinality(FixedString(1))", "LowCardinality(Date)", "LowCardinality(DateTime)", "LowCardinality(UInt8)", "LowCardinality(UInt16)", "LowCardinality(UInt32)", "LowCardinality(UInt64)", "LowCardinality(Int8)", "LowCardinality(Int16)", "LowCardinality(Int32)", "LowCardinality(Int64)", "LowCardinality(Float32)", "LowCardinality(Float64)", "LowCardinality(Nullable(String))", "LowCardinality(Nullable(String))", "LowCardinality(IPv4)", "LowCardinality(IPv6)"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/map.txt: -------------------------------------------------------------------------------- 1 | ["map_string_string", "map_fixedstring_string", "map_uint256_string", "map_int256_string", "map_float64_string", "map_decimal_string", "map_string_array_string"] 2 | ["String", "String", "String", "String", "String", "String", "String"] 3 | ["Map(String,String)", "Map(FixedString(2),String)", "Map(UInt256,String)", "Map(Int256,String)", "Map(Float64,String)", "Map(Decimal(9, 9),String)", "Map(String,Array(String))"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/nullable.txt: -------------------------------------------------------------------------------- 1 | ["nullable_uint8", "nullable_uint256", "nullable_int8", "nullable_int256", "nullable_float64", "nullable_decimal256", "nullable_string", "nullable_fixedstring", "nullable_uuid", "nullable_date", "nullable_datetime", "nullable_datetime64", "nullable_enum8", "nullable_nothing", "nullable_ipv4", "nullable_ipv6"] 2 | ["String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String", "String"] 3 | ["Nullable(UInt8)", "Nullable(UInt256)", "Nullable(Int8)", "Nullable(Int256)", "Nullable(Float64)", "Nullable(Decimal(76, 4))", "Nullable(String)", "Nullable(FixedString(1))", "Nullable(UUID)", "Nullable(Date)", "Nullable(DateTime)", "Nullable(DateTime64(0))", "Nullable(Enum8('a' = -128, 'b' = 127))", "Nullable(Nothing)", "Nullable(IPv4)", "Nullable(IPv6)"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/string.txt: -------------------------------------------------------------------------------- 1 | ["string"] 2 | ["String"] 3 | ["String"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/tuple.txt: -------------------------------------------------------------------------------- 1 | ["tuple_string_uint8", "tuple_string_lowcardinality_uint8", "tuple_string_nullable_uint8", "tuple_string_array_uint8", "tuple_string_tuple_uint8_nullable_nothing"] 2 | ["String", "String", "String", "String", "String"] 3 | ["Tuple(String, UInt8)", "Tuple(String, UInt8)", "Tuple(String, Nullable(UInt8))", "Tuple(String, Array(UInt8))", "Tuple(String, Tuple(UInt8, Nullable(Nothing)))"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-type/tests/files/uuid.txt: -------------------------------------------------------------------------------- 1 | ["uuid"] 2 | ["String"] 3 | ["UUID"] 4 | -------------------------------------------------------------------------------- /clickhouse-data-value/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-data-value" 3 | version = "0.3.1" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "ClickHouse Data Values" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/clickhouse-data-value" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [dependencies] 16 | pest = { version = "2.5", default-features = false, features = ["std"] } 17 | pest_derive = { version = "2.5", default-features = false, features = ["std"] } 18 | 19 | serde = { version = "1", default-features = false, features = ["derive"] } 20 | thiserror = { version = "1", default-features = false } 21 | 22 | chrono = { version = "0.4", default-features = false } 23 | 24 | [dev-dependencies] 25 | serde_json = { version = "1" } 26 | chrono = { version = "0.4", features = ["serde"] } 27 | -------------------------------------------------------------------------------- /clickhouse-data-value/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /clickhouse-data-value/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /clickhouse-data-value/README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-data-value 2 | 3 | * [Cargo package](https://crates.io/crates/clickhouse-data-value) 4 | -------------------------------------------------------------------------------- /clickhouse-data-value/src/date.rs: -------------------------------------------------------------------------------- 1 | use core::{ 2 | ops::{Deref, DerefMut}, 3 | str::FromStr, 4 | }; 5 | 6 | use chrono::NaiveDate as ChronoNaiveDate; 7 | use serde::{ 8 | de::{self, Visitor}, 9 | Deserialize, Deserializer, 10 | }; 11 | 12 | #[derive(PartialEq, Debug, Clone)] 13 | pub struct NaiveDate(pub ChronoNaiveDate); 14 | impl From for NaiveDate { 15 | fn from(inner: ChronoNaiveDate) -> Self { 16 | Self(inner) 17 | } 18 | } 19 | impl Deref for NaiveDate { 20 | type Target = ChronoNaiveDate; 21 | 22 | fn deref(&self) -> &Self::Target { 23 | &self.0 24 | } 25 | } 26 | impl DerefMut for NaiveDate { 27 | fn deref_mut(&mut self) -> &mut Self::Target { 28 | &mut self.0 29 | } 30 | } 31 | 32 | pub type ParseError = chrono::ParseError; 33 | impl FromStr for NaiveDate { 34 | type Err = ParseError; 35 | 36 | fn from_str(s: &str) -> Result { 37 | ChronoNaiveDate::parse_from_str(s, "%Y-%m-%d").map(Into::into) 38 | } 39 | } 40 | 41 | struct NaiveDateVisitor; 42 | impl<'de> Visitor<'de> for NaiveDateVisitor { 43 | type Value = NaiveDate; 44 | 45 | fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result { 46 | formatter.write_str("format %Y-%m-%d") 47 | } 48 | 49 | fn visit_str(self, string: &str) -> Result 50 | where 51 | E: de::Error, 52 | { 53 | string 54 | .parse() 55 | .map_err(|err: ParseError| de::Error::custom(err.to_string())) 56 | } 57 | } 58 | impl<'de> Deserialize<'de> for NaiveDate { 59 | fn deserialize(deserializer: D) -> Result 60 | where 61 | D: Deserializer<'de>, 62 | { 63 | deserializer.deserialize_str(NaiveDateVisitor) 64 | } 65 | } 66 | pub fn deserialize<'de, D>(d: D) -> Result 67 | where 68 | D: de::Deserializer<'de>, 69 | { 70 | d.deserialize_str(NaiveDateVisitor).map(|x| x.0) 71 | } 72 | 73 | #[cfg(test)] 74 | mod tests { 75 | use super::*; 76 | 77 | use chrono::NaiveDate; 78 | 79 | #[test] 80 | fn test_parse() -> Result<(), Box> { 81 | assert_eq!( 82 | "2021-03-01" 83 | .parse::() 84 | .map_err(|err| err.to_string())?, 85 | NaiveDate::from_ymd_opt(2021, 3, 1).expect("") 86 | ); 87 | 88 | Ok(()) 89 | } 90 | 91 | #[derive(Deserialize)] 92 | struct Row { 93 | #[serde(deserialize_with = "crate::date::deserialize")] 94 | date: chrono::NaiveDate, 95 | } 96 | 97 | #[test] 98 | fn test_de() -> Result<(), Box> { 99 | let deserializer = de::IntoDeserializer::::into_deserializer; 100 | assert_eq!( 101 | super::deserialize(deserializer("2021-03-01")).unwrap(), 102 | NaiveDate::from_ymd_opt(2021, 3, 1).expect("") 103 | ); 104 | 105 | let content = include_str!("../tests/files/date.txt"); 106 | let line = content.lines().next().unwrap(); 107 | 108 | let Row { date } = serde_json::from_str(line)?; 109 | assert_eq!(date, NaiveDate::from_ymd_opt(2021, 3, 1).expect("")); 110 | 111 | Ok(()) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /clickhouse-data-value/src/date_and_time_parser.rs: -------------------------------------------------------------------------------- 1 | use pest_derive::Parser; 2 | 3 | #[derive(Parser)] 4 | #[grammar = "grammars/date_and_time.pest"] 5 | pub(crate) struct DateAndTimeParser; 6 | -------------------------------------------------------------------------------- /clickhouse-data-value/src/grammars/date_and_time.pest: -------------------------------------------------------------------------------- 1 | datetime = { datetime_simple | datetime_iso | datetime_unix_timestamp } 2 | 3 | datetime_simple = { date ~ " " ~ time ~ ( "." ~ time_nf )? } 4 | datetime_iso = { date ~ "T" ~ time ~ ( "." ~ time_nf )? ~ "Z" } 5 | datetime_unix_timestamp = { unix_timestamp ~ ( "." ~ time_nf )? } 6 | 7 | date = @{ date_Y ~ "-" ~ date_m ~ "-" ~ date_d } 8 | time = @{ time_H ~ ":" ~ time_M ~ ":" ~ time_S } 9 | unix_timestamp = @{ ASCII_DIGIT{1, 10} } 10 | 11 | date_Y = @{ "20" ~ ASCII_DIGIT{2} | "199" ~ ASCII_DIGIT | "198" ~ ASCII_DIGIT | "197" ~ ASCII_DIGIT | "2101" | "2102" | "2103" | "2104" | "2105" } 12 | date_m = @{ "0" ~ ASCII_NONZERO_DIGIT | "10" | "11" | "12" } 13 | date_d = @{ "0" ~ ASCII_NONZERO_DIGIT | "1" ~ ASCII_DIGIT | "2" ~ ASCII_DIGIT | "30" | "31" } 14 | 15 | time_H = @{ "0" ~ ASCII_DIGIT | "1" ~ ASCII_DIGIT | "20" | "21" | "22" | "23" } 16 | time_M = @{ "0" ~ ASCII_DIGIT | "1" ~ ASCII_DIGIT | "2" ~ ASCII_DIGIT | "3" ~ ASCII_DIGIT | "4" ~ ASCII_DIGIT | "5" ~ ASCII_DIGIT } 17 | time_S = @{ "0" ~ ASCII_DIGIT | "1" ~ ASCII_DIGIT | "2" ~ ASCII_DIGIT | "3" ~ ASCII_DIGIT | "4" ~ ASCII_DIGIT | "5" ~ ASCII_DIGIT | "60" } 18 | 19 | time_nf = @{ ASCII_DIGIT{1, 9} } 20 | -------------------------------------------------------------------------------- /clickhouse-data-value/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod date_and_time_parser; 2 | 3 | pub mod date; 4 | pub mod datetime; 5 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/date.txt: -------------------------------------------------------------------------------- 1 | {"date":"2021-03-01"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime64_iso.txt: -------------------------------------------------------------------------------- 1 | {"datetime64_precision0_utc":"2021-03-01T01:02:03Z","datetime64_precision1_utc":"2021-03-01T01:02:03.1Z","datetime64_milli_utc":"2021-03-01T01:02:03.123Z","datetime64_milli_shanghai":"2021-02-28T17:02:03.123Z","datetime64_micro_utc":"2021-03-01T01:02:03.123456Z","datetime64_micro_shanghai":"2021-02-28T17:02:03.123456Z","datetime64_nano_utc":"2021-03-01T01:02:03.123456789Z","datetime64_nano_shanghai":"2021-02-28T17:02:03.123456789Z"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime64_simple.txt: -------------------------------------------------------------------------------- 1 | {"datetime64_precision0_utc":"2021-03-01 01:02:03","datetime64_precision1_utc":"2021-03-01 01:02:03.1","datetime64_milli_utc":"2021-03-01 01:02:03.123","datetime64_milli_shanghai":"2021-03-01 01:02:03.123","datetime64_micro_utc":"2021-03-01 01:02:03.123456","datetime64_micro_shanghai":"2021-03-01 01:02:03.123456","datetime64_nano_utc":"2021-03-01 01:02:03.123456789","datetime64_nano_shanghai":"2021-03-01 01:02:03.123456789"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime64_unix_timestamp.txt: -------------------------------------------------------------------------------- 1 | {"datetime64_precision0_utc":"1614560523","datetime64_precision1_utc":"1614560523.1","datetime64_milli_utc":"1614560523.123","datetime64_milli_shanghai":"1614531723.123","datetime64_micro_utc":"1614560523.123456","datetime64_micro_shanghai":"1614531723.123456","datetime64_nano_utc":"1614560523.123456789","datetime64_nano_shanghai":"1614531723.123456789"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime_iso.txt: -------------------------------------------------------------------------------- 1 | {"datetime_utc":"2021-03-01T01:02:03Z","datetime_shanghai":"2021-02-28T17:02:03Z"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime_simple.txt: -------------------------------------------------------------------------------- 1 | {"datetime_utc":"2021-03-01 01:02:03","datetime_shanghai":"2021-03-01 01:02:03"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/files/datetime_unix_timestamp.txt: -------------------------------------------------------------------------------- 1 | {"datetime_utc":"1614560523","datetime_shanghai":"1614531723"} 2 | -------------------------------------------------------------------------------- /clickhouse-data-value/tests/gen_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Prerequire 6 | # ./../clickhouse_tgz_archive/download.sh 7 | 8 | # ./tests/gen_files.sh 9 | 10 | script_path=$(cd $(dirname $0) ; pwd -P) 11 | script_path_root="${script_path}/" 12 | 13 | bin_server="${script_path_root}../../clickhouse_tgz_archive/clickhouse/usr/bin/clickhouse-server" 14 | bin_client="${script_path_root}../../clickhouse_tgz_archive/clickhouse/usr/bin/clickhouse-client" 15 | 16 | workdir=$(mktemp -d) 17 | 18 | mkdir -p "${workdir}/lib" 19 | path="${workdir}/lib/" 20 | 21 | mkdir -p "${workdir}/etc" 22 | config_file="${workdir}/etc/config.xml" 23 | tee "${config_file}" </dev/null 24 | 25 | 26 | trace 27 | true 28 | 29 | 30 | 9000 31 | 32 | ${path} 33 | 34 | 8589934592 35 | 5368709120 36 | true 37 | 38 | 39 | 40 | xxx 41 | 42 | 43 | ::/0 44 | 45 | 46 | default 47 | default 48 | 1 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | EOF 61 | 62 | mkdir -p "${workdir}/log" 63 | log_file="${workdir}/log/clickhouse-server.log" 64 | errorlog_file="${workdir}/log/clickhouse-server.err.log" 65 | 66 | mkdir -p "${workdir}/run" 67 | pid_file="${workdir}/run/clickhouse-server.pid" 68 | 69 | # https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port 70 | read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range 71 | tcp_port=$(comm -23 <(seq $LOWERPORT $UPPERPORT | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1) 72 | 73 | cleanup() { 74 | test -f "${pid_file}" && kill $(cat "${pid_file}") 75 | test -f "${errorlog_file}" && (cat "${errorlog_file}" | grep -v 'Connection reset by peer' | grep -v 'Broken pipe') 76 | rm -rf "${workdir}" 77 | } 78 | trap cleanup EXIT 79 | 80 | $(${bin_server} --config-file="${config_file}" --log-file="${log_file}" --errorlog-file="${errorlog_file}" --pid-file="${pid_file}" --daemon -- --path="${path}" --tcp_port=${tcp_port}) 81 | 82 | sleep 2 83 | 84 | files_path="${script_path_root}files" 85 | 86 | query_date=$(cat <<-END 87 | SELECT 88 | toDate('2021-03-01') as date 89 | END 90 | ) 91 | $(echo ${query_date} FORMAT JSONEachRow | ${bin_client} --port ${tcp_port} --password xxx > "${files_path}/date.txt") 92 | 93 | 94 | query_datetime=$(cat <<-END 95 | SELECT 96 | toDateTime('2021-03-01 01:02:03', 'UTC') as datetime_utc, 97 | toDateTime('2021-03-01 01:02:03', 'Asia/Shanghai') as datetime_shanghai 98 | END 99 | ) 100 | $(echo ${query_datetime} FORMAT JSONEachRow | ${bin_client} --date_time_output_format simple --port ${tcp_port} --password xxx > "${files_path}/datetime_simple.txt") 101 | $(echo ${query_datetime} FORMAT JSONEachRow | ${bin_client} --date_time_output_format iso --port ${tcp_port} --password xxx > "${files_path}/datetime_iso.txt") 102 | $(echo ${query_datetime} FORMAT JSONEachRow | ${bin_client} --date_time_output_format unix_timestamp --port ${tcp_port} --password xxx > "${files_path}/datetime_unix_timestamp.txt") 103 | 104 | query_datetime64=$(cat <<-END 105 | SELECT 106 | toDateTime64('2021-03-01 01:02:03.123456789', 0, 'UTC') as datetime64_precision0_utc, 107 | toDateTime64('2021-03-01 01:02:03.123456789', 1, 'UTC') as datetime64_precision1_utc, 108 | toDateTime64('2021-03-01 01:02:03.123456789', 3, 'UTC') as datetime64_milli_utc, 109 | toDateTime('2021-03-01 01:02:03.123456789', 3, 'Asia/Shanghai') as datetime64_milli_shanghai, 110 | toDateTime64('2021-03-01 01:02:03.123456789', 6, 'UTC') as datetime64_micro_utc, 111 | toDateTime('2021-03-01 01:02:03.123456789', 6, 'Asia/Shanghai') as datetime64_micro_shanghai, 112 | toDateTime64('2021-03-01 01:02:03.123456789', 9, 'UTC') as datetime64_nano_utc, 113 | toDateTime('2021-03-01 01:02:03.123456789', 9, 'Asia/Shanghai') as datetime64_nano_shanghai 114 | END 115 | ) 116 | $(echo ${query_datetime64} FORMAT JSONEachRow | ${bin_client} --date_time_output_format simple --port ${tcp_port} --password xxx > "${files_path}/datetime64_simple.txt") 117 | $(echo ${query_datetime64} FORMAT JSONEachRow | ${bin_client} --date_time_output_format iso --port ${tcp_port} --password xxx > "${files_path}/datetime64_iso.txt") 118 | $(echo ${query_datetime64} FORMAT JSONEachRow | ${bin_client} --date_time_output_format unix_timestamp --port ${tcp_port} --password xxx > "${files_path}/datetime64_unix_timestamp.txt") 119 | 120 | 121 | sleep 1 122 | -------------------------------------------------------------------------------- /clickhouse-format/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-format" 3 | version = "0.3.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "ClickHouse Formats" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/clickhouse-format" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [package.metadata.docs.rs] 16 | features = ["with-all"] 17 | 18 | [features] 19 | default = ["with-json"] 20 | 21 | with-all = ["with-tsv", "with-json"] 22 | 23 | with-tsv = ["csv", "serde"] 24 | with-json = ["serde_json", "serde", "thiserror", "serde-aux"] 25 | 26 | [dependencies] 27 | strum = { version = "0.24", default-features = false, features = ["derive"] } 28 | 29 | csv = { version = "1", default-features = false, optional = true } 30 | serde_json = { version = "1", default-features = false, features = ["std"], optional = true } 31 | serde = { version = "1", default-features = false, features = ["std", "derive"], optional = true } 32 | thiserror = { version = "1", default-features = false, optional = true } 33 | serde-aux = { version = "4", default-features = false, optional = true } 34 | 35 | [dev-dependencies] 36 | serde = { version = "1", features = ["derive"] } 37 | once_cell = { version = "1" } 38 | 39 | [package.metadata.cargo-all-features] 40 | skip_optional_dependencies = true 41 | -------------------------------------------------------------------------------- /clickhouse-format/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /clickhouse-format/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /clickhouse-format/README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-format 2 | 3 | * [ClickHouse Doc](https://clickhouse.tech/docs/en/interfaces/formats/) 4 | * [Cargo package](https://crates.io/crates/clickhouse-format) 5 | -------------------------------------------------------------------------------- /clickhouse-format/src/format_name.rs: -------------------------------------------------------------------------------- 1 | #[derive(strum::Display, strum::EnumString, PartialEq, Eq, Debug, Clone)] 2 | pub enum FormatName { 3 | // 4 | #[strum(serialize = "JSON")] 5 | Json, 6 | #[strum(serialize = "JSONStrings")] 7 | JsonStrings, 8 | #[strum(serialize = "JSONCompact")] 9 | JsonCompact, 10 | #[strum(serialize = "JSONCompactStrings")] 11 | JsonCompactStrings, 12 | // 13 | #[strum(serialize = "TSV")] 14 | Tsv, 15 | #[strum(serialize = "TSVRaw")] 16 | TsvRaw, 17 | #[strum(serialize = "TSVWithNames")] 18 | TsvWithNames, 19 | #[strum(serialize = "TSVWithNamesAndTypes")] 20 | TsvWithNamesAndTypes, 21 | // 22 | #[strum(serialize = "JSONEachRow")] 23 | JsonEachRow, 24 | #[strum(serialize = "JSONStringsEachRow")] 25 | JsonStringsEachRow, 26 | #[strum(serialize = "JSONCompactEachRow")] 27 | JsonCompactEachRow, 28 | #[strum(serialize = "JSONCompactStringsEachRow")] 29 | JsonCompactStringsEachRow, 30 | #[strum(serialize = "JSONEachRowWithProgress")] 31 | JsonEachRowWithProgress, 32 | #[strum(serialize = "JSONStringsEachRowWithProgress")] 33 | JsonStringsEachRowWithProgress, 34 | #[strum(serialize = "JSONCompactEachRowWithNamesAndTypes")] 35 | JsonCompactEachRowWithNamesAndTypes, 36 | #[strum(serialize = "JSONCompactStringsEachRowWithNamesAndTypes")] 37 | JsonCompactStringsEachRowWithNamesAndTypes, 38 | } 39 | -------------------------------------------------------------------------------- /clickhouse-format/src/input/json_compact_each_row.rs: -------------------------------------------------------------------------------- 1 | use serde::Serialize; 2 | use serde_json::{ser::CompactFormatter, Serializer}; 3 | 4 | use crate::{format_name::FormatName, input::Input}; 5 | 6 | pub struct JsonCompactEachRowInput { 7 | rows: Vec>, 8 | } 9 | impl JsonCompactEachRowInput { 10 | pub fn new(rows: Vec>) -> Self { 11 | Self { rows } 12 | } 13 | } 14 | 15 | impl Input for JsonCompactEachRowInput 16 | where 17 | T: Serialize, 18 | { 19 | type Error = serde_json::Error; 20 | 21 | fn format_name() -> FormatName { 22 | FormatName::JsonCompactEachRow 23 | } 24 | 25 | fn serialize(&self) -> Result, Self::Error> { 26 | let mut buf = vec![]; 27 | let mut ser_buf = Vec::with_capacity(128); 28 | 29 | for row in &self.rows { 30 | buf.push(b'['); 31 | for (i, item) in row.iter().enumerate() { 32 | ser_buf.clear(); 33 | let mut ser = Serializer::with_formatter(&mut ser_buf, CompactFormatter); 34 | item.serialize(&mut ser)?; 35 | 36 | buf.extend_from_slice(ser.into_inner()); 37 | 38 | if i < (row.len() - 1) { 39 | buf.extend_from_slice(b", "); 40 | } 41 | } 42 | buf.push(b']'); 43 | 44 | buf.push(b'\n'); 45 | } 46 | 47 | Ok(buf) 48 | } 49 | } 50 | 51 | #[cfg(test)] 52 | mod tests { 53 | use super::*; 54 | 55 | use std::{fs, path::PathBuf}; 56 | 57 | use crate::test_helpers::{TEST_ROW_1, TEST_ROW_2}; 58 | 59 | use serde_json::{Map, Value}; 60 | 61 | #[test] 62 | fn simple() -> Result<(), Box> { 63 | let file_path = PathBuf::new().join("tests/files/JSONCompactEachRow.txt"); 64 | let content = fs::read_to_string(&file_path)?; 65 | 66 | assert_eq!( 67 | JsonCompactEachRowInput::<()>::format_name(), 68 | file_path 69 | .file_stem() 70 | .unwrap() 71 | .to_string_lossy() 72 | .parse() 73 | .unwrap() 74 | ); 75 | 76 | let mut rows: Vec> = vec![]; 77 | rows.push(vec![ 78 | TEST_ROW_1.array1.to_owned().into(), 79 | TEST_ROW_1.array2.to_owned().into(), 80 | vec![ 81 | Value::Number(TEST_ROW_1.tuple1.to_owned().0.into()), 82 | Value::String(TEST_ROW_1.tuple1.to_owned().1), 83 | ] 84 | .into(), 85 | vec![ 86 | Value::Number(TEST_ROW_1.tuple2.to_owned().0.into()), 87 | Value::Null, 88 | ] 89 | .into(), 90 | Value::Object(TEST_ROW_1.map1.iter().fold(Map::new(), |mut m, (k, v)| { 91 | m.insert(k.to_owned(), Value::String(v.to_owned())); 92 | m 93 | })), 94 | ]); 95 | rows.push(vec![ 96 | TEST_ROW_2.array1.to_owned().into(), 97 | TEST_ROW_2.array2.to_owned().into(), 98 | vec![ 99 | Value::Number(TEST_ROW_2.tuple1.to_owned().0.into()), 100 | Value::String(TEST_ROW_2.tuple1.to_owned().1), 101 | ] 102 | .into(), 103 | vec![ 104 | Value::Number(TEST_ROW_2.tuple2.to_owned().0.into()), 105 | Value::String(TEST_ROW_2.tuple2.to_owned().1.unwrap()), 106 | ] 107 | .into(), 108 | Value::Object(TEST_ROW_2.map1.iter().fold(Map::new(), |mut m, (k, v)| { 109 | m.insert(k.to_owned(), Value::String(v.to_owned())); 110 | m 111 | })), 112 | ]); 113 | 114 | let bytes = JsonCompactEachRowInput::new(rows).serialize()?; 115 | assert_eq!(bytes, content.as_bytes()); 116 | 117 | Ok(()) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /clickhouse-format/src/input/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::format_name::FormatName; 2 | 3 | #[cfg(feature = "with-json")] 4 | pub mod json_compact_each_row; 5 | 6 | #[cfg(feature = "with-json")] 7 | pub use self::json_compact_each_row::JsonCompactEachRowInput; 8 | 9 | pub trait Input { 10 | type Error: std::error::Error; 11 | 12 | fn format_name() -> FormatName; 13 | fn serialize(&self) -> Result, Self::Error>; 14 | } 15 | -------------------------------------------------------------------------------- /clickhouse-format/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod format_name; 2 | pub mod input; 3 | pub mod output; 4 | 5 | #[cfg(test)] 6 | pub(crate) mod test_helpers; 7 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use serde::{de::DeserializeOwned, Deserialize}; 5 | use serde_json::Value; 6 | 7 | use crate::format_name::FormatName; 8 | 9 | use super::{Output, OutputResult}; 10 | 11 | pub struct JsonOutput { 12 | phantom: PhantomData, 13 | } 14 | impl Default for JsonOutput { 15 | fn default() -> Self { 16 | Self::new() 17 | } 18 | } 19 | impl JsonOutput { 20 | pub fn new() -> Self { 21 | Self { 22 | phantom: PhantomData, 23 | } 24 | } 25 | } 26 | pub type GeneralJsonOutput = JsonOutput>; 27 | 28 | impl Output for JsonOutput 29 | where 30 | T: DeserializeOwned, 31 | { 32 | type Row = T; 33 | type Info = JsonDataInfo; 34 | 35 | type Error = serde_json::Error; 36 | 37 | fn format_name() -> FormatName { 38 | FormatName::Json 39 | } 40 | 41 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 42 | let json_data: JsonData = serde_json::from_slice(slice)?; 43 | let JsonData { 44 | meta, 45 | data, 46 | rows, 47 | statistics, 48 | } = json_data; 49 | Ok(( 50 | data, 51 | JsonDataInfo { 52 | meta, 53 | rows, 54 | statistics, 55 | }, 56 | )) 57 | } 58 | } 59 | 60 | #[derive(Deserialize, Debug, Clone)] 61 | pub(crate) struct JsonData 62 | where 63 | T: Sized, 64 | { 65 | pub meta: Vec, 66 | pub data: Vec, 67 | pub rows: usize, 68 | pub statistics: JsonDataStatistics, 69 | } 70 | #[derive(Deserialize, Debug, Clone)] 71 | pub struct JsonDataMetaItem { 72 | pub name: String, 73 | pub r#type: String, 74 | } 75 | #[derive(Deserialize, Debug, Clone)] 76 | pub struct JsonDataStatistics { 77 | pub elapsed: f64, 78 | pub rows_read: usize, 79 | pub bytes_read: usize, 80 | } 81 | pub struct JsonDataInfo { 82 | pub meta: Vec, 83 | pub rows: usize, 84 | pub statistics: JsonDataStatistics, 85 | } 86 | 87 | #[cfg(test)] 88 | mod tests { 89 | use super::*; 90 | 91 | use std::{fs, path::PathBuf}; 92 | 93 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 94 | 95 | #[test] 96 | fn simple() -> Result<(), Box> { 97 | let file_path = PathBuf::new().join("tests/files/JSON.json"); 98 | let content = fs::read_to_string(&file_path)?; 99 | 100 | assert_eq!( 101 | GeneralJsonOutput::format_name(), 102 | file_path 103 | .file_stem() 104 | .unwrap() 105 | .to_string_lossy() 106 | .parse() 107 | .unwrap() 108 | ); 109 | 110 | let (rows, info) = GeneralJsonOutput::new().deserialize(content.as_bytes())?; 111 | assert_eq!( 112 | rows.first().unwrap().get("tuple1").unwrap(), 113 | &Value::Array(vec![1.into(), "a".into()]) 114 | ); 115 | assert_eq!(info.rows, 2); 116 | 117 | let (rows, info) = JsonOutput::::new().deserialize(content.as_bytes())?; 118 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 119 | assert_eq!(info.rows, 2); 120 | 121 | Ok(()) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use serde::de::DeserializeOwned; 5 | use serde_json::{Map, Value}; 6 | 7 | use crate::format_name::FormatName; 8 | 9 | use super::{ 10 | json::{JsonData, JsonDataInfo}, 11 | Output, OutputResult, 12 | }; 13 | 14 | pub struct JsonCompactOutput { 15 | phantom: PhantomData, 16 | } 17 | impl Default for JsonCompactOutput { 18 | fn default() -> Self { 19 | Self::new() 20 | } 21 | } 22 | impl JsonCompactOutput { 23 | pub fn new() -> Self { 24 | Self { 25 | phantom: PhantomData, 26 | } 27 | } 28 | } 29 | pub type GeneralJsonCompactOutput = JsonCompactOutput>; 30 | 31 | impl Output for JsonCompactOutput 32 | where 33 | T: DeserializeOwned, 34 | { 35 | type Row = T; 36 | type Info = JsonDataInfo; 37 | 38 | type Error = serde_json::Error; 39 | 40 | fn format_name() -> FormatName { 41 | FormatName::JsonCompact 42 | } 43 | 44 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 45 | self.deserialize_with::(slice) 46 | } 47 | } 48 | 49 | impl JsonCompactOutput 50 | where 51 | T: DeserializeOwned, 52 | { 53 | pub(crate) fn deserialize_with( 54 | &self, 55 | slice: &[u8], 56 | ) -> OutputResult<::Row, ::Info, ::Error> 57 | where 58 | V: DeserializeOwned + Into, 59 | { 60 | let json_data_tmp: JsonData> = serde_json::from_slice(slice)?; 61 | 62 | let keys: Vec<_> = json_data_tmp 63 | .meta 64 | .iter() 65 | .map(|x| x.name.to_owned()) 66 | .collect(); 67 | let mut data: Vec = vec![]; 68 | for values in json_data_tmp.data.into_iter() { 69 | let map: Map<_, _> = keys 70 | .iter() 71 | .zip(values) 72 | .map(|(k, v)| (k.to_owned(), v.into())) 73 | .collect(); 74 | data.push(serde_json::from_value(Value::Object(map))?); 75 | } 76 | 77 | Ok(( 78 | data, 79 | JsonDataInfo { 80 | meta: json_data_tmp.meta, 81 | rows: json_data_tmp.rows, 82 | statistics: json_data_tmp.statistics, 83 | }, 84 | )) 85 | } 86 | } 87 | 88 | #[cfg(test)] 89 | mod tests { 90 | use super::*; 91 | 92 | use std::{fs, path::PathBuf}; 93 | 94 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 95 | 96 | #[test] 97 | fn simple() -> Result<(), Box> { 98 | let file_path = PathBuf::new().join("tests/files/JSONCompact.json"); 99 | let content = fs::read_to_string(&file_path)?; 100 | 101 | assert_eq!( 102 | GeneralJsonCompactOutput::format_name(), 103 | file_path 104 | .file_stem() 105 | .unwrap() 106 | .to_string_lossy() 107 | .parse() 108 | .unwrap() 109 | ); 110 | 111 | let (rows, info) = GeneralJsonCompactOutput::new().deserialize(content.as_bytes())?; 112 | assert_eq!( 113 | rows.first().unwrap().get("tuple1").unwrap(), 114 | &Value::Array(vec![1.into(), "a".into()]) 115 | ); 116 | assert_eq!(info.rows, 2); 117 | 118 | let (rows, info) = JsonCompactOutput::::new().deserialize(content.as_bytes())?; 119 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 120 | assert_eq!(info.rows, 2); 121 | 122 | Ok(()) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact_each_row.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | collections::HashMap, 4 | io::{BufRead as _, Error as IoError}, 5 | }; 6 | 7 | use serde::de::DeserializeOwned; 8 | use serde_json::Value; 9 | 10 | use crate::format_name::FormatName; 11 | 12 | use super::{Output, OutputResult}; 13 | 14 | pub struct JsonCompactEachRowOutput { 15 | names: Vec, 16 | phantom: PhantomData, 17 | } 18 | impl JsonCompactEachRowOutput { 19 | pub fn new(names: Vec) -> Self { 20 | Self { 21 | names, 22 | phantom: PhantomData, 23 | } 24 | } 25 | } 26 | 27 | pub type GeneralJsonCompactEachRowOutput = JsonCompactEachRowOutput>; 28 | 29 | #[derive(thiserror::Error, Debug)] 30 | pub enum JsonCompactEachRowOutputError { 31 | #[error("IoError {0:?}")] 32 | IoError(#[from] IoError), 33 | #[error("SerdeJsonError {0:?}")] 34 | SerdeJsonError(#[from] serde_json::Error), 35 | } 36 | 37 | impl Output for JsonCompactEachRowOutput 38 | where 39 | T: DeserializeOwned, 40 | { 41 | type Row = T; 42 | type Info = (); 43 | 44 | type Error = JsonCompactEachRowOutputError; 45 | 46 | fn format_name() -> FormatName { 47 | FormatName::JsonCompactEachRow 48 | } 49 | 50 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 51 | let mut data: Vec = vec![]; 52 | 53 | for line in slice.lines() { 54 | let line = line?; 55 | let values: Vec = serde_json::from_str(&line)?; 56 | 57 | let row: T = serde_json::from_value(Value::Object( 58 | self.names.iter().cloned().zip(values).collect(), 59 | ))?; 60 | 61 | data.push(row); 62 | } 63 | 64 | Ok((data, ())) 65 | } 66 | } 67 | 68 | #[cfg(test)] 69 | mod tests { 70 | use super::*; 71 | 72 | use std::{fs, path::PathBuf}; 73 | 74 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 75 | 76 | #[test] 77 | fn simple() -> Result<(), Box> { 78 | let file_path = PathBuf::new().join("tests/files/JSONCompactEachRow.txt"); 79 | let content = fs::read_to_string(&file_path)?; 80 | 81 | assert_eq!( 82 | GeneralJsonCompactEachRowOutput::format_name(), 83 | file_path 84 | .file_stem() 85 | .unwrap() 86 | .to_string_lossy() 87 | .parse() 88 | .unwrap() 89 | ); 90 | 91 | let (rows, _info): (_, ()) = GeneralJsonCompactEachRowOutput::new(vec![ 92 | "array1".into(), 93 | "array2".into(), 94 | "tuple1".into(), 95 | "tuple2".into(), 96 | "map1".into(), 97 | ]) 98 | .deserialize(content.as_bytes())?; 99 | assert_eq!( 100 | rows.first().unwrap().get("tuple1").unwrap(), 101 | &Value::Array(vec![1.into(), "a".into()]) 102 | ); 103 | 104 | let (rows, _info): (_, ()) = JsonCompactEachRowOutput::::new(vec![ 105 | "array1".into(), 106 | "array2".into(), 107 | "tuple1".into(), 108 | "tuple2".into(), 109 | "map1".into(), 110 | ]) 111 | .deserialize(content.as_bytes())?; 112 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 113 | 114 | Ok(()) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact_each_row_with_names_and_types.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | collections::HashMap, 4 | io::{BufRead as _, Error as IoError}, 5 | }; 6 | 7 | use serde::de::DeserializeOwned; 8 | use serde_json::Value; 9 | 10 | use crate::format_name::FormatName; 11 | 12 | use super::{Output, OutputResult}; 13 | 14 | pub struct JsonCompactEachRowWithNamesAndTypesOutput { 15 | phantom: PhantomData, 16 | } 17 | impl Default for JsonCompactEachRowWithNamesAndTypesOutput { 18 | fn default() -> Self { 19 | Self::new() 20 | } 21 | } 22 | impl JsonCompactEachRowWithNamesAndTypesOutput { 23 | pub fn new() -> Self { 24 | Self { 25 | phantom: PhantomData, 26 | } 27 | } 28 | } 29 | 30 | pub type GeneralJsonCompactEachRowWithNamesAndTypesOutput = 31 | JsonCompactEachRowWithNamesAndTypesOutput>; 32 | 33 | #[derive(thiserror::Error, Debug)] 34 | pub enum JsonCompactEachRowWithNamesAndTypesOutputError { 35 | #[error("IoError {0:?}")] 36 | IoError(#[from] IoError), 37 | #[error("SerdeJsonError {0:?}")] 38 | SerdeJsonError(#[from] serde_json::Error), 39 | } 40 | 41 | impl Output for JsonCompactEachRowWithNamesAndTypesOutput 42 | where 43 | T: DeserializeOwned, 44 | { 45 | type Row = T; 46 | type Info = HashMap; 47 | 48 | type Error = JsonCompactEachRowWithNamesAndTypesOutputError; 49 | 50 | fn format_name() -> FormatName { 51 | FormatName::JsonCompactEachRowWithNamesAndTypes 52 | } 53 | 54 | fn deserialize(&self, mut slice: &[u8]) -> OutputResult { 55 | let mut data: Vec = vec![]; 56 | 57 | let mut buf = String::new(); 58 | 59 | slice.read_line(&mut buf)?; 60 | let names: Vec = serde_json::from_str(&buf)?; 61 | buf.clear(); 62 | 63 | slice.read_line(&mut buf)?; 64 | let types: Vec = serde_json::from_str(&buf)?; 65 | buf.clear(); 66 | 67 | for line in slice.lines() { 68 | let line = line?; 69 | let values: Vec = serde_json::from_str(&line)?; 70 | 71 | let row: T = 72 | serde_json::from_value(Value::Object(names.iter().cloned().zip(values).collect()))?; 73 | 74 | data.push(row); 75 | } 76 | 77 | Ok((data, names.into_iter().zip(types).collect())) 78 | } 79 | } 80 | 81 | #[cfg(test)] 82 | mod tests { 83 | use super::*; 84 | 85 | use std::{fs, path::PathBuf}; 86 | 87 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 88 | 89 | #[test] 90 | fn simple() -> Result<(), Box> { 91 | let file_path = PathBuf::new().join("tests/files/JSONCompactEachRowWithNamesAndTypes.txt"); 92 | let content = fs::read_to_string(&file_path)?; 93 | 94 | assert_eq!( 95 | GeneralJsonCompactEachRowWithNamesAndTypesOutput::format_name(), 96 | file_path 97 | .file_stem() 98 | .unwrap() 99 | .to_string_lossy() 100 | .parse() 101 | .unwrap() 102 | ); 103 | 104 | let (rows, info) = GeneralJsonCompactEachRowWithNamesAndTypesOutput::new() 105 | .deserialize(content.as_bytes())?; 106 | assert_eq!( 107 | rows.first().unwrap().get("tuple1").unwrap(), 108 | &Value::Array(vec![1.into(), "a".into()]) 109 | ); 110 | assert_eq!(info.get("array1"), Some(&"Array(UInt8)".to_owned())); 111 | 112 | let (rows, info) = JsonCompactEachRowWithNamesAndTypesOutput::::new() 113 | .deserialize(content.as_bytes())?; 114 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 115 | assert_eq!(info.get("array1"), Some(&"Array(UInt8)".to_owned())); 116 | 117 | Ok(()) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact_strings.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use serde::de::DeserializeOwned; 5 | 6 | use crate::format_name::FormatName; 7 | 8 | use super::{json::JsonDataInfo, json_compact::JsonCompactOutput, Output, OutputResult}; 9 | 10 | pub struct JsonCompactStringsOutput { 11 | phantom: PhantomData, 12 | } 13 | impl Default for JsonCompactStringsOutput { 14 | fn default() -> Self { 15 | Self::new() 16 | } 17 | } 18 | impl JsonCompactStringsOutput { 19 | pub fn new() -> Self { 20 | Self { 21 | phantom: PhantomData, 22 | } 23 | } 24 | } 25 | pub type GeneralJsonCompactStringsOutput = JsonCompactStringsOutput>; 26 | 27 | impl Output for JsonCompactStringsOutput 28 | where 29 | T: DeserializeOwned, 30 | { 31 | type Row = T; 32 | type Info = JsonDataInfo; 33 | 34 | type Error = serde_json::Error; 35 | 36 | fn format_name() -> FormatName { 37 | FormatName::JsonCompactStrings 38 | } 39 | 40 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 41 | JsonCompactOutput::new().deserialize_with::(slice) 42 | } 43 | } 44 | 45 | #[cfg(test)] 46 | mod tests { 47 | use super::*; 48 | 49 | use std::{fs, path::PathBuf}; 50 | 51 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 52 | 53 | #[test] 54 | fn simple() -> Result<(), Box> { 55 | let file_path = PathBuf::new().join("tests/files/JSONCompactStrings.json"); 56 | let content = fs::read_to_string(&file_path)?; 57 | 58 | assert_eq!( 59 | GeneralJsonCompactStringsOutput::format_name(), 60 | file_path 61 | .file_stem() 62 | .unwrap() 63 | .to_string_lossy() 64 | .parse() 65 | .unwrap() 66 | ); 67 | 68 | let (rows, info) = 69 | GeneralJsonCompactStringsOutput::new().deserialize(content.as_bytes())?; 70 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 71 | assert_eq!(info.rows, 2); 72 | 73 | let (rows, info) = 74 | JsonCompactStringsOutput::::new().deserialize(content.as_bytes())?; 75 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 76 | assert_eq!(info.rows, 2); 77 | 78 | Ok(()) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact_strings_each_row.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::de::DeserializeOwned; 4 | 5 | use crate::format_name::FormatName; 6 | 7 | use super::{json_compact_each_row::JsonCompactEachRowOutput, Output, OutputResult}; 8 | 9 | type Inner = JsonCompactEachRowOutput; 10 | 11 | pub struct JsonCompactStringsEachRowOutput { 12 | inner: Inner, 13 | } 14 | impl JsonCompactStringsEachRowOutput { 15 | pub fn new(names: Vec) -> Self { 16 | Self { 17 | inner: Inner::new(names), 18 | } 19 | } 20 | } 21 | 22 | pub type GeneralJsonCompactStringsEachRowOutput = 23 | JsonCompactStringsEachRowOutput>; 24 | 25 | impl Output for JsonCompactStringsEachRowOutput 26 | where 27 | T: DeserializeOwned, 28 | { 29 | type Row = as Output>::Row; 30 | type Info = as Output>::Info; 31 | 32 | type Error = as Output>::Error; 33 | 34 | fn format_name() -> crate::format_name::FormatName { 35 | FormatName::JsonCompactStringsEachRow 36 | } 37 | 38 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 39 | self.inner.deserialize(slice) 40 | } 41 | } 42 | 43 | #[cfg(test)] 44 | mod tests { 45 | use super::*; 46 | 47 | use std::{fs, path::PathBuf}; 48 | 49 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 50 | 51 | #[test] 52 | fn simple() -> Result<(), Box> { 53 | let file_path = PathBuf::new().join("tests/files/JSONCompactStringsEachRow.txt"); 54 | let content = fs::read_to_string(&file_path)?; 55 | 56 | assert_eq!( 57 | GeneralJsonCompactStringsEachRowOutput::format_name(), 58 | file_path 59 | .file_stem() 60 | .unwrap() 61 | .to_string_lossy() 62 | .parse() 63 | .unwrap() 64 | ); 65 | 66 | let (rows, _info): (_, ()) = GeneralJsonCompactStringsEachRowOutput::new(vec![ 67 | "array1".into(), 68 | "array2".into(), 69 | "tuple1".into(), 70 | "tuple2".into(), 71 | "map1".into(), 72 | ]) 73 | .deserialize(content.as_bytes())?; 74 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 75 | 76 | let (rows, _info): (_, ()) = JsonCompactStringsEachRowOutput::::new(vec![ 77 | "array1".into(), 78 | "array2".into(), 79 | "tuple1".into(), 80 | "tuple2".into(), 81 | "map1".into(), 82 | ]) 83 | .deserialize(content.as_bytes())?; 84 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 85 | 86 | Ok(()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_compact_strings_each_row_with_names_and_types.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::de::DeserializeOwned; 4 | 5 | use crate::format_name::FormatName; 6 | 7 | use super::{ 8 | json_compact_each_row_with_names_and_types::JsonCompactEachRowWithNamesAndTypesOutput, Output, 9 | OutputResult, 10 | }; 11 | 12 | type Inner = JsonCompactEachRowWithNamesAndTypesOutput; 13 | 14 | #[derive(Default)] 15 | pub struct JsonCompactStringsEachRowWithNamesAndTypesOutput { 16 | inner: Inner, 17 | } 18 | impl JsonCompactStringsEachRowWithNamesAndTypesOutput { 19 | pub fn new() -> Self { 20 | Self { 21 | inner: Inner::new(), 22 | } 23 | } 24 | } 25 | pub type GeneralJsonCompactStringsEachRowWithNamesAndTypesOutput = 26 | JsonCompactStringsEachRowWithNamesAndTypesOutput>; 27 | 28 | impl Output for JsonCompactStringsEachRowWithNamesAndTypesOutput 29 | where 30 | T: DeserializeOwned, 31 | { 32 | type Row = as Output>::Row; 33 | type Info = as Output>::Info; 34 | 35 | type Error = as Output>::Error; 36 | 37 | fn format_name() -> crate::format_name::FormatName { 38 | FormatName::JsonCompactStringsEachRowWithNamesAndTypes 39 | } 40 | 41 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 42 | self.inner.deserialize(slice) 43 | } 44 | } 45 | 46 | #[cfg(test)] 47 | mod tests { 48 | use super::*; 49 | 50 | use std::{fs, path::PathBuf}; 51 | 52 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 53 | 54 | #[test] 55 | fn simple() -> Result<(), Box> { 56 | let file_path = 57 | PathBuf::new().join("tests/files/JSONCompactStringsEachRowWithNamesAndTypes.txt"); 58 | let content = fs::read_to_string(&file_path)?; 59 | 60 | assert_eq!( 61 | GeneralJsonCompactStringsEachRowWithNamesAndTypesOutput::format_name(), 62 | file_path 63 | .file_stem() 64 | .unwrap() 65 | .to_string_lossy() 66 | .parse() 67 | .unwrap() 68 | ); 69 | 70 | let (rows, info) = GeneralJsonCompactStringsEachRowWithNamesAndTypesOutput::new() 71 | .deserialize(content.as_bytes())?; 72 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 73 | assert_eq!(info.get("array1"), Some(&"Array(UInt8)".to_owned())); 74 | 75 | let (rows, info) = 76 | JsonCompactStringsEachRowWithNamesAndTypesOutput::::new() 77 | .deserialize(content.as_bytes())?; 78 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 79 | assert_eq!(info.get("array1"), Some(&"Array(UInt8)".to_owned())); 80 | 81 | Ok(()) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_each_row.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | collections::HashMap, 4 | io::{BufRead as _, Error as IoError}, 5 | }; 6 | 7 | use serde::de::DeserializeOwned; 8 | use serde_json::Value; 9 | 10 | use crate::format_name::FormatName; 11 | 12 | use super::{Output, OutputResult}; 13 | 14 | pub struct JsonEachRowOutput { 15 | phantom: PhantomData, 16 | } 17 | impl Default for JsonEachRowOutput { 18 | fn default() -> Self { 19 | Self::new() 20 | } 21 | } 22 | impl JsonEachRowOutput { 23 | pub fn new() -> Self { 24 | Self { 25 | phantom: PhantomData, 26 | } 27 | } 28 | } 29 | 30 | pub type GeneralJsonEachRowOutput = JsonEachRowOutput>; 31 | 32 | #[derive(thiserror::Error, Debug)] 33 | pub enum JsonEachRowOutputError { 34 | #[error("IoError {0:?}")] 35 | IoError(#[from] IoError), 36 | #[error("SerdeJsonError {0:?}")] 37 | SerdeJsonError(#[from] serde_json::Error), 38 | } 39 | 40 | impl Output for JsonEachRowOutput 41 | where 42 | T: DeserializeOwned, 43 | { 44 | type Row = T; 45 | type Info = (); 46 | 47 | type Error = JsonEachRowOutputError; 48 | 49 | fn format_name() -> FormatName { 50 | FormatName::JsonEachRow 51 | } 52 | 53 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 54 | let mut data: Vec = vec![]; 55 | for line in slice.lines() { 56 | let line = line?; 57 | let row: T = serde_json::from_str(&line)?; 58 | data.push(row); 59 | } 60 | 61 | Ok((data, ())) 62 | } 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use super::*; 68 | 69 | use std::{fs, path::PathBuf}; 70 | 71 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 72 | 73 | #[test] 74 | fn simple() -> Result<(), Box> { 75 | let file_path = PathBuf::new().join("tests/files/JSONEachRow.txt"); 76 | let content = fs::read_to_string(&file_path)?; 77 | 78 | assert_eq!( 79 | GeneralJsonEachRowOutput::format_name(), 80 | file_path 81 | .file_stem() 82 | .unwrap() 83 | .to_string_lossy() 84 | .parse() 85 | .unwrap() 86 | ); 87 | 88 | let (rows, _info): (_, ()) = 89 | GeneralJsonEachRowOutput::new().deserialize(content.as_bytes())?; 90 | assert_eq!( 91 | rows.first().unwrap().get("tuple1").unwrap(), 92 | &Value::Array(vec![1.into(), "a".into()]) 93 | ); 94 | 95 | let (rows, _info): (_, ()) = 96 | JsonEachRowOutput::::new().deserialize(content.as_bytes())?; 97 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 98 | 99 | Ok(()) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_each_row_with_progress.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | collections::HashMap, 4 | io::{BufRead as _, Error as IoError}, 5 | }; 6 | 7 | use serde::{de::DeserializeOwned, Deserialize}; 8 | use serde_aux::field_attributes::deserialize_number_from_string; 9 | use serde_json::Value; 10 | 11 | use crate::format_name::FormatName; 12 | 13 | use super::{Output, OutputResult}; 14 | 15 | pub struct JsonEachRowWithProgressOutput { 16 | phantom: PhantomData, 17 | } 18 | impl Default for JsonEachRowWithProgressOutput { 19 | fn default() -> Self { 20 | Self::new() 21 | } 22 | } 23 | impl JsonEachRowWithProgressOutput { 24 | pub fn new() -> Self { 25 | Self { 26 | phantom: PhantomData, 27 | } 28 | } 29 | } 30 | 31 | pub type GeneralJsonEachRowWithProgressOutput = 32 | JsonEachRowWithProgressOutput>; 33 | 34 | #[derive(thiserror::Error, Debug)] 35 | pub enum JsonEachRowWithProgressOutputError { 36 | #[error("IoError {0:?}")] 37 | IoError(#[from] IoError), 38 | #[error("SerdeJsonError {0:?}")] 39 | SerdeJsonError(#[from] serde_json::Error), 40 | #[error("ProgressInTheWrongPosition")] 41 | ProgressInTheWrongPosition, 42 | #[error("ProgressMissing")] 43 | ProgressMissing, 44 | } 45 | 46 | impl Output for JsonEachRowWithProgressOutput 47 | where 48 | T: DeserializeOwned, 49 | { 50 | type Row = T; 51 | type Info = JsonEachRowProgress; 52 | 53 | type Error = JsonEachRowWithProgressOutputError; 54 | 55 | fn format_name() -> FormatName { 56 | FormatName::JsonEachRowWithProgress 57 | } 58 | 59 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 60 | let mut data: Vec = vec![]; 61 | let mut info = Option::::None; 62 | 63 | for line in slice.lines() { 64 | let line = line?; 65 | 66 | if info.is_some() { 67 | return Err(JsonEachRowWithProgressOutputError::ProgressInTheWrongPosition); 68 | } 69 | 70 | match serde_json::from_str::>(&line)? { 71 | JsonEachRowLine::Row { row } => { 72 | data.push(row); 73 | continue; 74 | } 75 | JsonEachRowLine::Progress { progress } => { 76 | info = Some(progress); 77 | break; 78 | } 79 | } 80 | } 81 | 82 | let info = info.ok_or(JsonEachRowWithProgressOutputError::ProgressMissing)?; 83 | 84 | Ok((data, info)) 85 | } 86 | } 87 | 88 | #[derive(Deserialize, Debug, Clone)] 89 | #[serde(untagged)] 90 | enum JsonEachRowLine 91 | where 92 | T: Sized, 93 | { 94 | Row { row: T }, 95 | Progress { progress: JsonEachRowProgress }, 96 | } 97 | 98 | #[derive(Deserialize, Debug, Clone)] 99 | pub struct JsonEachRowProgress { 100 | #[serde(deserialize_with = "deserialize_number_from_string")] 101 | pub read_rows: usize, 102 | #[serde(deserialize_with = "deserialize_number_from_string")] 103 | pub read_bytes: usize, 104 | #[serde(deserialize_with = "deserialize_number_from_string")] 105 | pub written_rows: usize, 106 | #[serde(deserialize_with = "deserialize_number_from_string")] 107 | pub written_bytes: usize, 108 | #[serde(deserialize_with = "deserialize_number_from_string")] 109 | pub total_rows_to_read: usize, 110 | } 111 | 112 | #[cfg(test)] 113 | mod tests { 114 | use super::*; 115 | 116 | use std::{fs, path::PathBuf}; 117 | 118 | use crate::test_helpers::{TestRow, TEST_ROW_1}; 119 | 120 | #[test] 121 | fn simple() -> Result<(), Box> { 122 | let file_path = PathBuf::new().join("tests/files/JSONEachRowWithProgress.txt"); 123 | let content = fs::read_to_string(&file_path)?; 124 | 125 | assert_eq!( 126 | GeneralJsonEachRowWithProgressOutput::format_name(), 127 | file_path 128 | .file_stem() 129 | .unwrap() 130 | .to_string_lossy() 131 | .parse() 132 | .unwrap() 133 | ); 134 | 135 | let (rows, info) = 136 | GeneralJsonEachRowWithProgressOutput::new().deserialize(content.as_bytes())?; 137 | assert_eq!( 138 | rows.first().unwrap().get("tuple1").unwrap(), 139 | &Value::Array(vec![1.into(), "a".into()]) 140 | ); 141 | assert_eq!(info.read_rows, 2); 142 | 143 | let (rows, info) = 144 | JsonEachRowWithProgressOutput::::new().deserialize(content.as_bytes())?; 145 | assert_eq!(rows.first().unwrap(), &*TEST_ROW_1); 146 | assert_eq!(info.read_rows, 2); 147 | 148 | Ok(()) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_strings.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::de::DeserializeOwned; 4 | 5 | use crate::format_name::FormatName; 6 | 7 | use super::{json::JsonOutput, Output, OutputResult}; 8 | 9 | type Inner = JsonOutput; 10 | 11 | #[derive(Default)] 12 | pub struct JsonStringsOutput { 13 | inner: Inner, 14 | } 15 | impl JsonStringsOutput { 16 | pub fn new() -> Self { 17 | Self { 18 | inner: Inner::new(), 19 | } 20 | } 21 | } 22 | pub type GeneralJsonStringsOutput = JsonStringsOutput>; 23 | 24 | impl Output for JsonStringsOutput 25 | where 26 | T: DeserializeOwned, 27 | { 28 | type Row = as Output>::Row; 29 | type Info = as Output>::Info; 30 | 31 | type Error = as Output>::Error; 32 | 33 | fn format_name() -> crate::format_name::FormatName { 34 | FormatName::JsonStrings 35 | } 36 | 37 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 38 | self.inner.deserialize(slice) 39 | } 40 | } 41 | 42 | #[cfg(test)] 43 | mod tests { 44 | use super::*; 45 | 46 | use std::{fs, path::PathBuf}; 47 | 48 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 49 | 50 | #[test] 51 | fn simple() -> Result<(), Box> { 52 | let file_path = PathBuf::new().join("tests/files/JSONStrings.json"); 53 | let content = fs::read_to_string(&file_path)?; 54 | 55 | assert_eq!( 56 | GeneralJsonStringsOutput::format_name(), 57 | file_path 58 | .file_stem() 59 | .unwrap() 60 | .to_string_lossy() 61 | .parse() 62 | .unwrap() 63 | ); 64 | 65 | let (rows, info) = GeneralJsonStringsOutput::new().deserialize(content.as_bytes())?; 66 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 67 | assert_eq!(info.rows, 2); 68 | 69 | let (rows, info) = 70 | JsonStringsOutput::::new().deserialize(content.as_bytes())?; 71 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 72 | assert_eq!(info.rows, 2); 73 | 74 | Ok(()) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_strings_each_row.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::de::DeserializeOwned; 4 | 5 | use crate::format_name::FormatName; 6 | 7 | use super::{json_each_row::JsonEachRowOutput, Output, OutputResult}; 8 | 9 | type Inner = JsonEachRowOutput; 10 | 11 | #[derive(Default)] 12 | pub struct JsonStringsEachRowOutput { 13 | inner: Inner, 14 | } 15 | impl JsonStringsEachRowOutput { 16 | pub fn new() -> Self { 17 | Self { 18 | inner: Inner::new(), 19 | } 20 | } 21 | } 22 | pub type GeneralJsonStringsEachRowOutput = JsonStringsEachRowOutput>; 23 | 24 | impl Output for JsonStringsEachRowOutput 25 | where 26 | T: DeserializeOwned, 27 | { 28 | type Row = as Output>::Row; 29 | type Info = as Output>::Info; 30 | 31 | type Error = as Output>::Error; 32 | 33 | fn format_name() -> crate::format_name::FormatName { 34 | FormatName::JsonStringsEachRow 35 | } 36 | 37 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 38 | self.inner.deserialize(slice) 39 | } 40 | } 41 | 42 | #[cfg(test)] 43 | mod tests { 44 | use super::*; 45 | 46 | use std::{fs, path::PathBuf}; 47 | 48 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 49 | 50 | #[test] 51 | fn simple() -> Result<(), Box> { 52 | let file_path = PathBuf::new().join("tests/files/JSONStringsEachRow.txt"); 53 | let content = fs::read_to_string(&file_path)?; 54 | 55 | assert_eq!( 56 | GeneralJsonStringsEachRowOutput::format_name(), 57 | file_path 58 | .file_stem() 59 | .unwrap() 60 | .to_string_lossy() 61 | .parse() 62 | .unwrap() 63 | ); 64 | 65 | let (rows, _info): (_, ()) = 66 | GeneralJsonStringsEachRowOutput::new().deserialize(content.as_bytes())?; 67 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 68 | 69 | let (rows, _info): (_, ()) = 70 | JsonStringsEachRowOutput::::new().deserialize(content.as_bytes())?; 71 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 72 | 73 | Ok(()) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/json_strings_each_row_with_progress.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::de::DeserializeOwned; 4 | 5 | use crate::format_name::FormatName; 6 | 7 | use super::{json_each_row_with_progress::JsonEachRowWithProgressOutput, Output, OutputResult}; 8 | 9 | type Inner = JsonEachRowWithProgressOutput; 10 | 11 | #[derive(Default)] 12 | pub struct JsonStringsEachRowWithProgressOutput { 13 | inner: Inner, 14 | } 15 | impl JsonStringsEachRowWithProgressOutput { 16 | pub fn new() -> Self { 17 | Self { 18 | inner: Inner::new(), 19 | } 20 | } 21 | } 22 | 23 | pub type GeneralJsonStringsEachRowWithProgressOutput = 24 | JsonStringsEachRowWithProgressOutput>; 25 | 26 | impl Output for JsonStringsEachRowWithProgressOutput 27 | where 28 | T: DeserializeOwned, 29 | { 30 | type Row = as Output>::Row; 31 | type Info = as Output>::Info; 32 | 33 | type Error = as Output>::Error; 34 | 35 | fn format_name() -> crate::format_name::FormatName { 36 | FormatName::JsonStringsEachRowWithProgress 37 | } 38 | 39 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 40 | self.inner.deserialize(slice) 41 | } 42 | } 43 | 44 | #[cfg(test)] 45 | mod tests { 46 | use super::*; 47 | 48 | use std::{fs, path::PathBuf}; 49 | 50 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 51 | 52 | #[test] 53 | fn simple() -> Result<(), Box> { 54 | let file_path = PathBuf::new().join("tests/files/JSONStringsEachRowWithProgress.txt"); 55 | let content = fs::read_to_string(&file_path)?; 56 | 57 | assert_eq!( 58 | GeneralJsonStringsEachRowWithProgressOutput::format_name(), 59 | file_path 60 | .file_stem() 61 | .unwrap() 62 | .to_string_lossy() 63 | .parse() 64 | .unwrap() 65 | ); 66 | 67 | let (rows, info) = 68 | GeneralJsonStringsEachRowWithProgressOutput::new().deserialize(content.as_bytes())?; 69 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 70 | assert_eq!(info.read_rows, 2); 71 | 72 | let (rows, info) = JsonStringsEachRowWithProgressOutput::::new() 73 | .deserialize(content.as_bytes())?; 74 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 75 | assert_eq!(info.read_rows, 2); 76 | 77 | Ok(()) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::format_name::FormatName; 2 | 3 | // 4 | #[cfg(feature = "with-tsv")] 5 | pub mod tsv; 6 | #[cfg(feature = "with-tsv")] 7 | pub mod tsv_raw; 8 | #[cfg(feature = "with-tsv")] 9 | pub mod tsv_with_names; 10 | #[cfg(feature = "with-tsv")] 11 | pub mod tsv_with_names_and_types; 12 | 13 | #[cfg(feature = "with-tsv")] 14 | pub use self::{ 15 | tsv::TsvOutput, tsv_raw::TsvRawOutput, tsv_with_names::TsvWithNamesOutput, 16 | tsv_with_names_and_types::TsvWithNamesAndTypesOutput, 17 | }; 18 | 19 | #[cfg(feature = "with-tsv")] 20 | pub type TabSeparatedOutput = self::tsv::TsvOutput; 21 | #[cfg(feature = "with-tsv")] 22 | pub type TabSeparatedRawOutput = self::tsv_raw::TsvRawOutput; 23 | #[cfg(feature = "with-tsv")] 24 | pub type TabSeparatedWithNamesOutput = self::tsv_with_names::TsvWithNamesOutput; 25 | #[cfg(feature = "with-tsv")] 26 | pub type TabSeparatedWithNamesAndTypesOutput = 27 | self::tsv_with_names_and_types::TsvWithNamesAndTypesOutput; 28 | 29 | // 30 | #[cfg(feature = "with-json")] 31 | pub mod json; 32 | #[cfg(feature = "with-json")] 33 | pub mod json_compact; 34 | #[cfg(feature = "with-json")] 35 | pub mod json_compact_strings; 36 | #[cfg(feature = "with-json")] 37 | pub mod json_strings; 38 | 39 | #[cfg(feature = "with-json")] 40 | pub use self::{ 41 | json::{GeneralJsonOutput, JsonOutput}, 42 | json_compact::{GeneralJsonCompactOutput, JsonCompactOutput}, 43 | json_compact_strings::{GeneralJsonCompactStringsOutput, JsonCompactStringsOutput}, 44 | json_strings::{GeneralJsonStringsOutput, JsonStringsOutput}, 45 | }; 46 | 47 | // 48 | #[cfg(feature = "with-json")] 49 | pub mod json_compact_each_row; 50 | #[cfg(feature = "with-json")] 51 | pub mod json_compact_each_row_with_names_and_types; 52 | #[cfg(feature = "with-json")] 53 | pub mod json_compact_strings_each_row; 54 | #[cfg(feature = "with-json")] 55 | pub mod json_compact_strings_each_row_with_names_and_types; 56 | #[cfg(feature = "with-json")] 57 | pub mod json_each_row; 58 | #[cfg(feature = "with-json")] 59 | pub mod json_each_row_with_progress; 60 | #[cfg(feature = "with-json")] 61 | pub mod json_strings_each_row; 62 | #[cfg(feature = "with-json")] 63 | pub mod json_strings_each_row_with_progress; 64 | 65 | #[cfg(feature = "with-json")] 66 | pub use self::{ 67 | json_compact_each_row::{GeneralJsonCompactEachRowOutput, JsonCompactEachRowOutput}, 68 | json_compact_each_row_with_names_and_types::{ 69 | GeneralJsonCompactEachRowWithNamesAndTypesOutput, JsonCompactEachRowWithNamesAndTypesOutput, 70 | }, 71 | json_compact_strings_each_row::{ 72 | GeneralJsonCompactStringsEachRowOutput, JsonCompactStringsEachRowOutput, 73 | }, 74 | json_compact_strings_each_row_with_names_and_types::{ 75 | GeneralJsonCompactStringsEachRowWithNamesAndTypesOutput, 76 | JsonCompactStringsEachRowWithNamesAndTypesOutput, 77 | }, 78 | json_each_row::{GeneralJsonEachRowOutput, JsonEachRowOutput}, 79 | json_each_row_with_progress::{ 80 | GeneralJsonEachRowWithProgressOutput, JsonEachRowWithProgressOutput, 81 | }, 82 | json_strings_each_row::{GeneralJsonStringsEachRowOutput, JsonStringsEachRowOutput}, 83 | json_strings_each_row_with_progress::{ 84 | GeneralJsonStringsEachRowWithProgressOutput, JsonStringsEachRowWithProgressOutput, 85 | }, 86 | }; 87 | 88 | pub trait Output { 89 | type Row; 90 | type Info; 91 | type Error: std::error::Error; 92 | 93 | fn format_name() -> FormatName; 94 | fn deserialize(&self, slice: &[u8]) -> OutputResult; 95 | } 96 | pub type OutputResult = Result<(Vec, Info), Error>; 97 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/tsv.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use csv::{ReaderBuilder, StringRecordsIntoIter}; 5 | use serde::de::DeserializeOwned; 6 | 7 | use crate::format_name::FormatName; 8 | 9 | use super::{tsv_raw::TsvRawOutput, Output, OutputResult}; 10 | 11 | pub struct TsvOutput { 12 | names: Option>, 13 | types: Option>, 14 | phantom: PhantomData, 15 | } 16 | impl Default for TsvOutput { 17 | fn default() -> Self { 18 | Self::new() 19 | } 20 | } 21 | impl TsvOutput { 22 | pub fn new() -> Self { 23 | Self { 24 | names: None, 25 | types: None, 26 | phantom: PhantomData, 27 | } 28 | } 29 | pub fn with_names(names: Vec) -> Self { 30 | Self { 31 | names: Some(names), 32 | types: None, 33 | phantom: PhantomData, 34 | } 35 | } 36 | pub fn with_names_and_types(names: Vec, types: Vec) -> Self { 37 | Self { 38 | names: Some(names), 39 | types: Some(types), 40 | phantom: PhantomData, 41 | } 42 | } 43 | pub(crate) fn from_raw_parts(names: Option>, types: Option>) -> Self { 44 | Self { 45 | names, 46 | types, 47 | phantom: PhantomData, 48 | } 49 | } 50 | } 51 | 52 | impl Output for TsvOutput 53 | where 54 | T: DeserializeOwned, 55 | { 56 | type Row = T; 57 | type Info = Option>; 58 | 59 | type Error = csv::Error; 60 | 61 | fn format_name() -> FormatName { 62 | FormatName::Tsv 63 | } 64 | 65 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 66 | let rdr = ReaderBuilder::new() 67 | .delimiter(b'\t') 68 | .has_headers(false) 69 | .from_reader(slice); 70 | 71 | self.deserialize_with_records(rdr.into_records()) 72 | } 73 | } 74 | impl TsvOutput 75 | where 76 | T: DeserializeOwned, 77 | { 78 | pub(crate) fn deserialize_with_records( 79 | &self, 80 | records: StringRecordsIntoIter<&[u8]>, 81 | ) -> OutputResult<::Row, ::Info, ::Error> { 82 | // TODO, unescape 83 | TsvRawOutput::from_raw_parts(self.names.to_owned(), self.types.to_owned()) 84 | .deserialize_with_records(records) 85 | } 86 | } 87 | 88 | #[cfg(test)] 89 | mod tests { 90 | use super::*; 91 | 92 | use std::{fs, path::PathBuf}; 93 | 94 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 95 | 96 | #[test] 97 | fn simple() -> Result<(), Box> { 98 | let file_path = PathBuf::new().join("tests/files/TSV.tsv"); 99 | let content = fs::read_to_string(&file_path)?; 100 | 101 | assert_eq!( 102 | TsvOutput::>::format_name(), 103 | file_path 104 | .file_stem() 105 | .unwrap() 106 | .to_string_lossy() 107 | .parse() 108 | .unwrap() 109 | ); 110 | 111 | let (rows, info) = TsvOutput::>::with_names(vec![ 112 | "array1".into(), 113 | "array2".into(), 114 | "tuple1".into(), 115 | "tuple2".into(), 116 | "map1".into(), 117 | ]) 118 | .deserialize(content.as_bytes())?; 119 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 120 | assert_eq!(info, None); 121 | 122 | let (rows, info) = TsvOutput::::new().deserialize(content.as_bytes())?; 123 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 124 | assert_eq!(info, None); 125 | 126 | Ok(()) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/tsv_raw.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use csv::{ReaderBuilder, StringRecord, StringRecordsIntoIter}; 5 | use serde::de::DeserializeOwned; 6 | 7 | use crate::format_name::FormatName; 8 | 9 | use super::{Output, OutputResult}; 10 | 11 | pub struct TsvRawOutput { 12 | names: Option>, 13 | types: Option>, 14 | phantom: PhantomData, 15 | } 16 | impl Default for TsvRawOutput { 17 | fn default() -> Self { 18 | Self::new() 19 | } 20 | } 21 | impl TsvRawOutput { 22 | pub fn new() -> Self { 23 | Self { 24 | names: None, 25 | types: None, 26 | phantom: PhantomData, 27 | } 28 | } 29 | pub fn with_names(names: Vec) -> Self { 30 | Self { 31 | names: Some(names), 32 | types: None, 33 | phantom: PhantomData, 34 | } 35 | } 36 | pub fn with_names_and_types(names: Vec, types: Vec) -> Self { 37 | Self { 38 | names: Some(names), 39 | types: Some(types), 40 | phantom: PhantomData, 41 | } 42 | } 43 | pub(crate) fn from_raw_parts(names: Option>, types: Option>) -> Self { 44 | Self { 45 | names, 46 | types, 47 | phantom: PhantomData, 48 | } 49 | } 50 | } 51 | 52 | impl Output for TsvRawOutput 53 | where 54 | T: DeserializeOwned, 55 | { 56 | type Row = T; 57 | type Info = Option>; 58 | 59 | type Error = csv::Error; 60 | 61 | fn format_name() -> FormatName { 62 | FormatName::TsvRaw 63 | } 64 | 65 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 66 | let rdr = ReaderBuilder::new() 67 | .delimiter(b'\t') 68 | .has_headers(false) 69 | .from_reader(slice); 70 | 71 | self.deserialize_with_records(rdr.into_records()) 72 | } 73 | } 74 | impl TsvRawOutput 75 | where 76 | T: DeserializeOwned, 77 | { 78 | pub(crate) fn deserialize_with_records( 79 | &self, 80 | records: StringRecordsIntoIter<&[u8]>, 81 | ) -> OutputResult<::Row, ::Info, ::Error> { 82 | let header = &self.names.to_owned().map(StringRecord::from); 83 | let mut data: Vec = vec![]; 84 | for record in records { 85 | let record = record?; 86 | let row: T = record.deserialize(header.as_ref())?; 87 | data.push(row); 88 | } 89 | 90 | let info = if let Some(types) = &self.types { 91 | self.names 92 | .to_owned() 93 | .map(|x| x.into_iter().zip(types.to_owned()).collect()) 94 | } else { 95 | None 96 | }; 97 | 98 | Ok((data, info)) 99 | } 100 | } 101 | 102 | #[cfg(test)] 103 | mod tests { 104 | use super::*; 105 | 106 | use std::{fs, path::PathBuf}; 107 | 108 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 109 | 110 | #[test] 111 | fn simple() -> Result<(), Box> { 112 | let file_path = PathBuf::new().join("tests/files/TSVRaw.tsv"); 113 | let content = fs::read_to_string(&file_path)?; 114 | 115 | assert_eq!( 116 | TsvRawOutput::>::format_name(), 117 | file_path 118 | .file_stem() 119 | .unwrap() 120 | .to_string_lossy() 121 | .parse() 122 | .unwrap() 123 | ); 124 | 125 | let (rows, info) = TsvRawOutput::>::with_names(vec![ 126 | "array1".into(), 127 | "array2".into(), 128 | "tuple1".into(), 129 | "tuple2".into(), 130 | "map1".into(), 131 | ]) 132 | .deserialize(content.as_bytes())?; 133 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 134 | assert_eq!(info, None); 135 | 136 | let (rows, info) = TsvRawOutput::::new().deserialize(content.as_bytes())?; 137 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 138 | assert_eq!(info, None); 139 | 140 | Ok(()) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/tsv_with_names.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::collections::HashMap; 3 | 4 | use csv::ReaderBuilder; 5 | use serde::de::DeserializeOwned; 6 | 7 | use crate::format_name::FormatName; 8 | 9 | use super::{tsv::TsvOutput, Output, OutputResult}; 10 | 11 | pub struct TsvWithNamesOutput { 12 | types: Option>, 13 | phantom: PhantomData, 14 | } 15 | impl Default for TsvWithNamesOutput { 16 | fn default() -> Self { 17 | Self::new() 18 | } 19 | } 20 | impl TsvWithNamesOutput { 21 | pub fn new() -> Self { 22 | Self { 23 | types: None, 24 | phantom: PhantomData, 25 | } 26 | } 27 | pub fn with_types(types: Vec) -> Self { 28 | Self { 29 | types: Some(types), 30 | phantom: PhantomData, 31 | } 32 | } 33 | } 34 | 35 | impl Output for TsvWithNamesOutput 36 | where 37 | T: DeserializeOwned, 38 | { 39 | type Row = T; 40 | type Info = Option>; 41 | 42 | type Error = csv::Error; 43 | 44 | fn format_name() -> FormatName { 45 | FormatName::TsvWithNames 46 | } 47 | 48 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 49 | let mut rdr = ReaderBuilder::new().delimiter(b'\t').from_reader(slice); 50 | 51 | let header = rdr.headers()?; 52 | let names: Vec = header.iter().map(ToOwned::to_owned).collect(); 53 | 54 | let records = rdr.into_records(); 55 | 56 | TsvOutput::from_raw_parts(Some(names), self.types.to_owned()) 57 | .deserialize_with_records(records) 58 | } 59 | } 60 | 61 | #[cfg(test)] 62 | mod tests { 63 | use super::*; 64 | 65 | use std::{fs, path::PathBuf}; 66 | 67 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 68 | 69 | #[test] 70 | fn simple() -> Result<(), Box> { 71 | let file_path = PathBuf::new().join("tests/files/TSVWithNames.tsv"); 72 | let content = fs::read_to_string(&file_path)?; 73 | 74 | assert_eq!( 75 | TsvWithNamesOutput::>::format_name(), 76 | file_path 77 | .file_stem() 78 | .unwrap() 79 | .to_string_lossy() 80 | .parse() 81 | .unwrap() 82 | ); 83 | 84 | let (rows, info) = 85 | TsvWithNamesOutput::>::new().deserialize(content.as_bytes())?; 86 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 87 | assert_eq!(info, None); 88 | 89 | let (rows, info) = 90 | TsvWithNamesOutput::::new().deserialize(content.as_bytes())?; 91 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 92 | assert_eq!(info, None); 93 | 94 | Ok(()) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /clickhouse-format/src/output/tsv_with_names_and_types.rs: -------------------------------------------------------------------------------- 1 | use core::marker::PhantomData; 2 | use std::{ 3 | collections::HashMap, 4 | io::{Error as IoError, ErrorKind as IoErrorKind}, 5 | }; 6 | 7 | use csv::ReaderBuilder; 8 | use serde::de::DeserializeOwned; 9 | 10 | use crate::format_name::FormatName; 11 | 12 | use super::{tsv::TsvOutput, Output, OutputResult}; 13 | 14 | pub struct TsvWithNamesAndTypesOutput { 15 | phantom: PhantomData, 16 | } 17 | impl Default for TsvWithNamesAndTypesOutput { 18 | fn default() -> Self { 19 | Self::new() 20 | } 21 | } 22 | impl TsvWithNamesAndTypesOutput { 23 | pub fn new() -> Self { 24 | Self { 25 | phantom: PhantomData, 26 | } 27 | } 28 | } 29 | 30 | impl Output for TsvWithNamesAndTypesOutput 31 | where 32 | T: DeserializeOwned, 33 | { 34 | type Row = T; 35 | type Info = Option>; 36 | 37 | type Error = csv::Error; 38 | 39 | fn format_name() -> FormatName { 40 | FormatName::TsvWithNamesAndTypes 41 | } 42 | 43 | fn deserialize(&self, slice: &[u8]) -> OutputResult { 44 | let mut rdr = ReaderBuilder::new().delimiter(b'\t').from_reader(slice); 45 | 46 | let header = rdr.headers()?; 47 | let names: Vec = header.iter().map(ToOwned::to_owned).collect(); 48 | 49 | let mut records = rdr.into_records(); 50 | 51 | let record = records 52 | .next() 53 | .ok_or_else(|| IoError::new(IoErrorKind::Other, ""))??; 54 | let types: Vec = record.iter().map(ToOwned::to_owned).collect(); 55 | 56 | TsvOutput::with_names_and_types(names, types).deserialize_with_records(records) 57 | } 58 | } 59 | 60 | #[cfg(test)] 61 | mod tests { 62 | use super::*; 63 | 64 | use std::{fs, path::PathBuf}; 65 | 66 | use crate::test_helpers::{TestStringsRow, TEST_STRINGS_ROW_1}; 67 | 68 | #[test] 69 | fn simple() -> Result<(), Box> { 70 | let file_path = PathBuf::new().join("tests/files/TSVWithNamesAndTypes.tsv"); 71 | let content = fs::read_to_string(&file_path)?; 72 | 73 | assert_eq!( 74 | TsvWithNamesAndTypesOutput::>::format_name(), 75 | file_path 76 | .file_stem() 77 | .unwrap() 78 | .to_string_lossy() 79 | .parse() 80 | .unwrap() 81 | ); 82 | 83 | let info_expected: HashMap = vec![ 84 | ("array1".into(), "Array(UInt8)".into()), 85 | ("array2".into(), "Array(String)".into()), 86 | ("tuple1".into(), "Tuple(UInt8, String)".into()), 87 | ("tuple2".into(), "Tuple(UInt8, Nullable(String))".into()), 88 | ("map1".into(), "Map(String,String)".into()), 89 | ] 90 | .into_iter() 91 | .collect(); 92 | 93 | let (rows, info) = TsvWithNamesAndTypesOutput::>::new() 94 | .deserialize(content.as_bytes())?; 95 | assert_eq!(rows.first().unwrap().get("tuple1").unwrap(), "(1,'a')"); 96 | assert_eq!(info, Some(info_expected.clone())); 97 | 98 | let (rows, info) = 99 | TsvWithNamesAndTypesOutput::::new().deserialize(content.as_bytes())?; 100 | assert_eq!(rows.first().unwrap(), &*TEST_STRINGS_ROW_1); 101 | assert_eq!(info, Some(info_expected)); 102 | 103 | Ok(()) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /clickhouse-format/src/test_helpers.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use once_cell::sync::Lazy; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Serialize, Deserialize, PartialEq, Debug)] 7 | pub(crate) struct TestRow { 8 | pub(crate) array1: Vec, 9 | pub(crate) array2: Vec, 10 | pub(crate) tuple1: (usize, String), 11 | pub(crate) tuple2: (usize, Option), 12 | pub(crate) map1: HashMap, 13 | } 14 | 15 | #[allow(dead_code)] 16 | pub(crate) static TEST_ROW_1: Lazy = Lazy::new(|| TestRow { 17 | array1: vec![1, 2], 18 | array2: vec!["a".into(), "b".into()], 19 | tuple1: (1, "a".into()), 20 | tuple2: (1, None), 21 | map1: vec![ 22 | ("1".into(), "Ready".into()), 23 | ("2".into(), "Steady".into()), 24 | ("3".into(), "Go".into()), 25 | ] 26 | .into_iter() 27 | .collect(), 28 | }); 29 | #[allow(dead_code)] 30 | pub(crate) static TEST_ROW_2: Lazy = Lazy::new(|| TestRow { 31 | array1: vec![3, 4], 32 | array2: vec!["c".into(), "d".into()], 33 | tuple1: (2, "b".into()), 34 | tuple2: (2, Some("b".into())), 35 | map1: vec![].into_iter().collect(), 36 | }); 37 | 38 | #[derive(Serialize, Deserialize, PartialEq, Debug)] 39 | pub(crate) struct TestStringsRow { 40 | pub(crate) array1: String, 41 | pub(crate) array2: String, 42 | pub(crate) tuple1: String, 43 | pub(crate) tuple2: String, 44 | pub(crate) map1: String, 45 | } 46 | 47 | #[allow(dead_code)] 48 | pub(crate) static TEST_STRINGS_ROW_1: Lazy = Lazy::new(|| TestStringsRow { 49 | array1: "[1,2]".into(), 50 | array2: "['a','b']".into(), 51 | tuple1: "(1,'a')".into(), 52 | tuple2: "(1,NULL)".into(), 53 | map1: "{'1':'Ready','2':'Steady','3':'Go'}".into(), 54 | }); 55 | #[allow(dead_code)] 56 | pub(crate) static TEST_STRINGS_ROW_2: Lazy = Lazy::new(|| TestStringsRow { 57 | array1: "[3,4]".into(), 58 | array2: "['c','d']".into(), 59 | tuple1: "(2,'b')".into(), 60 | tuple2: "(2,'b')".into(), 61 | map1: "{}".into(), 62 | }); 63 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSON.json: -------------------------------------------------------------------------------- 1 | { 2 | "meta": [ 3 | { 4 | "name": "array1", 5 | "type": "Array(UInt8)" 6 | }, 7 | { 8 | "name": "array2", 9 | "type": "Array(String)" 10 | }, 11 | { 12 | "name": "tuple1", 13 | "type": "Tuple(UInt8, String)" 14 | }, 15 | { 16 | "name": "tuple2", 17 | "type": "Tuple(UInt8, Nullable(String))" 18 | }, 19 | { 20 | "name": "map1", 21 | "type": "Map(String,String)" 22 | } 23 | ], 24 | "data": [ 25 | { 26 | "array1": [ 27 | 1, 28 | 2 29 | ], 30 | "array2": [ 31 | "a", 32 | "b" 33 | ], 34 | "tuple1": [ 35 | 1, 36 | "a" 37 | ], 38 | "tuple2": [ 39 | 1, 40 | null 41 | ], 42 | "map1": { 43 | "1": "Ready", 44 | "2": "Steady", 45 | "3": "Go" 46 | } 47 | }, 48 | { 49 | "array1": [ 50 | 3, 51 | 4 52 | ], 53 | "array2": [ 54 | "c", 55 | "d" 56 | ], 57 | "tuple1": [ 58 | 2, 59 | "b" 60 | ], 61 | "tuple2": [ 62 | 2, 63 | "b" 64 | ], 65 | "map1": {} 66 | } 67 | ], 68 | "rows": 2, 69 | "statistics": { 70 | "elapsed": 0.000403344, 71 | "rows_read": 2, 72 | "bytes_read": 207 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompact.json: -------------------------------------------------------------------------------- 1 | { 2 | "meta": [ 3 | { 4 | "name": "array1", 5 | "type": "Array(UInt8)" 6 | }, 7 | { 8 | "name": "array2", 9 | "type": "Array(String)" 10 | }, 11 | { 12 | "name": "tuple1", 13 | "type": "Tuple(UInt8, String)" 14 | }, 15 | { 16 | "name": "tuple2", 17 | "type": "Tuple(UInt8, Nullable(String))" 18 | }, 19 | { 20 | "name": "map1", 21 | "type": "Map(String,String)" 22 | } 23 | ], 24 | "data": [ 25 | [ 26 | [ 27 | 1, 28 | 2 29 | ], 30 | [ 31 | "a", 32 | "b" 33 | ], 34 | [ 35 | 1, 36 | "a" 37 | ], 38 | [ 39 | 1, 40 | null 41 | ], 42 | { 43 | "1": "Ready", 44 | "2": "Steady", 45 | "3": "Go" 46 | } 47 | ], 48 | [ 49 | [ 50 | 3, 51 | 4 52 | ], 53 | [ 54 | "c", 55 | "d" 56 | ], 57 | [ 58 | 2, 59 | "b" 60 | ], 61 | [ 62 | 2, 63 | "b" 64 | ], 65 | {} 66 | ] 67 | ], 68 | "rows": 2, 69 | "statistics": { 70 | "elapsed": 0.000315355, 71 | "rows_read": 2, 72 | "bytes_read": 207 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompactEachRow.txt: -------------------------------------------------------------------------------- 1 | [[1,2], ["a","b"], [1,"a"], [1,null], {"1":"Ready","2":"Steady","3":"Go"}] 2 | [[3,4], ["c","d"], [2,"b"], [2,"b"], {}] 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompactEachRowWithNamesAndTypes.txt: -------------------------------------------------------------------------------- 1 | ["array1", "array2", "tuple1", "tuple2", "map1"] 2 | ["Array(UInt8)", "Array(String)", "Tuple(UInt8, String)", "Tuple(UInt8, Nullable(String))", "Map(String,String)"] 3 | [[1,2], ["a","b"], [1,"a"], [1,null], {"1":"Ready","2":"Steady","3":"Go"}] 4 | [[3,4], ["c","d"], [2,"b"], [2,"b"], {}] 5 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompactStrings.json: -------------------------------------------------------------------------------- 1 | { 2 | "meta": [ 3 | { 4 | "name": "array1", 5 | "type": "Array(UInt8)" 6 | }, 7 | { 8 | "name": "array2", 9 | "type": "Array(String)" 10 | }, 11 | { 12 | "name": "tuple1", 13 | "type": "Tuple(UInt8, String)" 14 | }, 15 | { 16 | "name": "tuple2", 17 | "type": "Tuple(UInt8, Nullable(String))" 18 | }, 19 | { 20 | "name": "map1", 21 | "type": "Map(String,String)" 22 | } 23 | ], 24 | "data": [ 25 | [ 26 | "[1,2]", 27 | "['a','b']", 28 | "(1,'a')", 29 | "(1,NULL)", 30 | "{'1':'Ready','2':'Steady','3':'Go'}" 31 | ], 32 | [ 33 | "[3,4]", 34 | "['c','d']", 35 | "(2,'b')", 36 | "(2,'b')", 37 | "{}" 38 | ] 39 | ], 40 | "rows": 2, 41 | "statistics": { 42 | "elapsed": 0.000331107, 43 | "rows_read": 2, 44 | "bytes_read": 207 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompactStringsEachRow.txt: -------------------------------------------------------------------------------- 1 | ["[1,2]", "['a','b']", "(1,'a')", "(1,NULL)", "{'1':'Ready','2':'Steady','3':'Go'}"] 2 | ["[3,4]", "['c','d']", "(2,'b')", "(2,'b')", "{}"] 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONCompactStringsEachRowWithNamesAndTypes.txt: -------------------------------------------------------------------------------- 1 | ["array1", "array2", "tuple1", "tuple2", "map1"] 2 | ["Array(UInt8)", "Array(String)", "Tuple(UInt8, String)", "Tuple(UInt8, Nullable(String))", "Map(String,String)"] 3 | ["[1,2]", "['a','b']", "(1,'a')", "(1,NULL)", "{'1':'Ready','2':'Steady','3':'Go'}"] 4 | ["[3,4]", "['c','d']", "(2,'b')", "(2,'b')", "{}"] 5 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONEachRow.txt: -------------------------------------------------------------------------------- 1 | {"array1":[1,2],"array2":["a","b"],"tuple1":[1,"a"],"tuple2":[1,null],"map1":{"1":"Ready","2":"Steady","3":"Go"}} 2 | {"array1":[3,4],"array2":["c","d"],"tuple1":[2,"b"],"tuple2":[2,"b"],"map1":{}} 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONEachRowWithProgress.txt: -------------------------------------------------------------------------------- 1 | {"row":{"array1":[1,2],"array2":["a","b"],"tuple1":[1,"a"],"tuple2":[1,null],"map1":{"1":"Ready","2":"Steady","3":"Go"}}} 2 | {"row":{"array1":[3,4],"array2":["c","d"],"tuple1":[2,"b"],"tuple2":[2,"b"],"map1":{}}} 3 | {"progress":{"read_rows":"2","read_bytes":"207","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} 4 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONStrings.json: -------------------------------------------------------------------------------- 1 | { 2 | "meta": [ 3 | { 4 | "name": "array1", 5 | "type": "Array(UInt8)" 6 | }, 7 | { 8 | "name": "array2", 9 | "type": "Array(String)" 10 | }, 11 | { 12 | "name": "tuple1", 13 | "type": "Tuple(UInt8, String)" 14 | }, 15 | { 16 | "name": "tuple2", 17 | "type": "Tuple(UInt8, Nullable(String))" 18 | }, 19 | { 20 | "name": "map1", 21 | "type": "Map(String,String)" 22 | } 23 | ], 24 | "data": [ 25 | { 26 | "array1": "[1,2]", 27 | "array2": "['a','b']", 28 | "tuple1": "(1,'a')", 29 | "tuple2": "(1,NULL)", 30 | "map1": "{'1':'Ready','2':'Steady','3':'Go'}" 31 | }, 32 | { 33 | "array1": "[3,4]", 34 | "array2": "['c','d']", 35 | "tuple1": "(2,'b')", 36 | "tuple2": "(2,'b')", 37 | "map1": "{}" 38 | } 39 | ], 40 | "rows": 2, 41 | "statistics": { 42 | "elapsed": 0.000372392, 43 | "rows_read": 2, 44 | "bytes_read": 207 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONStringsEachRow.txt: -------------------------------------------------------------------------------- 1 | {"array1":"[1,2]","array2":"['a','b']","tuple1":"(1,'a')","tuple2":"(1,NULL)","map1":"{'1':'Ready','2':'Steady','3':'Go'}"} 2 | {"array1":"[3,4]","array2":"['c','d']","tuple1":"(2,'b')","tuple2":"(2,'b')","map1":"{}"} 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/JSONStringsEachRowWithProgress.txt: -------------------------------------------------------------------------------- 1 | {"row":{"array1":"[1,2]","array2":"['a','b']","tuple1":"(1,'a')","tuple2":"(1,NULL)","map1":"{'1':'Ready','2':'Steady','3':'Go'}"}} 2 | {"row":{"array1":"[3,4]","array2":"['c','d']","tuple1":"(2,'b')","tuple2":"(2,'b')","map1":"{}"}} 3 | {"progress":{"read_rows":"2","read_bytes":"207","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}} 4 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/TSV.tsv: -------------------------------------------------------------------------------- 1 | [1,2] ['a','b'] (1,'a') (1,NULL) {'1':'Ready','2':'Steady','3':'Go'} 2 | [3,4] ['c','d'] (2,'b') (2,'b') {} 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/TSVRaw.tsv: -------------------------------------------------------------------------------- 1 | [1,2] ['a','b'] (1,'a') (1,NULL) {'1':'Ready','2':'Steady','3':'Go'} 2 | [3,4] ['c','d'] (2,'b') (2,'b') {} 3 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/TSVWithNames.tsv: -------------------------------------------------------------------------------- 1 | array1 array2 tuple1 tuple2 map1 2 | [1,2] ['a','b'] (1,'a') (1,NULL) {'1':'Ready','2':'Steady','3':'Go'} 3 | [3,4] ['c','d'] (2,'b') (2,'b') {} 4 | -------------------------------------------------------------------------------- /clickhouse-format/tests/files/TSVWithNamesAndTypes.tsv: -------------------------------------------------------------------------------- 1 | array1 array2 tuple1 tuple2 map1 2 | Array(UInt8) Array(String) Tuple(UInt8, String) Tuple(UInt8, Nullable(String)) Map(String,String) 3 | [1,2] ['a','b'] (1,'a') (1,NULL) {'1':'Ready','2':'Steady','3':'Go'} 4 | [3,4] ['c','d'] (2,'b') (2,'b') {} 5 | -------------------------------------------------------------------------------- /clickhouse-format/tests/gen_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Prerequire 6 | # ./../clickhouse_tgz_archive/download.sh 7 | 8 | # ./tests/gen_files.sh 9 | 10 | script_path=$(cd $(dirname $0) ; pwd -P) 11 | script_path_root="${script_path}/" 12 | 13 | bin_server="${script_path_root}../../clickhouse_tgz_archive/clickhouse/usr/bin/clickhouse-server" 14 | bin_client="${script_path_root}../../clickhouse_tgz_archive/clickhouse/usr/bin/clickhouse-client" 15 | 16 | workdir=$(mktemp -d) 17 | 18 | mkdir -p "${workdir}/lib" 19 | path="${workdir}/lib/" 20 | 21 | mkdir -p "${workdir}/etc" 22 | config_file="${workdir}/etc/config.xml" 23 | tee "${config_file}" </dev/null 24 | 25 | 26 | trace 27 | true 28 | 29 | 30 | 9000 31 | 32 | ${path} 33 | 34 | 8589934592 35 | 5368709120 36 | true 37 | 38 | 39 | 40 | xxx 41 | 42 | 43 | ::/0 44 | 45 | 46 | default 47 | default 48 | 1 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | EOF 61 | 62 | mkdir -p "${workdir}/log" 63 | log_file="${workdir}/log/clickhouse-server.log" 64 | errorlog_file="${workdir}/log/clickhouse-server.err.log" 65 | 66 | mkdir -p "${workdir}/run" 67 | pid_file="${workdir}/run/clickhouse-server.pid" 68 | 69 | # https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port 70 | read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range 71 | tcp_port=$(comm -23 <(seq $LOWERPORT $UPPERPORT | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1) 72 | 73 | cleanup() { 74 | test -f "${pid_file}" && kill $(cat "${pid_file}") 75 | test -f "${errorlog_file}" && (cat "${errorlog_file}" | grep -v 'Connection reset by peer' | grep -v 'Broken pipe') 76 | rm -rf "${workdir}" 77 | } 78 | trap cleanup EXIT 79 | 80 | $(${bin_server} --config-file="${config_file}" --log-file="${log_file}" --errorlog-file="${errorlog_file}" --pid-file="${pid_file}" --daemon -- --path="${path}" --tcp_port=${tcp_port}) 81 | 82 | sleep 2 83 | 84 | files_path="${script_path_root}files" 85 | 86 | query_create_table=$(cat <<-END 87 | CREATE TABLE t_testing_format 88 | ( 89 | array1 Array(UInt8), 90 | array2 Array(String), 91 | tuple1 Tuple(UInt8, String), 92 | tuple2 Tuple(UInt8, Nullable(String)), 93 | map1 Map(String, String) 94 | ) ENGINE=Memory 95 | END 96 | ) 97 | $(echo ${query_create_table} | ${bin_client} --allow_experimental_map_type 1 --port ${tcp_port} --password xxx) 98 | 99 | query_insert=$(cat <<-END 100 | INSERT INTO t_testing_format VALUES 101 | ([1, 2], ['a', 'b'], (1, 'a'), (1, null), {'1':'Ready', '2':'Steady', '3':'Go'}), 102 | ([3, 4], ['c', 'd'], (2, 'b'), (2, 'b'), {}) 103 | END 104 | ) 105 | $(echo ${query_insert} | ${bin_client} --allow_experimental_map_type 1 --port ${tcp_port} --password xxx) 106 | 107 | query_select="SELECT array1, array2, tuple1, tuple2, map1 FROM t_testing_format" 108 | 109 | formats=("JSON" "JSONStrings" "JSONCompact" "JSONCompactStrings") 110 | for format in ${formats[*]}; do 111 | $(echo ${query_select} FORMAT ${format} | ${bin_client} --allow_experimental_map_type 1 --port ${tcp_port} --password xxx | python3 -m json.tool > "${files_path}/${format}.json") 112 | done 113 | 114 | formats=("TSV" "TSVRaw" "TSVWithNames" "TSVWithNamesAndTypes") 115 | for format in ${formats[*]}; do 116 | $(echo ${query_select} FORMAT ${format} | ${bin_client} --allow_experimental_map_type 1 --port ${tcp_port} --password xxx > "${files_path}/${format}.tsv") 117 | done 118 | 119 | # SKIP, because don't support tuple 120 | # formats=("CSV" "CSVWithNames") 121 | # for format in ${formats[*]}; do 122 | # $(echo ${query_select} FORMAT ${format} | ${bin_client} --allow_experimental_map_type 1 --format_csv_delimiter '|' --port ${tcp_port} --password xxx > "${files_path}/${format}.csv") 123 | # done 124 | 125 | formats=("JSONEachRow" "JSONStringsEachRow" "JSONCompactEachRow" "JSONCompactStringsEachRow" "JSONEachRowWithProgress" "JSONStringsEachRowWithProgress" "JSONCompactEachRowWithNamesAndTypes" "JSONCompactStringsEachRowWithNamesAndTypes") 126 | for format in ${formats[*]}; do 127 | $(echo ${query_select} FORMAT ${format} | ${bin_client} --allow_experimental_map_type 1 --port ${tcp_port} --password xxx > "${files_path}/${format}.txt") 128 | done 129 | 130 | query_drop_table="DROP TABLE t_testing_format" 131 | $(echo ${query_drop_table} | ${bin_client} --port ${tcp_port} --password xxx) 132 | 133 | sleep 1 134 | -------------------------------------------------------------------------------- /clickhouse-http-client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-http-client" 3 | version = "0.3.1" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "ClickHouse HTTP Client" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/clickhouse-http-client" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [features] 16 | default = [] 17 | 18 | with-format-all = ["with-format-tsv", "with-format-json"] 19 | 20 | with-format-tsv = ["clickhouse-format/with-tsv"] 21 | with-format-json = ["clickhouse-format/with-json"] 22 | 23 | _integration_tests = ["with-format-json"] 24 | 25 | [dependencies] 26 | clickhouse-format = { version = "0.3", default-features = false, path = "../clickhouse-format" } 27 | 28 | isahc = { version = "1", default-features = false, features = ["text-decoding"] } 29 | url = { version = "2", default-features = false } 30 | thiserror = { version = "1", default-features = false } 31 | 32 | [dev-dependencies] 33 | tokio = { version = "1", features = ["macros", "rt"] } 34 | env_logger = { version = "0.10" } 35 | 36 | serde = { version = "1", features = ["derive"] } 37 | serde_json = { version = "1" } 38 | chrono = { version = "0.4" } 39 | 40 | clickhouse-data-value = { version = "0.3", default-features = false, path = "../clickhouse-data-value" } 41 | 42 | [package.metadata.cargo-all-features] 43 | skip_feature_sets = [ 44 | ["_integration_tests"], 45 | ] 46 | -------------------------------------------------------------------------------- /clickhouse-http-client/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /clickhouse-http-client/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /clickhouse-http-client/README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-http-client 2 | 3 | * [ClickHouse Doc](https://clickhouse.tech/docs/en/interfaces/http/) 4 | * [Cargo package](https://crates.io/crates/clickhouse-http-client) 5 | -------------------------------------------------------------------------------- /clickhouse-http-client/src/client.rs: -------------------------------------------------------------------------------- 1 | use core::ops::{Deref, DerefMut}; 2 | 3 | use clickhouse_format::{format_name::FormatName, input::Input, output::Output}; 4 | use isahc::{ 5 | http::{response::Parts as ResponseParts, Method, Request, Response, StatusCode}, 6 | AsyncBody, AsyncReadResponseExt as _, HttpClient, HttpClientBuilder, 7 | }; 8 | 9 | use crate::{ 10 | client_config::{ 11 | ClientConfig, FORMAT_KEY_HEADER, FORMAT_KEY_URL_PARAMETER, QUERY_KEY_URL_PARAMETER, 12 | }, 13 | error::{ClientExecuteError, ClientInsertWithFormatError, ClientSelectWithFormatError, Error}, 14 | }; 15 | 16 | pub type Settings<'a> = Vec<(&'a str, &'a str)>; 17 | 18 | #[derive(Debug)] 19 | pub struct ClientBuilder { 20 | http_client_builder: HttpClientBuilder, 21 | client_config: ClientConfig, 22 | } 23 | impl Default for ClientBuilder { 24 | fn default() -> Self { 25 | Self::new() 26 | } 27 | } 28 | impl Deref for ClientBuilder { 29 | type Target = ClientConfig; 30 | 31 | fn deref(&self) -> &Self::Target { 32 | &self.client_config 33 | } 34 | } 35 | impl DerefMut for ClientBuilder { 36 | fn deref_mut(&mut self) -> &mut Self::Target { 37 | &mut self.client_config 38 | } 39 | } 40 | impl ClientBuilder { 41 | pub fn new() -> Self { 42 | Self { 43 | http_client_builder: HttpClientBuilder::new(), 44 | client_config: Default::default(), 45 | } 46 | } 47 | pub fn configurable(mut self, func: F) -> Self 48 | where 49 | F: FnOnce(HttpClientBuilder) -> HttpClientBuilder, 50 | { 51 | self.http_client_builder = func(self.http_client_builder); 52 | self 53 | } 54 | pub fn build(self) -> Result { 55 | Ok(Client { 56 | http_client: self.http_client_builder.build()?, 57 | client_config: self.client_config, 58 | }) 59 | } 60 | } 61 | 62 | #[derive(Debug, Clone)] 63 | pub struct Client { 64 | http_client: HttpClient, 65 | client_config: ClientConfig, 66 | } 67 | impl Deref for Client { 68 | type Target = ClientConfig; 69 | 70 | fn deref(&self) -> &Self::Target { 71 | &self.client_config 72 | } 73 | } 74 | impl DerefMut for Client { 75 | fn deref_mut(&mut self) -> &mut Self::Target { 76 | &mut self.client_config 77 | } 78 | } 79 | impl Client { 80 | pub fn new() -> Result { 81 | ClientBuilder::default().build() 82 | } 83 | 84 | pub async fn ping(&self) -> Result { 85 | let url = self.get_url(); 86 | let mut req = self.get_request(); 87 | 88 | let url = url.join("ping")?; 89 | 90 | *req.method_mut() = Method::GET; 91 | *req.uri_mut() = url.as_str().parse()?; 92 | 93 | let mut resp = self.http_client.send_async(req).await?; 94 | 95 | if resp.status() != StatusCode::OK { 96 | return Ok(false); 97 | } 98 | 99 | let resp_body_text = resp.text().await?; 100 | Ok(resp_body_text == self.get_http_server_default_response()) 101 | } 102 | 103 | // 104 | // 105 | // 106 | pub async fn execute( 107 | &self, 108 | sql: impl AsRef, 109 | settings: impl Into>>, 110 | ) -> Result<(), Error> { 111 | let resp = self.respond_execute(sql, settings, |req| req).await?; 112 | 113 | if !resp.status().is_success() { 114 | return Err(ClientExecuteError::StatusCodeMismatch(resp.status()).into()); 115 | } 116 | 117 | Ok(()) 118 | } 119 | 120 | pub async fn respond_execute( 121 | &self, 122 | sql: impl AsRef, 123 | settings: impl Into>>, 124 | mut pre_respond_fn: PreRF, 125 | ) -> Result, Error> 126 | where 127 | PreRF: FnMut(Request>) -> Request> + Send, 128 | { 129 | let mut url = self.get_url().to_owned(); 130 | let mut req = self.get_request(); 131 | 132 | if let Some(settings) = settings.into() { 133 | settings.iter().for_each(|(k, v)| { 134 | url.query_pairs_mut().append_pair(k, v); 135 | }); 136 | } 137 | 138 | *req.method_mut() = Method::POST; 139 | *req.uri_mut() = url.as_str().parse()?; 140 | 141 | let (parts, _) = req.into_parts(); 142 | let req = Request::from_parts(parts, sql.as_ref().as_bytes().to_owned()); 143 | 144 | let req = pre_respond_fn(req); 145 | 146 | let resp = self.http_client.send_async(req).await?; 147 | 148 | Ok(resp) 149 | } 150 | 151 | // 152 | // 153 | // 154 | pub async fn insert_with_format( 155 | &self, 156 | sql_prefix: impl AsRef, 157 | input: I, 158 | settings: impl Into>>, 159 | ) -> Result<(), Error> { 160 | let resp = self 161 | .respond_insert_with_format(sql_prefix, input, settings, |req| req) 162 | .await?; 163 | 164 | if !resp.status().is_success() { 165 | return Err(ClientInsertWithFormatError::StatusCodeMismatch(resp.status()).into()); 166 | } 167 | 168 | Ok(()) 169 | } 170 | 171 | pub async fn respond_insert_with_format( 172 | &self, 173 | sql_prefix: impl AsRef, 174 | input: I, 175 | settings: impl Into>>, 176 | pre_respond_fn: PreRF, 177 | ) -> Result, Error> 178 | where 179 | PreRF: FnMut(Request>) -> Request> + Send, 180 | { 181 | let format_name = I::format_name(); 182 | let format_bytes = input 183 | .serialize() 184 | .map_err(|err| ClientInsertWithFormatError::FormatSerError(err.to_string()))?; 185 | 186 | self.respond_insert_with_format_bytes( 187 | sql_prefix, 188 | format_name, 189 | format_bytes, 190 | settings, 191 | pre_respond_fn, 192 | ) 193 | .await 194 | } 195 | 196 | pub async fn insert_with_format_bytes( 197 | &self, 198 | sql_prefix: impl AsRef, 199 | format_name: FormatName, 200 | format_bytes: Vec, 201 | settings: impl Into>>, 202 | ) -> Result<(), Error> { 203 | let resp = self 204 | .respond_insert_with_format_bytes( 205 | sql_prefix, 206 | format_name, 207 | format_bytes, 208 | settings, 209 | |req| req, 210 | ) 211 | .await?; 212 | 213 | if !resp.status().is_success() { 214 | return Err(ClientInsertWithFormatError::StatusCodeMismatch(resp.status()).into()); 215 | } 216 | 217 | Ok(()) 218 | } 219 | 220 | pub async fn respond_insert_with_format_bytes( 221 | &self, 222 | sql_prefix: impl AsRef, 223 | format_name: FormatName, 224 | format_bytes: Vec, 225 | settings: impl Into>>, 226 | mut pre_respond_fn: PreRF, 227 | ) -> Result, Error> 228 | where 229 | PreRF: FnMut(Request>) -> Request> + Send, 230 | { 231 | let mut url = self.get_url().to_owned(); 232 | let mut req = self.get_request(); 233 | 234 | let mut sql = sql_prefix.as_ref().to_owned(); 235 | let sql_suffix = format!(" FORMAT {format_name}"); 236 | if !sql.ends_with(sql_suffix.as_str()) { 237 | sql.push_str(sql_suffix.as_str()); 238 | } 239 | 240 | url.query_pairs_mut() 241 | .append_pair(QUERY_KEY_URL_PARAMETER, sql.as_str()); 242 | 243 | if let Some(settings) = settings.into() { 244 | settings.iter().for_each(|(k, v)| { 245 | url.query_pairs_mut().append_pair(k, v); 246 | }); 247 | } 248 | 249 | *req.method_mut() = Method::POST; 250 | *req.uri_mut() = url.as_str().parse()?; 251 | 252 | let (parts, _) = req.into_parts(); 253 | let req = Request::from_parts(parts, format_bytes); 254 | 255 | let req = pre_respond_fn(req); 256 | 257 | let resp = self.http_client.send_async(req).await?; 258 | 259 | Ok(resp) 260 | } 261 | 262 | // 263 | // 264 | // 265 | pub async fn select_with_format( 266 | &self, 267 | sql: impl AsRef, 268 | output: O, 269 | settings: impl Into>>, 270 | ) -> Result<(Vec, O::Info), Error> { 271 | self.internal_select_with_format(sql, output, settings, |req| req) 272 | .await 273 | .map(|(_, x)| x) 274 | } 275 | 276 | pub async fn internal_select_with_format( 277 | &self, 278 | sql: impl AsRef, 279 | output: O, 280 | settings: impl Into>>, 281 | mut pre_respond_fn: PreRF, 282 | ) -> Result<(ResponseParts, (Vec, O::Info)), Error> 283 | where 284 | PreRF: FnMut(Request>) -> Request> + Send, 285 | { 286 | let mut url = self.get_url().to_owned(); 287 | let mut req = self.get_request(); 288 | 289 | url.query_pairs_mut().append_pair( 290 | FORMAT_KEY_URL_PARAMETER, 291 | O::format_name().to_string().as_str(), 292 | ); 293 | 294 | if let Some(settings) = settings.into() { 295 | settings.iter().for_each(|(k, v)| { 296 | url.query_pairs_mut().append_pair(k, v); 297 | }); 298 | } 299 | 300 | *req.method_mut() = Method::POST; 301 | *req.uri_mut() = url.as_str().parse()?; 302 | 303 | let (parts, _) = req.into_parts(); 304 | let req = Request::from_parts(parts, sql.as_ref().as_bytes().to_owned()); 305 | 306 | let req = pre_respond_fn(req); 307 | 308 | let resp = self.http_client.send_async(req).await?; 309 | 310 | if !resp.status().is_success() { 311 | return Err(ClientSelectWithFormatError::StatusCodeMismatch(resp.status()).into()); 312 | } 313 | 314 | let resp_format = resp.headers().get(FORMAT_KEY_HEADER); 315 | if resp_format.is_some() && resp_format.unwrap() != O::format_name().to_string().as_str() { 316 | return Err(ClientSelectWithFormatError::FormatMismatch( 317 | resp_format 318 | .unwrap() 319 | .to_str() 320 | .unwrap_or("Unknown") 321 | .to_string(), 322 | ) 323 | .into()); 324 | } 325 | 326 | let (parts, body) = resp.into_parts(); 327 | let (mut resp_parts, _) = Response::new(()).into_parts(); 328 | resp_parts.status = parts.status; 329 | resp_parts.version = parts.version; 330 | resp_parts.headers = parts.headers.to_owned(); 331 | let mut resp = Response::from_parts(parts, body); 332 | 333 | let mut resp_body_buf = Vec::with_capacity(4096); 334 | resp.copy_to(&mut resp_body_buf).await?; 335 | 336 | let rows_and_info = output 337 | .deserialize(&resp_body_buf[..]) 338 | .map_err(|err| ClientSelectWithFormatError::FormatDeError(err.to_string()))?; 339 | 340 | Ok((resp_parts, rows_and_info)) 341 | } 342 | } 343 | -------------------------------------------------------------------------------- /clickhouse-http-client/src/client_config.rs: -------------------------------------------------------------------------------- 1 | use isahc::http::{ 2 | header::{HeaderMap, InvalidHeaderValue}, 3 | Request, 4 | }; 5 | use url::{ParseError, Url}; 6 | 7 | pub const DATABASE_KEY_URL_PARAMETER: &str = "database"; 8 | pub const DATABASE_KEY_HEADER: &str = "X-ClickHouse-Database"; 9 | 10 | pub const USERNAME_KEY_URL_PARAMETER: &str = "user"; 11 | pub const USERNAME_KEY_HEADER: &str = "X-ClickHouse-User"; 12 | 13 | pub const PASSWORD_KEY_URL_PARAMETER: &str = "password"; 14 | pub const PASSWORD_KEY_HEADER: &str = "X-ClickHouse-Key"; 15 | 16 | pub const FORMAT_KEY_URL_PARAMETER: &str = "default_format"; 17 | pub const FORMAT_KEY_HEADER: &str = "X-ClickHouse-Format"; 18 | 19 | pub const QUERY_KEY_URL_PARAMETER: &str = "query"; 20 | 21 | pub const SUMMARY_KEY_HEADER: &str = "X-ClickHouse-Summary"; 22 | pub const QUERY_ID_KEY_HEADER: &str = "X-ClickHouse-Query-Id"; 23 | 24 | const HTTP_SERVER_DEFAULT_RESPONSE_DEFAULT: &str = "Ok.\n"; 25 | 26 | #[derive(Debug, Clone)] 27 | pub struct ClientConfig { 28 | url: Url, 29 | header_map: HeaderMap, 30 | http_server_default_response: Option, 31 | } 32 | impl Default for ClientConfig { 33 | fn default() -> Self { 34 | Self { 35 | url: "http://localhost:8123/".parse().unwrap(), 36 | header_map: HeaderMap::new(), 37 | http_server_default_response: None, 38 | } 39 | } 40 | } 41 | impl ClientConfig { 42 | pub fn set_url(&mut self, val: impl AsRef) -> Result<&mut Self, ParseError> { 43 | let mut query_pairs = self.url.query_pairs(); 44 | let database = query_pairs 45 | .find(|(k, _)| k == DATABASE_KEY_URL_PARAMETER) 46 | .map(|(_, v)| v.to_string()); 47 | let username = query_pairs 48 | .find(|(k, _)| k == USERNAME_KEY_URL_PARAMETER) 49 | .map(|(_, v)| v.to_string()); 50 | let password = query_pairs 51 | .find(|(k, _)| k == PASSWORD_KEY_URL_PARAMETER) 52 | .map(|(_, v)| v.to_string()); 53 | 54 | let mut url: Url = val.as_ref().parse()?; 55 | if let Some(ref database) = database { 56 | url.query_pairs_mut() 57 | .append_pair(DATABASE_KEY_URL_PARAMETER, database); 58 | } 59 | if let Some(ref username) = username { 60 | url.query_pairs_mut() 61 | .append_pair(USERNAME_KEY_URL_PARAMETER, username); 62 | } 63 | if let Some(ref password) = password { 64 | url.query_pairs_mut() 65 | .append_pair(PASSWORD_KEY_URL_PARAMETER, password); 66 | } 67 | 68 | self.url = url; 69 | Ok(self) 70 | } 71 | pub fn set_database_to_url_parameter(&mut self, val: impl AsRef) -> &mut Self { 72 | self.url 73 | .query_pairs_mut() 74 | .append_pair(DATABASE_KEY_URL_PARAMETER, val.as_ref()); 75 | self 76 | } 77 | pub fn set_database_to_header( 78 | &mut self, 79 | val: impl AsRef, 80 | ) -> Result<&mut Self, InvalidHeaderValue> { 81 | self.header_map 82 | .insert(DATABASE_KEY_HEADER, val.as_ref().parse()?); 83 | Ok(self) 84 | } 85 | 86 | pub fn set_username_to_url_parameter(&mut self, val: impl AsRef) -> &mut Self { 87 | self.url 88 | .query_pairs_mut() 89 | .append_pair(USERNAME_KEY_URL_PARAMETER, val.as_ref()); 90 | self 91 | } 92 | pub fn set_username_to_header( 93 | &mut self, 94 | val: impl AsRef, 95 | ) -> Result<&mut Self, InvalidHeaderValue> { 96 | self.header_map 97 | .insert(USERNAME_KEY_HEADER, val.as_ref().parse()?); 98 | Ok(self) 99 | } 100 | 101 | pub fn set_password_to_url_parameter(&mut self, val: impl AsRef) -> &mut Self { 102 | self.url 103 | .query_pairs_mut() 104 | .append_pair(PASSWORD_KEY_URL_PARAMETER, val.as_ref()); 105 | self 106 | } 107 | pub fn set_password_to_header( 108 | &mut self, 109 | val: impl AsRef, 110 | ) -> Result<&mut Self, InvalidHeaderValue> { 111 | self.header_map 112 | .insert(PASSWORD_KEY_HEADER, val.as_ref().parse()?); 113 | Ok(self) 114 | } 115 | 116 | pub fn set_http_server_default_response(&mut self, val: impl Into) -> &mut Self { 117 | self.http_server_default_response = Some(val.into()); 118 | self 119 | } 120 | } 121 | impl ClientConfig { 122 | pub(crate) fn get_url(&self) -> &Url { 123 | &self.url 124 | } 125 | pub(crate) fn get_request(&self) -> Request<()> { 126 | let mut req = Request::new(()); 127 | 128 | *req.headers_mut() = self.header_map.to_owned(); 129 | 130 | req 131 | } 132 | pub(crate) fn get_http_server_default_response(&self) -> &str { 133 | self.http_server_default_response 134 | .as_deref() 135 | .unwrap_or(HTTP_SERVER_DEFAULT_RESPONSE_DEFAULT) 136 | } 137 | } 138 | 139 | #[cfg(test)] 140 | mod tests { 141 | use super::*; 142 | 143 | #[test] 144 | fn with_default() -> Result<(), Box> { 145 | let config = ClientConfig::default(); 146 | assert_eq!(config.url.as_str(), "http://localhost:8123/"); 147 | assert_eq!(config.header_map.len(), 0); 148 | 149 | Ok(()) 150 | } 151 | 152 | #[test] 153 | fn with_set_url() -> Result<(), Box> { 154 | let mut config = ClientConfig::default(); 155 | config.set_url("http://127.0.0.1:8123/foo/?bar=1")?; 156 | assert_eq!(config.url.as_str(), "http://127.0.0.1:8123/foo/?bar=1"); 157 | assert_eq!(config.header_map.len(), 0); 158 | 159 | Ok(()) 160 | } 161 | 162 | #[test] 163 | fn with_set_database() -> Result<(), Box> { 164 | let mut config = ClientConfig::default(); 165 | config 166 | .set_url("http://127.0.0.1:8123/foo/?bar=1")? 167 | .set_database_to_url_parameter("db"); 168 | assert_eq!( 169 | config.url.as_str(), 170 | "http://127.0.0.1:8123/foo/?bar=1&database=db" 171 | ); 172 | assert_eq!(config.header_map.len(), 0); 173 | 174 | let mut config = ClientConfig::default(); 175 | config.set_database_to_header("db")?; 176 | assert_eq!(config.url.as_str(), "http://localhost:8123/"); 177 | assert_eq!(config.header_map.len(), 1); 178 | assert_eq!( 179 | config.header_map.get("X-ClickHouse-Database").unwrap(), 180 | "db" 181 | ); 182 | 183 | Ok(()) 184 | } 185 | 186 | #[test] 187 | fn with_set_username() -> Result<(), Box> { 188 | let mut config = ClientConfig::default(); 189 | config 190 | .set_url("http://127.0.0.1:8123/foo/?bar=1")? 191 | .set_username_to_url_parameter("user"); 192 | assert_eq!( 193 | config.url.as_str(), 194 | "http://127.0.0.1:8123/foo/?bar=1&user=user" 195 | ); 196 | assert_eq!(config.header_map.len(), 0); 197 | 198 | let mut config = ClientConfig::default(); 199 | config.set_username_to_header("user")?; 200 | assert_eq!(config.url.as_str(), "http://localhost:8123/"); 201 | assert_eq!(config.header_map.len(), 1); 202 | assert_eq!(config.header_map.get("X-ClickHouse-User").unwrap(), "user"); 203 | 204 | Ok(()) 205 | } 206 | 207 | #[test] 208 | fn with_set_password() -> Result<(), Box> { 209 | let mut config = ClientConfig::default(); 210 | config 211 | .set_url("http://127.0.0.1:8123/foo/?bar=1")? 212 | .set_password_to_url_parameter("password"); 213 | assert_eq!( 214 | config.url.as_str(), 215 | "http://127.0.0.1:8123/foo/?bar=1&password=password" 216 | ); 217 | assert_eq!(config.header_map.len(), 0); 218 | 219 | let mut config = ClientConfig::default(); 220 | config.set_password_to_header("password")?; 221 | assert_eq!(config.url.as_str(), "http://localhost:8123/"); 222 | assert_eq!(config.header_map.len(), 1); 223 | assert_eq!( 224 | config.header_map.get("X-ClickHouse-Key").unwrap(), 225 | "password" 226 | ); 227 | 228 | Ok(()) 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /clickhouse-http-client/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error as IoError; 2 | 3 | use isahc::http; 4 | 5 | #[derive(thiserror::Error, Debug)] 6 | pub enum Error { 7 | #[error("IsahcError {0:?}")] 8 | IsahcError(#[from] isahc::Error), 9 | #[error("HttpInvalidUri {0:?}")] 10 | HttpInvalidUri(#[from] http::uri::InvalidUri), 11 | #[error("UrlParseError {0:?}")] 12 | UrlParseError(#[from] url::ParseError), 13 | #[error("IoError {0:?}")] 14 | IoError(#[from] IoError), 15 | // 16 | #[error("ClientExecuteError {0:?}")] 17 | ClientExecuteError(#[from] ClientExecuteError), 18 | #[error("ClientInsertWithFormatError {0:?}")] 19 | ClientInsertWithFormatError(#[from] ClientInsertWithFormatError), 20 | #[error("ClientSelectWithFormatError {0:?}")] 21 | ClientSelectWithFormatError(#[from] ClientSelectWithFormatError), 22 | } 23 | 24 | #[derive(thiserror::Error, Debug)] 25 | pub enum ClientExecuteError { 26 | #[error("StatusCodeMismatch {0:?}")] 27 | StatusCodeMismatch(http::StatusCode), 28 | } 29 | 30 | #[derive(thiserror::Error, Debug)] 31 | pub enum ClientInsertWithFormatError { 32 | #[error("FormatSerError {0:?}")] 33 | FormatSerError(String), 34 | #[error("StatusCodeMismatch {0:?}")] 35 | StatusCodeMismatch(http::StatusCode), 36 | } 37 | 38 | #[derive(thiserror::Error, Debug)] 39 | pub enum ClientSelectWithFormatError { 40 | #[error("StatusCodeMismatch {0:?}")] 41 | StatusCodeMismatch(http::StatusCode), 42 | #[error("FormatMismatch {0:?}")] 43 | FormatMismatch(String), 44 | #[error("FormatDeError {0:?}")] 45 | FormatDeError(String), 46 | } 47 | -------------------------------------------------------------------------------- /clickhouse-http-client/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use clickhouse_format; 2 | pub use isahc; 3 | 4 | pub mod client; 5 | pub mod client_config; 6 | pub mod error; 7 | 8 | pub use self::client::{Client, ClientBuilder}; 9 | pub use self::error::Error; 10 | -------------------------------------------------------------------------------- /clickhouse-http-client/tests/integration_tests.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "_integration_tests")] 2 | #[path = "integration_tests"] 3 | mod integration_tests { 4 | mod helpers; 5 | 6 | #[cfg(test)] 7 | mod ping; 8 | 9 | #[cfg(test)] 10 | mod curd; 11 | } 12 | -------------------------------------------------------------------------------- /clickhouse-http-client/tests/integration_tests/curd.rs: -------------------------------------------------------------------------------- 1 | use chrono::{NaiveDateTime, Utc}; 2 | use clickhouse_http_client::clickhouse_format::{ 3 | input::JsonCompactEachRowInput, output::JsonCompactEachRowWithNamesAndTypesOutput, 4 | }; 5 | use serde::Deserialize; 6 | use serde_json::Value; 7 | 8 | use super::helpers::*; 9 | 10 | #[derive(Deserialize, Debug)] 11 | pub struct Event { 12 | #[serde(rename = "event_id")] 13 | pub id: u32, 14 | #[serde(deserialize_with = "clickhouse_data_value::datetime::deserialize")] 15 | pub created_at: NaiveDateTime, 16 | } 17 | 18 | #[tokio::test] 19 | async fn simple() -> Result<(), Box> { 20 | init_logger(); 21 | 22 | let client = get_client()?; 23 | 24 | client 25 | .execute( 26 | r#" 27 | CREATE TABLE t_testing_events 28 | ( 29 | event_id UInt32, 30 | created_at Datetime('UTC') 31 | ) ENGINE=Memory 32 | "#, 33 | None, 34 | ) 35 | .await?; 36 | 37 | let rows: Vec> = vec![ 38 | vec![1.into(), Utc::now().timestamp().into()], 39 | vec![2.into(), Utc::now().timestamp().into()], 40 | ]; 41 | client 42 | .insert_with_format( 43 | "INSERT INTO t_testing_events (event_id, created_at)", 44 | JsonCompactEachRowInput::new(rows), 45 | None, 46 | ) 47 | .await?; 48 | 49 | let (events, info) = client 50 | .select_with_format( 51 | "SELECT * FROM t_testing_events", 52 | JsonCompactEachRowWithNamesAndTypesOutput::::new(), 53 | None, 54 | ) 55 | .await?; 56 | println!("{events:?}"); 57 | println!("{info:?}"); 58 | assert_eq!(events.len(), 2); 59 | let event = events.first().unwrap(); 60 | assert_eq!(event.id, 1); 61 | 62 | let (events, info) = client 63 | .select_with_format( 64 | "SELECT * FROM t_testing_events", 65 | JsonCompactEachRowWithNamesAndTypesOutput::::new(), 66 | vec![("date_time_output_format", "iso")], 67 | ) 68 | .await?; 69 | println!("{events:?}"); 70 | println!("{info:?}"); 71 | 72 | client.execute("DROP TABLE t_testing_events", None).await?; 73 | 74 | Ok(()) 75 | } 76 | -------------------------------------------------------------------------------- /clickhouse-http-client/tests/integration_tests/helpers.rs: -------------------------------------------------------------------------------- 1 | use core::time::Duration; 2 | use std::env; 3 | 4 | use clickhouse_http_client::{isahc::config::Configurable as _, Client, ClientBuilder}; 5 | 6 | pub(super) fn get_client() -> Result> { 7 | let mut client = get_anonymous_client()?; 8 | client.set_username_to_header("default")?; 9 | client.set_password_to_header("xxx")?; 10 | 11 | Ok(client) 12 | } 13 | 14 | pub(super) fn get_anonymous_client() -> Result> { 15 | let mut client_builder = ClientBuilder::new() 16 | .configurable(|http_client_builder| http_client_builder.timeout(Duration::from_secs(1))); 17 | if let Ok(http_url) = env::var("CLICKHOUSE_HTTP_URL") { 18 | client_builder.set_url(http_url).unwrap(); 19 | } 20 | 21 | Ok(client_builder.build()?) 22 | } 23 | 24 | pub(super) fn init_logger() { 25 | let _ = env_logger::builder().is_test(true).try_init(); 26 | } 27 | -------------------------------------------------------------------------------- /clickhouse-http-client/tests/integration_tests/ping.rs: -------------------------------------------------------------------------------- 1 | use super::helpers::*; 2 | 3 | #[tokio::test] 4 | async fn simple() -> Result<(), Box> { 5 | init_logger(); 6 | 7 | let client = get_client()?; 8 | 9 | client.ping().await?; 10 | 11 | Ok(()) 12 | } 13 | 14 | #[tokio::test] 15 | async fn with_anonymous() -> Result<(), Box> { 16 | init_logger(); 17 | 18 | let client = get_anonymous_client()?; 19 | 20 | client.ping().await?; 21 | 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /clickhouse-http-client/tests/run_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Prerequire 6 | # ./../clickhouse_tgz_archive/download.sh 7 | 8 | # CLICKHOUSE_SERVER_BIN=/bin/clickhouse-server ./tests/run_integration_tests.sh 9 | # RUST_BACKTRACE=1 RUST_LOG=trace ./tests/run_integration_tests.sh 10 | 11 | script_path=$(cd $(dirname $0) ; pwd -P) 12 | script_path_root="${script_path}/" 13 | 14 | bin_default="${script_path_root}../../clickhouse_tgz_archive/clickhouse/bin/clickhouse-server" 15 | bin="${CLICKHOUSE_SERVER_BIN:-${bin_default}}" 16 | 17 | workdir=$(mktemp -d) 18 | 19 | mkdir -p "${workdir}/lib" 20 | path="${workdir}/lib/" 21 | 22 | mkdir -p "${workdir}/etc" 23 | config_file="${workdir}/etc/config.xml" 24 | tee "${config_file}" </dev/null 25 | 26 | 27 | trace 28 | true 29 | 30 | 31 | 8123 32 | 33 | ${path} 34 | 35 | 8589934592 36 | 5368709120 37 | true 38 | 39 | 40 | 41 | xxx 42 | 43 | 44 | ::/0 45 | 46 | 47 | default 48 | default 49 | 1 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | EOF 62 | 63 | mkdir -p "${workdir}/log" 64 | log_file="${workdir}/log/clickhouse-server.log" 65 | errorlog_file="${workdir}/log/clickhouse-server.err.log" 66 | 67 | mkdir -p "${workdir}/run" 68 | pid_file="${workdir}/run/clickhouse-server.pid" 69 | 70 | # https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port 71 | read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range 72 | http_port=$(comm -23 <(seq $LOWERPORT $UPPERPORT | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1) 73 | 74 | cleanup() { 75 | test -f "${pid_file}" && kill $(cat "${pid_file}") 76 | test -f "${errorlog_file}" && (cat "${errorlog_file}" | grep -v 'Connection reset by peer' | grep -v 'Broken pipe') 77 | rm -rf "${workdir}" 78 | } 79 | trap cleanup EXIT 80 | 81 | $(${bin} --config-file="${config_file}" --log-file="${log_file}" --errorlog-file="${errorlog_file}" --pid-file="${pid_file}" --daemon -- --path="${path}" --http_port=${http_port}) 82 | 83 | sleep 2 84 | 85 | export CLICKHOUSE_HTTP_URL="http://127.0.0.1:${http_port}" 86 | 87 | cargo test -p clickhouse-http-client --features _integration_tests -- --nocapture 88 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-postgres-client" 3 | version = "0.2.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "ClickHouse Postgres Client" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/clickhouse-postgres-client" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [package.metadata.docs.rs] 16 | features = ["all-types", "runtime-tokio-native-tls", "num-bigint"] 17 | 18 | [features] 19 | default = [] 20 | 21 | # types 22 | all-types = ["chrono", "bigdecimal", "uuid"] 23 | chrono = ["sqlx-clickhouse-ext/chrono", "clickhouse-data-value"] 24 | bigdecimal = ["sqlx-clickhouse-ext/bigdecimal"] 25 | uuid = ["sqlx-clickhouse-ext/uuid"] 26 | 27 | # runtimes 28 | runtime-actix-native-tls = ["sqlx-clickhouse-ext/runtime-actix-native-tls"] 29 | runtime-async-std-native-tls = ["sqlx-clickhouse-ext/runtime-async-std-native-tls"] 30 | runtime-tokio-native-tls = ["sqlx-clickhouse-ext/runtime-tokio-native-tls"] 31 | 32 | runtime-actix-rustls = ["sqlx-clickhouse-ext/runtime-actix-rustls"] 33 | runtime-async-std-rustls = ["sqlx-clickhouse-ext/runtime-async-std-rustls"] 34 | runtime-tokio-rustls = ["sqlx-clickhouse-ext/runtime-tokio-rustls"] 35 | 36 | # 37 | _integration_tests = ["all-types", "runtime-tokio-native-tls", "num-bigint"] 38 | 39 | [dependencies] 40 | sqlx-clickhouse-ext = { version = "0.2", default-features = false, features = ["postgres"], path = "../sqlx-clickhouse-ext" } 41 | url = { version = "2", default-features = false } 42 | 43 | num-bigint = { version = "0.4", default-features = false, features = ["std"], optional = true } 44 | 45 | clickhouse-data-value = { version = "0.3", default-features = false, optional = true, path = "../clickhouse-data-value"} 46 | 47 | [dev-dependencies] 48 | tokio = { version = "1", features = ["macros", "rt"]} 49 | chrono04 = { version = "0.4", package = "chrono" } 50 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /clickhouse-postgres-client/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /clickhouse-postgres-client/README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-postgres-client 2 | 3 | * [Cargo package](https://crates.io/crates/clickhouse-postgres-client) 4 | 5 | ## Publish 6 | 7 | ``` 8 | cargo publish --features runtime-tokio-native-tls --dry-run 9 | ``` 10 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/src/connect_options.rs: -------------------------------------------------------------------------------- 1 | use core::{ 2 | ops::{Deref, DerefMut}, 3 | str::FromStr, 4 | }; 5 | use std::env::{set_var, var, VarError}; 6 | 7 | use sqlx_clickhouse_ext::sqlx_core::{error::Error, postgres::PgConnectOptions}; 8 | use url::Url; 9 | 10 | #[derive(Debug, Clone)] 11 | pub struct ClickhousePgConnectOptions { 12 | pub(crate) inner: PgConnectOptions, 13 | } 14 | impl ClickhousePgConnectOptions { 15 | pub fn new() -> Self { 16 | update_env(); 17 | 18 | Self { 19 | inner: PgConnectOptions::new(), 20 | } 21 | } 22 | 23 | pub fn into_inner(self) -> PgConnectOptions { 24 | self.inner 25 | } 26 | } 27 | 28 | impl Default for ClickhousePgConnectOptions { 29 | fn default() -> Self { 30 | Self::new() 31 | } 32 | } 33 | 34 | impl Deref for ClickhousePgConnectOptions { 35 | type Target = PgConnectOptions; 36 | 37 | fn deref(&self) -> &Self::Target { 38 | &self.inner 39 | } 40 | } 41 | impl DerefMut for ClickhousePgConnectOptions { 42 | fn deref_mut(&mut self) -> &mut Self::Target { 43 | &mut self.inner 44 | } 45 | } 46 | 47 | impl FromStr for ClickhousePgConnectOptions { 48 | type Err = Error; 49 | 50 | fn from_str(s: &str) -> Result { 51 | update_env(); 52 | 53 | let s = update_url(s)?; 54 | 55 | PgConnectOptions::from_str(&s).map(|inner| Self { inner }) 56 | } 57 | } 58 | 59 | // 60 | const PORT_DEFAULT_STR: &str = "9005"; 61 | const SSL_MODE_PREFER: &str = "prefer"; 62 | const SSL_MODE_DISABLE: &str = "disable"; 63 | 64 | fn update_env() { 65 | if let Err(VarError::NotPresent) = var("PGPORT") { 66 | set_var("PGPORT", PORT_DEFAULT_STR) 67 | } 68 | 69 | match var("PGSSLMODE") { 70 | Ok(str) if str == SSL_MODE_PREFER => set_var("PGSSLMODE", SSL_MODE_DISABLE), 71 | Err(VarError::NotPresent) => set_var("PGSSLMODE", SSL_MODE_DISABLE), 72 | _ => (), 73 | } 74 | } 75 | 76 | fn update_url(s: &str) -> Result { 77 | let mut url: Url = s 78 | .parse() 79 | .map_err(|err: url::ParseError| Error::Configuration(err.into()))?; 80 | 81 | url.query_pairs() 82 | .map(|(k, v)| (k.to_string(), v.to_string())) 83 | .collect::>() 84 | .into_iter() 85 | .fold(url.query_pairs_mut().clear(), |ser, (key, value)| { 86 | match key.as_ref() { 87 | "sslmode" => { 88 | if value == SSL_MODE_PREFER { 89 | ser.append_pair(&key, SSL_MODE_DISABLE); 90 | } else { 91 | ser.append_pair(&key, &value); 92 | } 93 | } 94 | "ssl-mode" => { 95 | if value == SSL_MODE_PREFER { 96 | ser.append_pair(&key, SSL_MODE_DISABLE); 97 | } else { 98 | ser.append_pair(&key, &value); 99 | } 100 | } 101 | _ => { 102 | ser.append_pair(&key, &value); 103 | } 104 | }; 105 | ser 106 | }); 107 | 108 | Ok(url.to_string()) 109 | } 110 | 111 | #[cfg(test)] 112 | mod tests { 113 | use super::*; 114 | 115 | use std::env::remove_var; 116 | 117 | #[test] 118 | fn test_update_env() { 119 | remove_var("PGPORT"); 120 | remove_var("PGSSLMODE"); 121 | update_env(); 122 | assert_eq!(var("PGPORT").unwrap(), "9005"); 123 | assert_eq!(var("PGSSLMODE").unwrap(), "disable"); 124 | 125 | remove_var("PGPORT"); 126 | remove_var("PGSSLMODE"); 127 | set_var("PGSSLMODE", "prefer"); 128 | update_env(); 129 | assert_eq!(var("PGPORT").unwrap(), "9005"); 130 | assert_eq!(var("PGSSLMODE").unwrap(), "disable"); 131 | } 132 | 133 | #[test] 134 | fn test_update_url() { 135 | let uri = "postgres:///?sslmode=prefer"; 136 | assert_eq!(update_url(uri).unwrap(), "postgres:///?sslmode=disable"); 137 | 138 | let uri = "postgres:///?ssl-mode=prefer"; 139 | assert_eq!(update_url(uri).unwrap(), "postgres:///?ssl-mode=disable"); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/src/executor.rs: -------------------------------------------------------------------------------- 1 | pub use sqlx_clickhouse_ext::executor::ExecutorExt as ClickhouseExecutor; 2 | 3 | use crate::{ 4 | row::ClickhousePgRow, ClickhousePgConnection, ClickhousePgPool, ClickhousePgPoolConnection, 5 | }; 6 | 7 | impl<'c, 'q, 'async_trait> ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow> 8 | for &'c ClickhousePgPool 9 | where 10 | 'c: 'async_trait, 11 | 'q: 'async_trait, 12 | { 13 | } 14 | 15 | impl<'c, 'q, 'async_trait> ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow> 16 | for &'c mut ClickhousePgPoolConnection 17 | where 18 | 'c: 'async_trait, 19 | 'q: 'async_trait, 20 | { 21 | } 22 | 23 | impl<'c, 'q, 'async_trait> ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow> 24 | for &'c mut ClickhousePgConnection 25 | where 26 | 'c: 'async_trait, 27 | 'q: 'async_trait, 28 | { 29 | } 30 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use sqlx_clickhouse_ext; 2 | 3 | pub mod connect_options; 4 | pub mod executor; 5 | pub mod row; 6 | pub mod type_info; 7 | pub use self::connect_options::ClickhousePgConnectOptions; 8 | pub use self::executor::ClickhouseExecutor; 9 | pub use self::row::ClickhousePgRow; 10 | pub use self::type_info::ClickhousePgValue; 11 | 12 | pub type SqlxError = sqlx_clickhouse_ext::sqlx_core::error::Error; 13 | pub type ClickhousePgPool = sqlx_clickhouse_ext::sqlx_core::postgres::PgPool; 14 | pub type ClickhousePgPoolOptions = sqlx_clickhouse_ext::sqlx_core::postgres::PgPoolOptions; 15 | pub type ClickhousePgPoolConnection = sqlx_clickhouse_ext::sqlx_core::pool::PoolConnection< 16 | sqlx_clickhouse_ext::sqlx_core::postgres::Postgres, 17 | >; 18 | pub type ClickhousePgConnection = sqlx_clickhouse_ext::sqlx_core::postgres::PgConnection; 19 | 20 | // 21 | use sqlx_clickhouse_ext::sqlx_core::{ 22 | connection::ConnectOptions as _, database::Database as SqlxDatabase, 23 | }; 24 | 25 | pub async fn connect(url: &str) -> Result { 26 | connect_with(&url.parse()?).await 27 | } 28 | 29 | pub async fn connect_with( 30 | options: &ClickhousePgConnectOptions, 31 | ) -> Result { 32 | options.inner.connect().await 33 | } 34 | 35 | pub async fn execute<'c, 'q, 'async_trait, E>(sql: &'q str, executor: E) -> Result<(), SqlxError> 36 | where 37 | E: ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow>, 38 | 'c: 'async_trait, 39 | 'q: 'async_trait, 40 | ClickhousePgRow: From<::Row>, 41 | { 42 | ClickhouseExecutor::execute(executor, sql).await 43 | } 44 | 45 | pub async fn fetch_all<'c, 'q, 'async_trait, E>( 46 | sql: &'q str, 47 | executor: E, 48 | ) -> Result, SqlxError> 49 | where 50 | E: ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow>, 51 | 'c: 'async_trait, 52 | 'q: 'async_trait, 53 | ClickhousePgRow: From<::Row>, 54 | { 55 | ClickhouseExecutor::fetch_all(executor, sql).await 56 | } 57 | 58 | pub async fn fetch_one<'c, 'q, 'async_trait, E>( 59 | sql: &'q str, 60 | executor: E, 61 | ) -> Result 62 | where 63 | E: ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow>, 64 | 'c: 'async_trait, 65 | 'q: 'async_trait, 66 | ClickhousePgRow: From<::Row>, 67 | { 68 | ClickhouseExecutor::fetch_one(executor, sql).await 69 | } 70 | 71 | pub async fn fetch_optional<'c, 'q, 'async_trait, E>( 72 | sql: &'q str, 73 | executor: E, 74 | ) -> Result, SqlxError> 75 | where 76 | E: ClickhouseExecutor<'c, 'q, 'async_trait, ClickhousePgRow>, 77 | 'c: 'async_trait, 78 | 'q: 'async_trait, 79 | ClickhousePgRow: From<::Row>, 80 | { 81 | ClickhouseExecutor::fetch_optional(executor, sql).await 82 | } 83 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/src/row.rs: -------------------------------------------------------------------------------- 1 | use sqlx_clickhouse_ext::{ 2 | sqlx_core::{ 3 | column::Column as _, column::ColumnIndex, error::Error, postgres::PgRow, row::Row as _, 4 | type_info::TypeInfo as _, value::ValueRef as _, 5 | }, 6 | value::ValueRefTryGet as _, 7 | }; 8 | 9 | use crate::type_info::{ClickhousePgType, ClickhousePgValue}; 10 | 11 | pub struct ClickhousePgRow { 12 | inner: PgRow, 13 | } 14 | impl From for ClickhousePgRow { 15 | fn from(row: PgRow) -> Self { 16 | Self { inner: row } 17 | } 18 | } 19 | impl ClickhousePgRow { 20 | pub fn try_get_data(&self) -> Result, Error> { 21 | let mut array = vec![]; 22 | for (i, column) in self.inner.columns().iter().enumerate() { 23 | array.push((column.name(), self.try_get_value(i)?)); 24 | } 25 | Ok(array) 26 | } 27 | 28 | pub fn try_get_value(&self, index: I) -> Result 29 | where 30 | I: ColumnIndex, 31 | { 32 | let index = index.index(&self.inner)?; 33 | 34 | let value = self.inner.try_get_raw(index)?; 35 | 36 | let cpt = ClickhousePgType::try_from((value.type_info().name(), index))?; 37 | 38 | match cpt { 39 | ClickhousePgType::Char => (value, index).try_get().map(ClickhousePgValue::Char), 40 | ClickhousePgType::Int2 => { 41 | // Fix v23 42 | if let Ok(bytes) = value.as_bytes() { 43 | match bytes { 44 | b"true" => { 45 | return Ok(ClickhousePgValue::Bool(true)); 46 | } 47 | b"false" => { 48 | return Ok(ClickhousePgValue::Bool(false)); 49 | } 50 | _ => {} 51 | } 52 | } 53 | (value, index).try_get().map(ClickhousePgValue::I16) 54 | } 55 | ClickhousePgType::Int4 => (value, index).try_get().map(ClickhousePgValue::I32), 56 | ClickhousePgType::Int8 => (value, index).try_get().map(ClickhousePgValue::I64), 57 | ClickhousePgType::Float4 => (value, index).try_get().map(ClickhousePgValue::F32), 58 | ClickhousePgType::Float8 => (value, index).try_get().map(ClickhousePgValue::F64), 59 | ClickhousePgType::Varchar => (value, index).try_get().map(ClickhousePgValue::String), 60 | #[cfg(feature = "chrono")] 61 | ClickhousePgType::Date => (value, index).try_get().map(ClickhousePgValue::NaiveDate), 62 | #[cfg(feature = "bigdecimal")] 63 | ClickhousePgType::Numeric => { 64 | (value, index).try_get().map(ClickhousePgValue::BigDecimal) 65 | } 66 | #[cfg(feature = "uuid")] 67 | ClickhousePgType::Uuid => (value, index).try_get().map(ClickhousePgValue::Uuid), 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/tests/integration_tests.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "_integration_tests")] 2 | #[path = "integration_tests"] 3 | mod integration_tests { 4 | mod helpers; 5 | 6 | #[cfg(test)] 7 | mod data_types; 8 | } 9 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/tests/integration_tests/helpers.rs: -------------------------------------------------------------------------------- 1 | use std::{env, fs, path::PathBuf}; 2 | 3 | use clickhouse_postgres_client::{ClickhousePgConnection, ClickhousePgValue}; 4 | 5 | pub(super) async fn get_conn( 6 | init_sqls: &[&str], 7 | ) -> Result> { 8 | let mut conn = clickhouse_postgres_client::connect( 9 | env::var("CLICKHOUSE_DATABASE_URL") 10 | .unwrap_or_else(|_| "postgres://default:xxx@127.0.0.1:9005".to_string()) 11 | .as_str(), 12 | ) 13 | .await?; 14 | 15 | for sql in init_sqls { 16 | clickhouse_postgres_client::execute(sql, &mut conn).await?; 17 | } 18 | 19 | Ok(conn) 20 | } 21 | 22 | pub(super) async fn execute( 23 | sql: impl AsRef, 24 | conn: &mut ClickhousePgConnection, 25 | ) -> Result<(), Box> { 26 | clickhouse_postgres_client::execute(sql.as_ref(), conn).await?; 27 | 28 | Ok(()) 29 | } 30 | 31 | pub(super) async fn fetch_one_and_get_data( 32 | sql: impl AsRef, 33 | conn: &mut ClickhousePgConnection, 34 | ) -> Result, Box> { 35 | let row = clickhouse_postgres_client::fetch_one(sql.as_ref(), conn).await?; 36 | 37 | let data = row 38 | .try_get_data()? 39 | .into_iter() 40 | .map(|(name, value)| (name.to_string(), value)) 41 | .collect(); 42 | 43 | Ok(data) 44 | } 45 | 46 | pub(super) fn get_sql(path: &str) -> String { 47 | fs::read_to_string(PathBuf::new().join(format!("../clickhouse_sqls/data_types/{path}.sql"))) 48 | .unwrap() 49 | } 50 | 51 | pub(super) fn get_setting_sql(path: &str) -> String { 52 | fs::read_to_string(PathBuf::new().join(format!("../clickhouse_sqls/settings/{path}.sql"))) 53 | .unwrap() 54 | } 55 | -------------------------------------------------------------------------------- /clickhouse-postgres-client/tests/run_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Prerequire 6 | # ./../clickhouse_tgz_archive/download.sh 7 | 8 | # CLICKHOUSE_SERVER_BIN=/bin/clickhouse-server ./tests/run_integration_tests.sh 9 | # RUST_BACKTRACE=1 ./tests/run_integration_tests.sh 10 | 11 | script_path=$(cd $(dirname $0) ; pwd -P) 12 | script_path_root="${script_path}/" 13 | 14 | bin_default="${script_path_root}../../clickhouse_tgz_archive/clickhouse/bin/clickhouse-server" 15 | bin="${CLICKHOUSE_SERVER_BIN:-${bin_default}}" 16 | 17 | workdir=$(mktemp -d) 18 | 19 | mkdir -p "${workdir}/lib" 20 | path="${workdir}/lib/" 21 | 22 | mkdir -p "${workdir}/etc" 23 | config_file="${workdir}/etc/config.xml" 24 | tee "${config_file}" </dev/null 25 | 26 | 27 | trace 28 | true 29 | 30 | 31 | 9005 32 | 33 | ${path} 34 | 35 | 8589934592 36 | 5368709120 37 | true 38 | 39 | 40 | 41 | xxx 42 | 43 | 44 | ::/0 45 | 46 | 47 | default 48 | default 49 | 1 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | EOF 62 | 63 | mkdir -p "${workdir}/log" 64 | log_file="${workdir}/log/clickhouse-server.log" 65 | errorlog_file="${workdir}/log/clickhouse-server.err.log" 66 | 67 | mkdir -p "${workdir}/run" 68 | pid_file="${workdir}/run/clickhouse-server.pid" 69 | 70 | # https://unix.stackexchange.com/questions/55913/whats-the-easiest-way-to-find-an-unused-local-port 71 | read LOWERPORT UPPERPORT < /proc/sys/net/ipv4/ip_local_port_range 72 | postgresql_port=$(comm -23 <(seq $LOWERPORT $UPPERPORT | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | shuf | head -n 1) 73 | 74 | cleanup() { 75 | test -f "${pid_file}" && kill $(cat "${pid_file}") 76 | test -f "${errorlog_file}" && (cat "${errorlog_file}" | grep -v 'Connection reset by peer' | grep -v 'Broken pipe') 77 | rm -rf "${workdir}" 78 | } 79 | trap cleanup EXIT 80 | 81 | $(${bin} --config-file="${config_file}" --log-file="${log_file}" --errorlog-file="${errorlog_file}" --pid-file="${pid_file}" --daemon -- --path="${path}" --postgresql_port=${postgresql_port}) 82 | 83 | sleep 2 84 | 85 | export CLICKHOUSE_DATABASE_URL="postgres://default:xxx@127.0.0.1:${postgresql_port}" 86 | 87 | cargo test -p clickhouse-postgres-client --features _integration_tests -- --nocapture 88 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/array/select_with_Nullable.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | array(1, 2, NULL) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/array/select_with_String.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | array('a', 'b') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/array/select_with_UInt8.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | array(1, 2) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/boolean/create_table.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE t_boolean 2 | ( 3 | true_val Boolean, 4 | false_val Boolean 5 | ) ENGINE = Memory 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/boolean/drop_table.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE t_boolean 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/boolean/insert.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO t_boolean VALUES (1, 0) 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/boolean/select.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | true_val, 3 | toTypeName(true_val) AS true_ty, 4 | false_val, 5 | toTypeName(false_val) AS false_ty 6 | FROM t_boolean 7 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/date/select.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDate('2021-03-01') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/datetime/select_with_UTC.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDateTime('2021-03-01 01:02:03', 'UTC') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/datetime64/select_with_micro_and_UTC.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDateTime64('2021-03-01 01:02:03.123456789', 6, 'UTC') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/datetime64/select_with_milli_and_UTC.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDateTime64('2021-03-01 01:02:03.123456789', 3, 'UTC') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/datetime64/select_with_nano_and_UTC.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDateTime64('2021-03-01 01:02:03.123456789', 9, 'UTC') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/decimal/select_Decimal128.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDecimal128(toString(-1.111), 5) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/decimal/select_Decimal256.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDecimal256(toString(-1.111), 5) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/decimal/select_Decimal32.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDecimal32(toString(-1.111), 5) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/decimal/select_Decimal64.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toDecimal64(toString(-1.111), 5) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/domains_ipv4/select_f_IPv4NumToString.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | IPv4NumToString(toIPv4('127.0.0.1')) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/domains_ipv4/select_f_isIPv4String.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | isIPv4String('127.0.0.1') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/domains_ipv4/select_f_toIPv4.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toIPv4('127.0.0.1') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/domains_ipv4/select_f_toUInt32.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt32(toIPv4('127.0.0.1')) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/domains_ipv6/select_f_hex.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | hex(toIPv6('2a02:aa08:e000:3100::2')) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/enum/select_Enum16.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST('a', 'Enum16(\'a\'=-32768, \'b\'=32767)') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/enum/select_Enum8.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST('a', 'Enum(\'a\'=-128, \'b\'=127)') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/enum/select_cast_Enum16.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST(CAST('a', 'Enum16(\'a\'=1, \'b\'=2)'), 'Int16') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/enum/select_cast_Enum8.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST(CAST('a', 'Enum(\'a\'=1, \'b\'=2)'), 'Int8') AS a_val, 3 | toTypeName(a_val) AS a_ty, 4 | CAST(CAST('b', 'Enum(\'a\'=1, \'b\'=2)'), 'Int16') AS b_val, 5 | toTypeName(b_val) AS b_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/fixedstring/select.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toFixedString('foo', 8) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/float/select_-Inf.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | -0.5 / 0 AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/float/select_Float32.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toFloat32(1 - 0.9) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/float/select_Float64.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | 1 - 0.9 AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/float/select_Inf.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | 0.5 / 0 AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/float/select_NaN.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | 0 / 0 AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/geo/create_table_Point.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE t_geo_point 2 | ( 3 | p Point 4 | ) ENGINE = Memory 5 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/geo/drop_table_Point.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE t_geo_point 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/geo/insert_Point.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO t_geo_point VALUES ((10, 10)) 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/geo/select_Point.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | p AS val, 3 | toTypeName(val) AS ty 4 | FROM t_geo_point 5 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int128.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt128('-170141183460469231731687303715884105728') AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt128('170141183460469231731687303715884105727') AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int16.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt16(-32768) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt16(32767) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int256.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt256('-57896044618658097711785492504343953926634992332820282019728792003956564819968') AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt256('57896044618658097711785492504343953926634992332820282019728792003956564819967') AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int32.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt32(-2147483648) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt32(2147483647) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int64.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt64(-9223372036854775808) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt64(9223372036854775807) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int8.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt8(-128) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toInt8(127) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_Int8_0-9.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toInt8(0) AS a_val, 3 | toTypeName(a_val) AS a_ty, 4 | toInt8(9) AS b_val, 5 | toTypeName(b_val) AS b_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_UInt16.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt16(0) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toUInt16(65535) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_UInt256.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt256(0) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toUInt256('115792089237316195423570985008687907853269984665640564039457584007913129639935') AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_UInt32.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt32(0) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toUInt32(4294967295) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_UInt64.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt64(0) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toUInt64(18446744073709551615) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/int-uint/select_UInt8.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUInt8(0) AS min_val, 3 | toTypeName(min_val) AS min_ty, 4 | toUInt8(255) AS max_val, 5 | toTypeName(max_val) AS max_ty 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/map/select_with_UInt8_and_String.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/nullable/create_table.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE t_null 2 | ( 3 | x Int16, 4 | y Nullable(Int16) 5 | ) ENGINE = Memory 6 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/nullable/drop_table.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE t_null 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/nullable/insert.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO t_null VALUES (1, NULL) 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/nullable/select_f_ifNull.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | x AS x_val, 3 | toTypeName(x_val) AS x_ty, 4 | ifNull(y, -1) AS y_val, 5 | toTypeName(y_val) AS y_ty 6 | FROM t_null 7 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/nullable/select_f_toNullable.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toNullable(10) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/string/select_f_toString_date.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toString(toDate('2021-03-01')) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/tuple/select_with_UInt8_and_Nullable.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | tuple(1, NULL) AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/tuple/select_with_UInt8_and_String.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | tuple(1, 'a') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/data_types/uuid/select.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS val, 3 | toTypeName(val) AS ty 4 | -------------------------------------------------------------------------------- /clickhouse_sqls/settings/enable_allow_experimental_geo_types.sql: -------------------------------------------------------------------------------- 1 | SET allow_experimental_geo_types = 1 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/settings/enable_allow_experimental_map_type.sql: -------------------------------------------------------------------------------- 1 | SET allow_experimental_map_type = 1 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/settings/set_date_time_output_format_to_iso.sql: -------------------------------------------------------------------------------- 1 | SET date_time_output_format = 'iso' 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/settings/set_date_time_output_format_to_simple.sql: -------------------------------------------------------------------------------- 1 | SET date_time_output_format = 'simple' 2 | -------------------------------------------------------------------------------- /clickhouse_sqls/settings/set_date_time_output_format_to_unix_timestamp.sql: -------------------------------------------------------------------------------- 1 | SET date_time_output_format = 'unix_timestamp' 2 | -------------------------------------------------------------------------------- /clickhouse_tgz_archive/.gitignore: -------------------------------------------------------------------------------- 1 | /*.tgz 2 | /clickhouse 3 | -------------------------------------------------------------------------------- /clickhouse_tgz_archive/README.md: -------------------------------------------------------------------------------- 1 | ## ClickHouse server 2 | 3 | Ref https://clickhouse.tech/docs/en/getting-started/install/#from-tgz-archives 4 | -------------------------------------------------------------------------------- /clickhouse_tgz_archive/download.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # ./download.sh v23.1.3.5-stable 6 | 7 | tag="${1:-v23.1.3.5-stable}" 8 | version=`echo $tag | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'` 9 | 10 | script_path=$(cd $(dirname $0) ; pwd -P) 11 | script_path_root="${script_path}/" 12 | 13 | rm -rf "${script_path_root}clickhouse-common-static-${version}-amd64.tgz" 14 | rm -rf "${script_path_root}clickhouse-server-${version}-amd64.tgz" 15 | rm -rf "${script_path_root}clickhouse-client-${version}-amd64.tgz" 16 | wget -O "${script_path_root}clickhouse-common-static-${version}-amd64.tgz" "https://github.com/ClickHouse/ClickHouse/releases/download/${tag}/clickhouse-common-static-${version}-amd64.tgz" 17 | wget -O "${script_path_root}clickhouse-server-${version}-amd64.tgz" "https://github.com/ClickHouse/ClickHouse/releases/download/${tag}/clickhouse-server-${version}-amd64.tgz" 18 | wget -O "${script_path_root}clickhouse-client-${version}-amd64.tgz" "https://github.com/ClickHouse/ClickHouse/releases/download/${tag}/clickhouse-client-${version}-amd64.tgz" 19 | 20 | tgz_root_path="${script_path_root}clickhouse" 21 | rm -rf "${tgz_root_path}" 22 | mkdir "${tgz_root_path}" 23 | tar -zxvf "${script_path_root}clickhouse-common-static-${version}-amd64.tgz" -C "${tgz_root_path}" --strip-components=2 24 | tar -zxvf "${script_path_root}clickhouse-server-${version}-amd64.tgz" -C "${tgz_root_path}" --strip-components=2 25 | tar -zxvf "${script_path_root}clickhouse-client-${version}-amd64.tgz" -C "${tgz_root_path}" --strip-components=2 26 | -------------------------------------------------------------------------------- /demos/http_client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-demo-http-client" 3 | version = "0.1.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | clickhouse-http-client = { features = ["with-format-json"], path = "../../clickhouse-http-client" } 9 | 10 | serde = { version = "1", features = ["derive"] } 11 | 12 | futures-executor = { version = "0.3" } 13 | -------------------------------------------------------------------------------- /demos/http_client/src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | cargo run -p clickhouse-demo-http-client -- "http://127.0.0.1:8123" default xxx system 3 | */ 4 | 5 | use core::time::Duration; 6 | use std::env; 7 | 8 | use clickhouse_http_client::{ 9 | clickhouse_format::output::JsonCompactEachRowWithNamesAndTypesOutput, 10 | isahc::config::Configurable, ClientBuilder, 11 | }; 12 | use futures_executor::block_on; 13 | use serde::Deserialize; 14 | 15 | fn main() -> Result<(), Box> { 16 | block_on(run()) 17 | } 18 | 19 | async fn run() -> Result<(), Box> { 20 | let url = env::args().nth(1); 21 | let username = env::args().nth(2); 22 | let password = env::args().nth(3); 23 | let database = env::args().nth(4); 24 | 25 | let mut client_builder = ClientBuilder::new() 26 | .configurable(|http_client_builder| http_client_builder.timeout(Duration::from_secs(1))); 27 | if let Some(url) = url { 28 | client_builder.set_url(url)?; 29 | } 30 | if let Some(username) = username { 31 | client_builder.set_username_to_header(username)?; 32 | } 33 | if let Some(password) = password { 34 | client_builder.set_password_to_header(password)?; 35 | } 36 | if let Some(database) = database { 37 | client_builder.set_database_to_header(database)?; 38 | } 39 | let client = client_builder.build()?; 40 | 41 | let (databases, _) = client 42 | .select_with_format( 43 | "show databases", 44 | JsonCompactEachRowWithNamesAndTypesOutput::::new(), 45 | None, 46 | ) 47 | .await?; 48 | 49 | println!("databases: {:?}", databases); 50 | 51 | Ok(()) 52 | } 53 | 54 | #[derive(Deserialize, Debug)] 55 | pub struct Database { 56 | pub name: String, 57 | } 58 | -------------------------------------------------------------------------------- /demos/postgres_client/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "clickhouse-demo-postgres-client" 3 | version = "0.1.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | 7 | [[bin]] 8 | name = "clickhouse-demo-postgres-client-conn" 9 | path = "src/conn.rs" 10 | 11 | [[bin]] 12 | name = "clickhouse-demo-postgres-client-pool" 13 | path = "src/pool.rs" 14 | 15 | [dependencies] 16 | clickhouse-postgres-client = { default-features = false, features = ["all-types", "runtime-tokio-native-tls"], path = "../../clickhouse-postgres-client" } 17 | 18 | tokio = { version = "1", default-features = false, features = ["macros"] } 19 | -------------------------------------------------------------------------------- /demos/postgres_client/src/conn.rs: -------------------------------------------------------------------------------- 1 | /* 2 | cargo run -p clickhouse-demo-postgres-client --bin clickhouse-demo-postgres-client-conn postgres://default:xxx@127.0.0.1:9005 3 | */ 4 | 5 | use std::env; 6 | 7 | use clickhouse_postgres_client::{connect, execute, fetch_all}; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | run().await 12 | } 13 | 14 | async fn run() -> Result<(), Box> { 15 | let database_url = env::args().nth(1).unwrap_or_else(|| { 16 | env::var("DATABASE_URL") 17 | .unwrap_or_else(|_| "postgres://default:xxx@127.0.0.1:9005".to_owned()) 18 | }); 19 | 20 | let mut conn = connect(&database_url).await?; 21 | 22 | execute("use default", &mut conn).await?; 23 | 24 | let rows = fetch_all("show databases", &mut conn).await?; 25 | for row in rows.iter() { 26 | println!("data: {:?}", row.try_get_data()); 27 | } 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /demos/postgres_client/src/pool.rs: -------------------------------------------------------------------------------- 1 | /* 2 | cargo run -p clickhouse-demo-postgres-client --bin clickhouse-demo-postgres-client-pool postgres://default:xxx@127.0.0.1:9005 3 | */ 4 | 5 | use std::env; 6 | 7 | use clickhouse_postgres_client::{ 8 | execute, fetch_all, ClickhousePgConnectOptions, ClickhousePgPoolOptions, 9 | }; 10 | 11 | #[tokio::main] 12 | async fn main() -> Result<(), Box> { 13 | run().await 14 | } 15 | 16 | async fn run() -> Result<(), Box> { 17 | let database_url = env::args().nth(1).unwrap_or_else(|| { 18 | env::var("DATABASE_URL") 19 | .unwrap_or_else(|_| "postgres://default:xxx@127.0.0.1:9005".to_owned()) 20 | }); 21 | 22 | let pool = ClickhousePgPoolOptions::new() 23 | .max_connections(1) 24 | .connect_lazy_with( 25 | database_url 26 | .parse::()? 27 | .into_inner(), 28 | ); 29 | 30 | let mut pool_conn = pool.acquire().await?; 31 | 32 | execute("use default", &mut pool_conn).await?; 33 | 34 | let rows = fetch_all("show databases", &mut pool_conn).await?; 35 | for row in rows.iter() { 36 | println!("data: {:?}", row.try_get_data()); 37 | } 38 | 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sqlx-clickhouse-ext" 3 | version = "0.2.0" 4 | authors = ["vkill "] 5 | edition = "2021" 6 | description = "SQLx ClickHouse Extension" 7 | license = "Apache-2.0 OR MIT" 8 | repository = "https://github.com/bk-rs/clickhouse-rs" 9 | homepage = "https://github.com/bk-rs/clickhouse-rs" 10 | documentation = "https://docs.rs/sqlx-clickhouse-ext" 11 | keywords = [] 12 | categories = [] 13 | readme = "README.md" 14 | 15 | [package.metadata.docs.rs] 16 | features = ["postgres", "all-types", "runtime-tokio-native-tls"] 17 | 18 | [features] 19 | default = [] 20 | 21 | # databases 22 | postgres = ["sqlx-core/postgres"] 23 | 24 | # types 25 | all-types = ["chrono", "bigdecimal", "uuid"] 26 | chrono = ["sqlx-core/chrono"] 27 | bigdecimal = ["sqlx-core/bigdecimal"] 28 | uuid = ["sqlx-core/uuid"] 29 | 30 | # runtimes 31 | runtime-actix-native-tls = ["sqlx-core/runtime-actix-native-tls"] 32 | runtime-async-std-native-tls = ["sqlx-core/runtime-async-std-native-tls"] 33 | runtime-tokio-native-tls = ["sqlx-core/runtime-tokio-native-tls"] 34 | 35 | runtime-actix-rustls = ["sqlx-core/runtime-actix-rustls"] 36 | runtime-async-std-rustls = ["sqlx-core/runtime-async-std-rustls"] 37 | runtime-tokio-rustls = ["sqlx-core/runtime-tokio-rustls"] 38 | 39 | [dependencies] 40 | sqlx-core = { version = "0.6", default-features = false } 41 | futures-core = { version = "0.3", default-features = false } 42 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | ../LICENSE-APACHE -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | ../LICENSE-MIT -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/README.md: -------------------------------------------------------------------------------- 1 | # sqlx-clickhouse-ext 2 | 3 | * [Cargo package](https://crates.io/crates/sqlx-clickhouse-ext) 4 | 5 | ## Publish 6 | 7 | ``` 8 | cargo publish --features runtime-tokio-native-tls --dry-run 9 | ``` 10 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/src/error.rs: -------------------------------------------------------------------------------- 1 | use core::any::type_name; 2 | 3 | use sqlx_core::{database::Database, error::BoxDynError, type_info::TypeInfo as _, types::Type}; 4 | 5 | // https://github.com/launchbadge/sqlx/blob/v0.5.1/sqlx-core/src/error.rs#L136 6 | pub(crate) fn mismatched_types>(ty: &DB::TypeInfo) -> BoxDynError { 7 | // TODO: `#name` only produces `TINYINT` but perhaps we want to show `TINYINT(1)` 8 | format!( 9 | "mismatched types; Rust type `{}` (as SQL type `{}`) is not compatible with SQL type `{}`", 10 | type_name::(), 11 | T::type_info().name(), 12 | ty.name() 13 | ) 14 | .into() 15 | } 16 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/src/executor.rs: -------------------------------------------------------------------------------- 1 | use futures_core::future::BoxFuture; 2 | use sqlx_core::{database::Database, error::Error, executor::Executor}; 3 | 4 | pub trait ExecutorExt<'c, 'q, 'async_trait, T>: Executor<'c> 5 | where 6 | 'c: 'async_trait, 7 | 'q: 'async_trait, 8 | T: 'async_trait, 9 | T: From<::Row>, 10 | Self: Sync + 'async_trait, 11 | { 12 | fn execute(self, sql: &'q str) -> BoxFuture<'async_trait, Result<(), Error>> { 13 | Box::pin(async move { Executor::<'c>::execute(self, sql).await.map(|_| ()) }) 14 | } 15 | 16 | fn fetch_all(self, sql: &'q str) -> BoxFuture<'async_trait, Result, Error>> { 17 | Box::pin(async move { 18 | Executor::<'c>::fetch_all(self, sql) 19 | .await 20 | .map(|rows| rows.into_iter().map(|row| row.into()).collect()) 21 | }) 22 | } 23 | 24 | fn fetch_one(self, sql: &'q str) -> BoxFuture<'async_trait, Result> { 25 | Box::pin(async move { 26 | Executor::<'c>::fetch_one(self, sql) 27 | .await 28 | .map(|row| row.into()) 29 | }) 30 | } 31 | 32 | fn fetch_optional(self, sql: &'q str) -> BoxFuture<'async_trait, Result, Error>> { 33 | Box::pin(async move { 34 | Executor::<'c>::fetch_optional(self, sql) 35 | .await 36 | .map(|option_row| option_row.map(|row| row.into())) 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use sqlx_core; 2 | 3 | mod error; 4 | pub mod executor; 5 | pub mod value; 6 | -------------------------------------------------------------------------------- /sqlx-clickhouse-ext/src/value.rs: -------------------------------------------------------------------------------- 1 | pub use sqlx_core::{ 2 | database::{Database, HasValueRef}, 3 | decode::Decode, 4 | error::Error, 5 | type_info::TypeInfo as _, 6 | types::Type, 7 | value::ValueRef as _, 8 | }; 9 | 10 | use crate::error::mismatched_types; 11 | 12 | pub trait ValueRefTryGet<'r, DB, T> 13 | where 14 | DB: Database, 15 | T: Decode<'r, DB> + Type, 16 | { 17 | fn try_get(self) -> Result; 18 | } 19 | 20 | impl<'r, DB, T> ValueRefTryGet<'r, DB, T> for (>::ValueRef, usize) 21 | where 22 | DB: Database, 23 | T: Decode<'r, DB> + Type, 24 | { 25 | fn try_get(self) -> Result { 26 | let (value, index) = self; 27 | 28 | // https://github.com/launchbadge/sqlx/blob/v0.5.1/sqlx-core/src/row.rs#L111 29 | if !value.is_null() { 30 | let ty = value.type_info(); 31 | 32 | if !ty.is_null() && !T::compatible(&ty) { 33 | return Err(Error::ColumnDecode { 34 | index: format!("{index:?}"), 35 | source: mismatched_types::(&ty), 36 | }); 37 | } 38 | } 39 | 40 | T::decode(value).map_err(|source| Error::ColumnDecode { 41 | index: format!("{index:?}"), 42 | source, 43 | }) 44 | } 45 | } 46 | --------------------------------------------------------------------------------