├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── .github └── workflows │ └── ci.yml ├── README.md ├── src ├── error.rs ├── decimal.rs ├── lib.rs ├── de.rs └── ser.rs ├── benches └── serde.rs └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | .vscode/ 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [0.2.0] - 2023-05-16 11 | 12 | ### Changed 13 | 14 | - `Decimal::NaN` is now encoded as larger than `+Infinity`. 15 | 16 | ## [0.1.1] - 2023-04-12 17 | 18 | ### Added 19 | 20 | - Add serialize and deserialize for `i128` and `u128`. 21 | 22 | ## [0.1.0] - 2022-11-29 23 | 24 | - Initial release. 25 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "memcomparable" 3 | version = "0.2.0" 4 | edition = "2021" 5 | description = "A memcomparable serialization format." 6 | homepage = "https://github.com/risingwavelabs/memcomparable" 7 | repository = "https://github.com/risingwavelabs/memcomparable" 8 | keywords = ["memcomparable", "serde"] 9 | license = "Apache-2.0" 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [features] 13 | decimal = ["rust_decimal"] 14 | 15 | [dependencies] 16 | bytes = "1" 17 | rust_decimal = { version = "1", optional = true } 18 | serde = "1" 19 | thiserror = "1" 20 | 21 | [dev-dependencies] 22 | criterion = "0.4" 23 | rand = "0.8" 24 | rust_decimal = { version = "1", features = ["rand"] } 25 | serde = { version = "1", features = ["derive"] } 26 | 27 | [[bench]] 28 | name = "serde" 29 | harness = false 30 | 31 | [package.metadata.docs.rs] 32 | all-features = true 33 | rustdoc-args = ["--cfg", "docsrs"] 34 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | env: 12 | CARGO_TERM_COLOR: always 13 | 14 | jobs: 15 | fmt: 16 | runs-on: ubuntu-22.04 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: actions-rs/toolchain@v1 20 | with: 21 | profile: minimal 22 | toolchain: stable 23 | components: rustfmt, clippy 24 | - name: Check code format 25 | uses: actions-rs/cargo@v1 26 | with: 27 | command: fmt 28 | args: --all -- --check 29 | - name: Clippy 30 | uses: actions-rs/cargo@v1 31 | with: 32 | command: clippy 33 | args: --all-targets --all-features -- -D warnings 34 | 35 | test: 36 | runs-on: ubuntu-22.04 37 | steps: 38 | - uses: actions/checkout@v2 39 | - uses: actions-rs/toolchain@v1 40 | with: 41 | profile: minimal 42 | toolchain: stable 43 | - name: Build 44 | uses: actions-rs/cargo@v1 45 | with: 46 | command: build 47 | args: --all-targets --all-features 48 | - name: Test 49 | uses: actions-rs/cargo@v1 50 | with: 51 | command: test 52 | args: --all-features --no-fail-fast 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # memcomparable 2 | 3 | [![Crate](https://img.shields.io/crates/v/memcomparable.svg)](https://crates.io/crates/memcomparable) 4 | [![Docs](https://docs.rs/memcomparable/badge.svg)](https://docs.rs/memcomparable) 5 | [![CI](https://github.com/risingwavelabs/memcomparable/workflows/CI/badge.svg?branch=main)](https://github.com/risingwavelabs/memcomparable/actions) 6 | 7 | A memcomparable serialization format. 8 | 9 | The memcomparable format allows comparison of two values by using the simple memcmp function. 10 | 11 | ## Installation 12 | 13 | Add the `memcomparable` to your Cargo.toml: 14 | 15 | ```sh 16 | $ cargo add memcomparable 17 | ``` 18 | 19 | ## Usage 20 | 21 | ```rust 22 | // serialize 23 | let key1 = memcomparable::to_vec(&"hello").unwrap(); 24 | let key2 = memcomparable::to_vec(&"world").unwrap(); 25 | assert!(key1 < key2); 26 | 27 | // deserialize 28 | let v1: String = memcomparable::from_slice(&key1).unwrap(); 29 | let v2: String = memcomparable::from_slice(&key2).unwrap(); 30 | assert_eq!(v1, "hello"); 31 | assert_eq!(v2, "world"); 32 | ``` 33 | 34 | ### Optional Features 35 | 36 | - `decimal`: Enable (de)serialization for Decimal type. 37 | 38 | See [the documentation](https://docs.rs/memcomparable) for more details. 39 | 40 | ## License 41 | 42 | Apache License 2.0. Please refer to [LICENSE](LICENSE) for more information. 43 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::fmt::Display; 16 | 17 | use serde::{de, ser}; 18 | use thiserror::Error; 19 | 20 | /// The result of a serialization or deserialization operation. 21 | pub type Result = std::result::Result; 22 | 23 | /// An error that can be produced during (de)serializing. 24 | #[allow(missing_docs)] 25 | #[derive(Error, Clone, Debug, PartialEq, Eq)] 26 | pub enum Error { 27 | #[error("{0}")] 28 | Message(String), 29 | #[error("unexpected end of input")] 30 | Eof, 31 | #[error("unsupported type: {0}")] 32 | NotSupported(&'static str), 33 | #[error("invalid bool encoding: {0}")] 34 | InvalidBoolEncoding(u8), 35 | #[error("invalid char encoding: {0}")] 36 | InvalidCharEncoding(u32), 37 | #[error("invalid tag encoding: {0}")] 38 | InvalidTagEncoding(usize), 39 | #[error("invalid sequence encoding: {0}")] 40 | InvalidSeqEncoding(u8), 41 | #[error("invalid UTF8: {0}")] 42 | InvalidUtf8(#[from] std::string::FromUtf8Error), 43 | #[error("invalid bytes encoding: {0}")] 44 | InvalidBytesEncoding(u8), 45 | #[error("invalid decimal encoding: {0}")] 46 | InvalidDecimalEncoding(u8), 47 | #[error("trailing characters")] 48 | TrailingCharacters, 49 | } 50 | 51 | impl ser::Error for Error { 52 | fn custom(msg: T) -> Self { 53 | Error::Message(msg.to_string()) 54 | } 55 | } 56 | 57 | impl de::Error for Error { 58 | fn custom(msg: T) -> Self { 59 | Error::Message(msg.to_string()) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /benches/serde.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use criterion::{criterion_group, criterion_main, Criterion}; 16 | 17 | criterion_group!(benches, decimal); 18 | criterion_main!(benches); 19 | 20 | #[cfg(not(feature = "decimal"))] 21 | fn decimal(_c: &mut Criterion) {} 22 | 23 | #[cfg(feature = "decimal")] 24 | fn decimal(c: &mut Criterion) { 25 | use memcomparable::{Decimal, Deserializer, Serializer}; 26 | 27 | // generate decimals 28 | let mut decimals = vec![]; 29 | for _ in 0..10 { 30 | decimals.push(Decimal::Normalized(rand::random())); 31 | } 32 | 33 | c.bench_function("serialize_decimal", |b| { 34 | let mut i = 0; 35 | b.iter(|| { 36 | let mut ser = Serializer::new(vec![]); 37 | ser.serialize_decimal(decimals[i]).unwrap(); 38 | i += 1; 39 | if i == decimals.len() { 40 | i = 0; 41 | } 42 | }) 43 | }); 44 | 45 | c.bench_function("deserialize_decimal", |b| { 46 | let encodings = decimals 47 | .iter() 48 | .map(|d| { 49 | let mut ser = Serializer::new(vec![]); 50 | ser.serialize_decimal(*d).unwrap(); 51 | ser.into_inner() 52 | }) 53 | .collect::>(); 54 | let mut i = 0; 55 | b.iter(|| { 56 | Deserializer::new(encodings[i].as_slice()) 57 | .deserialize_decimal() 58 | .unwrap(); 59 | i += 1; 60 | if i == decimals.len() { 61 | i = 0; 62 | } 63 | }) 64 | }); 65 | } 66 | -------------------------------------------------------------------------------- /src/decimal.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::{Deserializer, Error, Serializer}; 16 | use std::fmt::Display; 17 | use std::str::FromStr; 18 | 19 | /// An extended decimal number with `NaN`, `-Inf` and `Inf`. 20 | #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 21 | #[cfg_attr(docsrs, doc(cfg(feature = "decimal")))] 22 | pub enum Decimal { 23 | /// Negative infinity. 24 | NegInf, 25 | /// Normalized value. 26 | Normalized(rust_decimal::Decimal), 27 | /// Infinity. 28 | Inf, 29 | /// Not a Number. 30 | NaN, 31 | } 32 | 33 | impl Decimal { 34 | /// A constant representing 0. 35 | pub const ZERO: Self = Decimal::Normalized(rust_decimal::Decimal::ZERO); 36 | 37 | /// Serialize the decimal into a vector. 38 | pub fn to_vec(&self) -> crate::Result> { 39 | let mut serializer = Serializer::new(vec![]); 40 | serializer.serialize_decimal(*self)?; 41 | Ok(serializer.into_inner()) 42 | } 43 | 44 | /// Deserialize a decimal value from a memcomparable bytes. 45 | pub fn from_slice(bytes: &[u8]) -> crate::Result { 46 | let mut deserializer = Deserializer::new(bytes); 47 | let t = deserializer.deserialize_decimal()?; 48 | if !deserializer.has_remaining() { 49 | Ok(t) 50 | } else { 51 | Err(Error::TrailingCharacters) 52 | } 53 | } 54 | } 55 | 56 | impl From for Decimal { 57 | fn from(decimal: rust_decimal::Decimal) -> Self { 58 | Decimal::Normalized(decimal) 59 | } 60 | } 61 | 62 | impl FromStr for Decimal { 63 | type Err = rust_decimal::Error; 64 | 65 | fn from_str(s: &str) -> Result { 66 | match s { 67 | "nan" | "NaN" => Ok(Decimal::NaN), 68 | "-inf" | "-Inf" => Ok(Decimal::NegInf), 69 | "inf" | "Inf" => Ok(Decimal::Inf), 70 | _ => Ok(Decimal::Normalized(s.parse()?)), 71 | } 72 | } 73 | } 74 | 75 | impl Display for Decimal { 76 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 77 | match self { 78 | Decimal::NaN => write!(f, "NaN"), 79 | Decimal::NegInf => write!(f, "-Inf"), 80 | Decimal::Inf => write!(f, "Inf"), 81 | Decimal::Normalized(n) => write!(f, "{}", n), 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! A memcomparable serialization format. 16 | //! 17 | //! The memcomparable format allows comparison of two values by using the simple memcmp function. 18 | //! 19 | //! # Usage 20 | //! 21 | //! ``` 22 | //! // serialize 23 | //! let key1 = memcomparable::to_vec(&"hello").unwrap(); 24 | //! let key2 = memcomparable::to_vec(&"world").unwrap(); 25 | //! assert!(key1 < key2); 26 | //! 27 | //! // deserialize 28 | //! let v1: String = memcomparable::from_slice(&key1).unwrap(); 29 | //! let v2: String = memcomparable::from_slice(&key2).unwrap(); 30 | //! assert_eq!(v1, "hello"); 31 | //! assert_eq!(v2, "world"); 32 | //! ``` 33 | //! 34 | //! # Features 35 | //! 36 | //! - `decimal`: Enable (de)serialization for [`Decimal`] type. 37 | //! - [`Serializer::serialize_decimal`] 38 | //! - [`Deserializer::deserialize_decimal`] 39 | //! 40 | //! # Format 41 | //! 42 | //! The serialization format follows [MySQL's memcomparable format](https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format). 43 | //! 44 | //! | Type | Length (bytes) | 45 | //! | --------------------------------------------- | -------------------- | 46 | //! | `bool` | 1 | 47 | //! | `char` | 4 | 48 | //! | `i8`/`i16`/`i32`/`i64`/`u8`/`u16`/`u32`/`u64` | 1/2/4/8 | 49 | //! | `f32`/`f64` | 4/8 | 50 | //! | `Decimal` | Variable | 51 | //! | `str`/`bytes` | (L + 7) / 8 x 9 | 52 | //! | `Option` | 1 + len(T) | 53 | //! | `&[T]` | (1 + len(T)) x L + 1 | 54 | //! | `(T1, T2, ..)` | sum(len(Ti)) | 55 | //! | `struct { a: T1, b: T2, .. }` | sum(len(Ti)) | 56 | //! | `enum { V1, V2, .. }` | 1 + len(Vi) | 57 | //! 58 | //! **WARN: The format is not guaranteed to be stable in minor version change, e.g. 0.1 -> 0.2.** 59 | 60 | #![deny(missing_docs)] 61 | #![cfg_attr(docsrs, feature(doc_cfg))] 62 | 63 | mod de; 64 | #[cfg(feature = "decimal")] 65 | mod decimal; 66 | mod error; 67 | mod ser; 68 | 69 | pub use de::{from_slice, Deserializer}; 70 | #[cfg(feature = "decimal")] 71 | pub use decimal::Decimal; 72 | pub use error::{Error, Result}; 73 | pub use ser::{to_vec, Serializer}; 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /src/de.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use bytes::Buf; 16 | use serde::de::{ 17 | self, DeserializeSeed, EnumAccess, IntoDeserializer, SeqAccess, VariantAccess, Visitor, 18 | }; 19 | 20 | #[cfg(feature = "decimal")] 21 | use crate::decimal::Decimal; 22 | use crate::error::{Error, Result}; 23 | 24 | const BYTES_CHUNK_SIZE: usize = 8; 25 | const BYTES_CHUNK_UNIT_SIZE: usize = BYTES_CHUNK_SIZE + 1; 26 | 27 | /// A structure that deserializes memcomparable bytes into Rust values. 28 | pub struct Deserializer { 29 | input: MaybeFlip, 30 | input_len: usize, 31 | } 32 | 33 | impl Deserializer { 34 | /// Creates a deserializer from a buffer. 35 | pub fn new(input: B) -> Self { 36 | Deserializer { 37 | input_len: input.remaining(), 38 | input: MaybeFlip { input, flip: false }, 39 | } 40 | } 41 | 42 | /// Set whether data is serialized in reverse order. 43 | /// 44 | /// If set, all bits will be flipped in serialization. 45 | pub fn set_reverse(&mut self, reverse: bool) { 46 | self.input.flip = reverse; 47 | } 48 | 49 | /// Unwrap the inner buffer from the `Deserializer`. 50 | pub fn into_inner(self) -> B { 51 | self.input.input 52 | } 53 | 54 | /// Check if the inner buffer still has remaining data. 55 | pub fn has_remaining(&self) -> bool { 56 | self.input.input.has_remaining() 57 | } 58 | 59 | /// Return the position of inner buffer from the `Deserializer`. 60 | pub fn position(&self) -> usize { 61 | self.input_len - self.input.input.remaining() 62 | } 63 | 64 | /// Advance the position of inner buffer from the `Deserializer`. 65 | pub fn advance(&mut self, cnt: usize) { 66 | self.input.input.advance(cnt) 67 | } 68 | } 69 | 70 | /// Deserialize an instance of type `T` from a memcomparable bytes. 71 | pub fn from_slice<'a, T>(bytes: &'a [u8]) -> Result 72 | where 73 | T: serde::Deserialize<'a>, 74 | { 75 | let mut deserializer = Deserializer::new(bytes); 76 | let t = T::deserialize(&mut deserializer)?; 77 | if deserializer.input.is_empty() { 78 | Ok(t) 79 | } else { 80 | Err(Error::TrailingCharacters) 81 | } 82 | } 83 | 84 | /// A wrapper around `Buf` that can flip bits when getting data. 85 | struct MaybeFlip { 86 | input: B, 87 | flip: bool, 88 | } 89 | 90 | macro_rules! def_method { 91 | ($name:ident, $ty:ty) => { 92 | fn $name(&mut self) -> $ty { 93 | let v = self.input.$name(); 94 | if self.flip { 95 | !v 96 | } else { 97 | v 98 | } 99 | } 100 | }; 101 | } 102 | 103 | impl MaybeFlip { 104 | def_method!(get_u8, u8); 105 | 106 | def_method!(get_u16, u16); 107 | 108 | def_method!(get_u32, u32); 109 | 110 | def_method!(get_u64, u64); 111 | 112 | def_method!(get_u128, u128); 113 | 114 | fn copy_to_slice(&mut self, dst: &mut [u8]) { 115 | self.input.copy_to_slice(dst); 116 | if self.flip { 117 | dst.iter_mut().for_each(|x| *x = !*x); 118 | } 119 | } 120 | 121 | fn is_empty(&self) -> bool { 122 | self.input.remaining() == 0 123 | } 124 | } 125 | 126 | impl Deserializer { 127 | fn read_bytes(&mut self) -> Result> { 128 | match self.input.get_u8() { 129 | 0 => return Ok(vec![]), // empty slice 130 | 1 => {} // non-empty slice 131 | v => return Err(Error::InvalidBytesEncoding(v)), 132 | } 133 | let mut bytes = vec![]; 134 | let mut chunk = [0u8; BYTES_CHUNK_UNIT_SIZE]; // chunk + chunk_len 135 | loop { 136 | self.input.copy_to_slice(&mut chunk); 137 | match chunk[8] { 138 | len @ 1..=8 => { 139 | bytes.extend_from_slice(&chunk[..len as usize]); 140 | return Ok(bytes); 141 | } 142 | 9 => bytes.extend_from_slice(&chunk[..8]), 143 | v => return Err(Error::InvalidBytesEncoding(v)), 144 | } 145 | } 146 | } 147 | 148 | /// Skip the next byte array. Return the length of it. 149 | pub fn skip_bytes(&mut self) -> Result { 150 | match self.input.get_u8() { 151 | 0 => return Ok(0), // empty slice 152 | 1 => {} // non-empty slice 153 | v => return Err(Error::InvalidBytesEncoding(v)), 154 | } 155 | let mut total_len = 0; 156 | loop { 157 | self.advance(BYTES_CHUNK_SIZE); 158 | match self.input.get_u8() { 159 | len @ 1..=8 => return Ok(total_len + len as usize), 160 | 9 => total_len += 8, 161 | v => return Err(Error::InvalidBytesEncoding(v)), 162 | } 163 | } 164 | } 165 | } 166 | 167 | // Format Reference: 168 | // https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format 169 | // https://haxisnake.github.io/2020/11/06/TIDB源码学习笔记-基本类型编解码方案/ 170 | impl<'de, 'a, B: Buf + 'de> de::Deserializer<'de> for &'a mut Deserializer { 171 | type Error = Error; 172 | 173 | fn deserialize_any(self, _visitor: V) -> Result 174 | where 175 | V: Visitor<'de>, 176 | { 177 | Err(Error::NotSupported("deserialize_any")) 178 | } 179 | 180 | fn deserialize_bool(self, visitor: V) -> Result 181 | where 182 | V: Visitor<'de>, 183 | { 184 | match self.input.get_u8() { 185 | 1 => visitor.visit_bool(true), 186 | 0 => visitor.visit_bool(false), 187 | value => Err(Error::InvalidBoolEncoding(value)), 188 | } 189 | } 190 | 191 | fn deserialize_i8(self, visitor: V) -> Result 192 | where 193 | V: Visitor<'de>, 194 | { 195 | let v = (self.input.get_u8() ^ (1 << 7)) as i8; 196 | visitor.visit_i8(v) 197 | } 198 | 199 | fn deserialize_i16(self, visitor: V) -> Result 200 | where 201 | V: Visitor<'de>, 202 | { 203 | let v = (self.input.get_u16() ^ (1 << 15)) as i16; 204 | visitor.visit_i16(v) 205 | } 206 | 207 | fn deserialize_i32(self, visitor: V) -> Result 208 | where 209 | V: Visitor<'de>, 210 | { 211 | let v = (self.input.get_u32() ^ (1 << 31)) as i32; 212 | visitor.visit_i32(v) 213 | } 214 | 215 | fn deserialize_i64(self, visitor: V) -> Result 216 | where 217 | V: Visitor<'de>, 218 | { 219 | let v = (self.input.get_u64() ^ (1 << 63)) as i64; 220 | visitor.visit_i64(v) 221 | } 222 | 223 | fn deserialize_i128(self, visitor: V) -> Result 224 | where 225 | V: Visitor<'de>, 226 | { 227 | let v = (self.input.get_u128() ^ (1 << 127)) as i128; 228 | visitor.visit_i128(v) 229 | } 230 | 231 | fn deserialize_u8(self, visitor: V) -> Result 232 | where 233 | V: Visitor<'de>, 234 | { 235 | visitor.visit_u8(self.input.get_u8()) 236 | } 237 | 238 | fn deserialize_u16(self, visitor: V) -> Result 239 | where 240 | V: Visitor<'de>, 241 | { 242 | visitor.visit_u16(self.input.get_u16()) 243 | } 244 | 245 | fn deserialize_u32(self, visitor: V) -> Result 246 | where 247 | V: Visitor<'de>, 248 | { 249 | visitor.visit_u32(self.input.get_u32()) 250 | } 251 | 252 | fn deserialize_u64(self, visitor: V) -> Result 253 | where 254 | V: Visitor<'de>, 255 | { 256 | visitor.visit_u64(self.input.get_u64()) 257 | } 258 | 259 | fn deserialize_u128(self, visitor: V) -> Result 260 | where 261 | V: Visitor<'de>, 262 | { 263 | visitor.visit_u128(self.input.get_u128()) 264 | } 265 | 266 | fn deserialize_f32(self, visitor: V) -> Result 267 | where 268 | V: Visitor<'de>, 269 | { 270 | let u = self.input.get_u32(); 271 | let u = if u & (1 << 31) != 0 { 272 | u & !(1 << 31) 273 | } else { 274 | !u 275 | }; 276 | visitor.visit_f32(f32::from_bits(u)) 277 | } 278 | 279 | fn deserialize_f64(self, visitor: V) -> Result 280 | where 281 | V: Visitor<'de>, 282 | { 283 | let u = self.input.get_u64(); 284 | let u = if u & (1 << 63) != 0 { 285 | u & !(1 << 63) 286 | } else { 287 | !u 288 | }; 289 | visitor.visit_f64(f64::from_bits(u)) 290 | } 291 | 292 | fn deserialize_char(self, visitor: V) -> Result 293 | where 294 | V: Visitor<'de>, 295 | { 296 | let u = self.input.get_u32(); 297 | visitor.visit_char(char::from_u32(u).ok_or(Error::InvalidCharEncoding(u))?) 298 | } 299 | 300 | fn deserialize_str(self, _visitor: V) -> Result 301 | where 302 | V: Visitor<'de>, 303 | { 304 | Err(Error::NotSupported("borrowed str")) 305 | } 306 | 307 | fn deserialize_string(self, visitor: V) -> Result 308 | where 309 | V: Visitor<'de>, 310 | { 311 | let bytes = self.read_bytes()?; 312 | visitor.visit_string(String::from_utf8(bytes)?) 313 | } 314 | 315 | fn deserialize_bytes(self, visitor: V) -> Result 316 | where 317 | V: Visitor<'de>, 318 | { 319 | let bytes = self.read_bytes()?; 320 | visitor.visit_bytes(&bytes) 321 | } 322 | 323 | fn deserialize_byte_buf(self, visitor: V) -> Result 324 | where 325 | V: Visitor<'de>, 326 | { 327 | let bytes = self.read_bytes()?; 328 | visitor.visit_byte_buf(bytes) 329 | } 330 | 331 | fn deserialize_option(self, visitor: V) -> Result 332 | where 333 | V: Visitor<'de>, 334 | { 335 | match self.input.get_u8() { 336 | 0 => visitor.visit_none(), 337 | 1 => visitor.visit_some(self), 338 | t => Err(Error::InvalidTagEncoding(t as usize)), 339 | } 340 | } 341 | 342 | fn deserialize_unit(self, visitor: V) -> Result 343 | where 344 | V: Visitor<'de>, 345 | { 346 | visitor.visit_unit() 347 | } 348 | 349 | fn deserialize_unit_struct(self, _name: &'static str, visitor: V) -> Result 350 | where 351 | V: Visitor<'de>, 352 | { 353 | self.deserialize_unit(visitor) 354 | } 355 | 356 | // As is done here, serializers are encouraged to treat newtype structs as 357 | // insignificant wrappers around the data they contain. That means not 358 | // parsing anything other than the contained value. 359 | fn deserialize_newtype_struct(self, _name: &'static str, visitor: V) -> Result 360 | where 361 | V: Visitor<'de>, 362 | { 363 | visitor.visit_newtype_struct(self) 364 | } 365 | 366 | fn deserialize_seq(self, visitor: V) -> Result 367 | where 368 | V: Visitor<'de>, 369 | { 370 | struct Access<'a, B: Buf> { 371 | deserializer: &'a mut Deserializer, 372 | } 373 | impl<'de, 'a, B: Buf + 'de> SeqAccess<'de> for Access<'a, B> { 374 | type Error = Error; 375 | 376 | fn next_element_seed(&mut self, seed: T) -> Result> 377 | where 378 | T: DeserializeSeed<'de>, 379 | { 380 | match self.deserializer.input.get_u8() { 381 | 1 => Ok(Some(DeserializeSeed::deserialize( 382 | seed, 383 | &mut *self.deserializer, 384 | )?)), 385 | 0 => Ok(None), 386 | value => Err(Error::InvalidSeqEncoding(value)), 387 | } 388 | } 389 | } 390 | 391 | visitor.visit_seq(Access { deserializer: self }) 392 | } 393 | 394 | fn deserialize_tuple(self, len: usize, visitor: V) -> Result 395 | where 396 | V: Visitor<'de>, 397 | { 398 | struct Access<'a, B: Buf> { 399 | deserializer: &'a mut Deserializer, 400 | len: usize, 401 | } 402 | 403 | impl<'de, 'a, B: Buf + 'de> SeqAccess<'de> for Access<'a, B> { 404 | type Error = Error; 405 | 406 | fn next_element_seed(&mut self, seed: T) -> Result> 407 | where 408 | T: DeserializeSeed<'de>, 409 | { 410 | if self.len > 0 { 411 | self.len -= 1; 412 | let value = DeserializeSeed::deserialize(seed, &mut *self.deserializer)?; 413 | Ok(Some(value)) 414 | } else { 415 | Ok(None) 416 | } 417 | } 418 | 419 | fn size_hint(&self) -> Option { 420 | Some(self.len) 421 | } 422 | } 423 | 424 | visitor.visit_seq(Access { 425 | deserializer: self, 426 | len, 427 | }) 428 | } 429 | 430 | fn deserialize_tuple_struct( 431 | self, 432 | _name: &'static str, 433 | len: usize, 434 | visitor: V, 435 | ) -> Result 436 | where 437 | V: Visitor<'de>, 438 | { 439 | self.deserialize_tuple(len, visitor) 440 | } 441 | 442 | fn deserialize_map(self, _visitor: V) -> Result 443 | where 444 | V: Visitor<'de>, 445 | { 446 | Err(Error::NotSupported("map")) 447 | } 448 | 449 | fn deserialize_struct( 450 | self, 451 | _name: &'static str, 452 | fields: &'static [&'static str], 453 | visitor: V, 454 | ) -> Result 455 | where 456 | V: Visitor<'de>, 457 | { 458 | self.deserialize_tuple(fields.len(), visitor) 459 | } 460 | 461 | fn deserialize_enum( 462 | self, 463 | _name: &'static str, 464 | _variants: &'static [&'static str], 465 | visitor: V, 466 | ) -> Result 467 | where 468 | V: Visitor<'de>, 469 | { 470 | impl<'de, 'a, B: Buf + 'de> EnumAccess<'de> for &'a mut Deserializer { 471 | type Error = Error; 472 | type Variant = Self; 473 | 474 | fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant)> 475 | where 476 | V: DeserializeSeed<'de>, 477 | { 478 | let idx = self.input.get_u8() as u32; 479 | let val: Result<_> = seed.deserialize(idx.into_deserializer()); 480 | Ok((val?, self)) 481 | } 482 | } 483 | 484 | visitor.visit_enum(self) 485 | } 486 | 487 | fn deserialize_identifier(self, _visitor: V) -> Result 488 | where 489 | V: Visitor<'de>, 490 | { 491 | Err(Error::NotSupported("deserialize_identifier")) 492 | } 493 | 494 | fn deserialize_ignored_any(self, _visitor: V) -> Result 495 | where 496 | V: Visitor<'de>, 497 | { 498 | Err(Error::NotSupported("deserialize_ignored_any")) 499 | } 500 | } 501 | 502 | // `VariantAccess` is provided to the `Visitor` to give it the ability to see 503 | // the content of the single variant that it decided to deserialize. 504 | impl<'de, 'a, B: Buf + 'de> VariantAccess<'de> for &'a mut Deserializer { 505 | type Error = Error; 506 | 507 | fn unit_variant(self) -> Result<()> { 508 | Ok(()) 509 | } 510 | 511 | fn newtype_variant_seed(self, seed: T) -> Result 512 | where 513 | T: DeserializeSeed<'de>, 514 | { 515 | seed.deserialize(self) 516 | } 517 | 518 | fn tuple_variant(self, len: usize, visitor: V) -> Result 519 | where 520 | V: Visitor<'de>, 521 | { 522 | serde::de::Deserializer::deserialize_tuple(self, len, visitor) 523 | } 524 | 525 | fn struct_variant(self, fields: &'static [&'static str], visitor: V) -> Result 526 | where 527 | V: Visitor<'de>, 528 | { 529 | serde::de::Deserializer::deserialize_tuple(self, fields.len(), visitor) 530 | } 531 | } 532 | 533 | impl Deserializer { 534 | /// Deserialize a decimal value. 535 | /// 536 | /// # Example 537 | /// ``` 538 | /// let buf = [0x15]; 539 | /// let mut de = memcomparable::Deserializer::new(&buf[..]); 540 | /// let v = de.deserialize_decimal().unwrap(); 541 | /// assert_eq!(v.to_string(), "0"); 542 | /// ``` 543 | #[cfg(feature = "decimal")] 544 | #[cfg_attr(docsrs, doc(cfg(feature = "decimal")))] 545 | pub fn deserialize_decimal(&mut self) -> Result { 546 | // decode exponent 547 | let flag = self.input.get_u8(); 548 | let exponent = match flag { 549 | 0x07 => return Ok(Decimal::NegInf), 550 | 0x08 => !self.input.get_u8() as i8, 551 | 0x09..=0x13 => (0x13 - flag) as i8, 552 | 0x14 => -(self.input.get_u8() as i8), 553 | 0x15 => return Ok(Decimal::ZERO), 554 | 0x16 => -!(self.input.get_u8() as i8), 555 | 0x17..=0x21 => (flag - 0x17) as i8, 556 | 0x22 => self.input.get_u8() as i8, 557 | 0x23 => return Ok(Decimal::Inf), 558 | 0x24 => return Ok(Decimal::NaN), 559 | b => return Err(Error::InvalidDecimalEncoding(b)), 560 | }; 561 | // decode mantissa 562 | let neg = (0x07..0x15).contains(&flag); 563 | let mut mantissa: i128 = 0; 564 | let mut mlen = 0i8; 565 | loop { 566 | let mut b = self.input.get_u8(); 567 | if neg { 568 | b = !b; 569 | } 570 | let x = b / 2; 571 | mantissa = mantissa * 100 + x as i128; 572 | mlen += 1; 573 | if b & 1 == 0 { 574 | break; 575 | } 576 | } 577 | 578 | // get scale 579 | let mut scale = (mlen - exponent) * 2; 580 | if scale <= 0 { 581 | // e.g. 1(mantissa) + 2(exponent) (which is 100). 582 | for _i in 0..-scale { 583 | mantissa *= 10; 584 | } 585 | scale = 0; 586 | } else if mantissa % 10 == 0 { 587 | // Remove unnecessary zeros. 588 | // e.g. 0.01_11_10 should be 0.01_11_1 589 | mantissa /= 10; 590 | scale -= 1; 591 | } 592 | 593 | if neg { 594 | mantissa = -mantissa; 595 | } 596 | Ok(rust_decimal::Decimal::from_i128_with_scale(mantissa, scale as u32).into()) 597 | } 598 | } 599 | 600 | #[cfg(test)] 601 | mod tests { 602 | use serde::Deserialize; 603 | 604 | use super::*; 605 | 606 | #[test] 607 | fn test_unit() { 608 | assert_eq!(from_slice::<()>(&[]), Ok(())); 609 | assert_eq!(from_slice::<()>(&[0]), Err(Error::TrailingCharacters)); 610 | 611 | #[derive(Debug, PartialEq, Eq, Deserialize)] 612 | struct UnitStruct; 613 | assert_eq!(from_slice::(&[]).unwrap(), UnitStruct); 614 | } 615 | 616 | #[test] 617 | fn test_bool() { 618 | assert_eq!(from_slice::(&[0]), Ok(false)); 619 | assert_eq!(from_slice::(&[1]), Ok(true)); 620 | assert_eq!(from_slice::(&[2]), Err(Error::InvalidBoolEncoding(2))); 621 | } 622 | 623 | #[test] 624 | fn test_option() { 625 | assert_eq!(from_slice::>(&[0]).unwrap(), None); 626 | assert_eq!(from_slice::>(&[1, 0x12]).unwrap(), Some(0x12)); 627 | } 628 | 629 | #[test] 630 | fn test_tuple() { 631 | assert_eq!( 632 | from_slice::<(i8, i16, i32, i64, i128)>(&[ 633 | 0x92, 0x92, 0x34, 0x92, 0x34, 0x56, 0x78, 0x92, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 634 | 0x21, 0x81, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 635 | 0x54, 0x32, 0x10, 636 | ]) 637 | .unwrap(), 638 | ( 639 | 0x12, 640 | 0x1234, 641 | 0x12345678, 642 | 0x1234_5678_8765_4321, 643 | 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210 644 | ) 645 | ); 646 | 647 | #[derive(Debug, PartialEq, Eq, Deserialize)] 648 | struct TupleStruct(u8, u16, u32, u64, u128); 649 | assert_eq!( 650 | from_slice::(&[ 651 | 0x12, 0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 652 | 0x21, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 653 | 0x54, 0x32, 0x10, 654 | ]) 655 | .unwrap(), 656 | TupleStruct( 657 | 0x12, 658 | 0x1234, 659 | 0x12345678, 660 | 0x1234_5678_8765_4321, 661 | 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210 662 | ) 663 | ); 664 | 665 | #[derive(Debug, PartialEq, Eq, Deserialize)] 666 | struct NewTypeStruct(char); 667 | assert_eq!( 668 | from_slice::(&[0, 0, 0, b'G']).unwrap(), 669 | NewTypeStruct('G') 670 | ); 671 | } 672 | 673 | #[test] 674 | fn test_vec() { 675 | assert_eq!( 676 | from_slice::>(&[1, 0x01, 1, 0x02, 1, 0x03, 0]).unwrap(), 677 | vec![1, 2, 3] 678 | ); 679 | assert_eq!( 680 | from_slice::>(&[1, 0x01, 2]), 681 | Err(Error::InvalidSeqEncoding(2)) 682 | ); 683 | } 684 | 685 | #[test] 686 | fn test_enum() { 687 | #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize)] 688 | enum TestEnum { 689 | Unit, 690 | NewType(u8), 691 | Tuple(u8, u8), 692 | Struct { a: u8, b: u8 }, 693 | } 694 | 695 | assert_eq!(from_slice::(&[0]).unwrap(), TestEnum::Unit); 696 | assert_eq!( 697 | from_slice::(&[1, 0x12]).unwrap(), 698 | TestEnum::NewType(0x12) 699 | ); 700 | assert_eq!( 701 | from_slice::(&[2, 0x12, 0x34]).unwrap(), 702 | TestEnum::Tuple(0x12, 0x34) 703 | ); 704 | assert_eq!( 705 | from_slice::(&[3, 0x12, 0x34]).unwrap(), 706 | TestEnum::Struct { a: 0x12, b: 0x34 } 707 | ); 708 | } 709 | 710 | #[test] 711 | fn test_struct() { 712 | #[derive(Debug, PartialEq, PartialOrd, Deserialize)] 713 | struct Test { 714 | a: bool, 715 | b: f32, 716 | c: f64, 717 | } 718 | assert_eq!( 719 | from_slice::(&[1, 0x80, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0]).unwrap(), 720 | Test { 721 | a: true, 722 | b: 0.0, 723 | c: 0.0, 724 | } 725 | ); 726 | } 727 | 728 | #[test] 729 | fn test_string() { 730 | assert_eq!(from_slice::(&[0]).unwrap(), "".to_string()); 731 | assert_eq!( 732 | from_slice::(&[1, b'1', b'2', b'3', 0, 0, 0, 0, 0, 3]).unwrap(), 733 | "123".to_string() 734 | ); 735 | assert_eq!( 736 | from_slice::(&[1, b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', 8]).unwrap(), 737 | "12345678".to_string() 738 | ); 739 | assert_eq!( 740 | from_slice::(&[ 741 | 1, b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', 9, b'9', b'0', 0, 0, 0, 0, 0, 0, 742 | 2 743 | ]) 744 | .unwrap(), 745 | "1234567890".to_string() 746 | ); 747 | assert_eq!( 748 | from_slice::(&[1, 0, 0, 0, 0, 0, 0, 0, 0, 10]), 749 | Err(Error::InvalidBytesEncoding(10)) 750 | ); 751 | assert_eq!( 752 | from_slice::(&[2]), 753 | Err(Error::InvalidBytesEncoding(2)) 754 | ); 755 | } 756 | 757 | #[test] 758 | #[cfg(feature = "decimal")] 759 | fn test_decimal() { 760 | // Notice: decimals like 100.00 will be decoding as 100. 761 | 762 | let decimals = [ 763 | "-inf", 764 | "-123456789012345678901234", 765 | "-1234567890.1234", 766 | "-233.3", 767 | "-0.001", 768 | "0", 769 | "0.001", 770 | "0.01111", 771 | "50", 772 | "100", 773 | "12345", 774 | "41721.900909090909090909090909", 775 | "123456789012345678901234", 776 | "inf", 777 | "nan", 778 | ]; 779 | let mut last_encoding = vec![]; 780 | for s in decimals { 781 | let decimal: Decimal = s.parse().unwrap(); 782 | let encoding = serialize_decimal(decimal); 783 | assert_eq!(deserialize_decimal(&encoding), decimal); 784 | assert!(encoding > last_encoding); 785 | last_encoding = encoding; 786 | } 787 | } 788 | 789 | #[cfg(feature = "decimal")] 790 | fn serialize_decimal(decimal: impl Into) -> Vec { 791 | let mut serializer = crate::Serializer::new(vec![]); 792 | serializer.serialize_decimal(decimal.into()).unwrap(); 793 | serializer.into_inner() 794 | } 795 | 796 | #[cfg(feature = "decimal")] 797 | fn deserialize_decimal(bytes: &[u8]) -> Decimal { 798 | let mut deserializer = Deserializer::new(bytes); 799 | deserializer.deserialize_decimal().unwrap() 800 | } 801 | } 802 | -------------------------------------------------------------------------------- /src/ser.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Singularity Data 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use bytes::BufMut; 16 | use serde::{ser, Serialize}; 17 | 18 | #[cfg(feature = "decimal")] 19 | use crate::decimal::Decimal; 20 | use crate::error::{Error, Result}; 21 | 22 | /// A structure for serializing Rust values into a memcomparable bytes. 23 | pub struct Serializer { 24 | output: MaybeFlip, 25 | } 26 | 27 | impl Serializer { 28 | /// Create a new `Serializer`. 29 | pub fn new(buffer: B) -> Self { 30 | Serializer { 31 | output: MaybeFlip { 32 | output: buffer, 33 | flip: false, 34 | }, 35 | } 36 | } 37 | 38 | /// Unwrap the inner buffer from the `Serializer`. 39 | pub fn into_inner(self) -> B { 40 | self.output.output 41 | } 42 | 43 | /// Set whether data is serialized in reverse order. 44 | pub fn set_reverse(&mut self, reverse: bool) { 45 | self.output.flip = reverse; 46 | } 47 | } 48 | 49 | /// Serialize the given data structure as a memcomparable byte vector. 50 | pub fn to_vec(value: &impl Serialize) -> Result> { 51 | let mut serializer = Serializer::new(vec![]); 52 | value.serialize(&mut serializer)?; 53 | Ok(serializer.into_inner()) 54 | } 55 | 56 | /// A wrapper around `BufMut` that can flip bits when putting data. 57 | struct MaybeFlip { 58 | output: B, 59 | flip: bool, 60 | } 61 | 62 | macro_rules! def_method { 63 | ($name:ident, $ty:ty) => { 64 | fn $name(&mut self, value: $ty) { 65 | self.output.$name(if self.flip { !value } else { value }); 66 | } 67 | }; 68 | } 69 | 70 | impl MaybeFlip { 71 | def_method!(put_u8, u8); 72 | 73 | def_method!(put_u16, u16); 74 | 75 | def_method!(put_u32, u32); 76 | 77 | def_method!(put_u64, u64); 78 | 79 | def_method!(put_u128, u128); 80 | 81 | fn put_slice(&mut self, src: &[u8]) { 82 | for &val in src { 83 | let val = if self.flip { !val } else { val }; 84 | self.output.put_u8(val); 85 | } 86 | } 87 | 88 | fn put_bytes(&mut self, val: u8, cnt: usize) { 89 | let val = if self.flip { !val } else { val }; 90 | self.output.put_bytes(val, cnt); 91 | } 92 | } 93 | 94 | // Format Reference: 95 | // https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format 96 | // https://haxisnake.github.io/2020/11/06/TIDB源码学习笔记-基本类型编解码方案/ 97 | impl<'a, B: BufMut> ser::Serializer for &'a mut Serializer { 98 | type Error = Error; 99 | type Ok = (); 100 | type SerializeMap = Self; 101 | type SerializeSeq = Self; 102 | type SerializeStruct = Self; 103 | type SerializeStructVariant = Self; 104 | type SerializeTuple = Self; 105 | type SerializeTupleStruct = Self; 106 | type SerializeTupleVariant = Self; 107 | 108 | fn serialize_bool(self, v: bool) -> Result<()> { 109 | self.serialize_u8(v as u8) 110 | } 111 | 112 | fn serialize_i8(self, v: i8) -> Result<()> { 113 | let u = v as u8 ^ (1 << 7); 114 | self.serialize_u8(u) 115 | } 116 | 117 | fn serialize_i16(self, v: i16) -> Result<()> { 118 | let u = v as u16 ^ (1 << 15); 119 | self.serialize_u16(u) 120 | } 121 | 122 | fn serialize_i32(self, v: i32) -> Result<()> { 123 | let u = v as u32 ^ (1 << 31); 124 | self.serialize_u32(u) 125 | } 126 | 127 | fn serialize_i64(self, v: i64) -> Result<()> { 128 | let u = v as u64 ^ (1 << 63); 129 | self.serialize_u64(u) 130 | } 131 | 132 | fn serialize_i128(self, v: i128) -> Result<()> { 133 | let u = v as u128 ^ (1 << 127); 134 | self.serialize_u128(u) 135 | } 136 | 137 | fn serialize_u8(self, v: u8) -> Result<()> { 138 | self.output.put_u8(v); 139 | Ok(()) 140 | } 141 | 142 | fn serialize_u16(self, v: u16) -> Result<()> { 143 | self.output.put_u16(v); 144 | Ok(()) 145 | } 146 | 147 | fn serialize_u32(self, v: u32) -> Result<()> { 148 | self.output.put_u32(v); 149 | Ok(()) 150 | } 151 | 152 | fn serialize_u64(self, v: u64) -> Result<()> { 153 | self.output.put_u64(v); 154 | Ok(()) 155 | } 156 | 157 | fn serialize_u128(self, v: u128) -> Result<()> { 158 | self.output.put_u128(v); 159 | Ok(()) 160 | } 161 | 162 | fn serialize_f32(self, mut v: f32) -> Result<()> { 163 | if v.is_nan() { 164 | v = f32::NAN; // normalize pos/neg NaN 165 | } else if v == 0.0 { 166 | v = 0.0; // normalize pos/neg zero 167 | } 168 | let u = v.to_bits(); 169 | let u = if v.is_sign_positive() { 170 | u | (1 << 31) 171 | } else { 172 | !u 173 | }; 174 | self.output.put_u32(u); 175 | Ok(()) 176 | } 177 | 178 | fn serialize_f64(self, mut v: f64) -> Result<()> { 179 | if v.is_nan() { 180 | v = f64::NAN; // normalize pos/neg NaN 181 | } else if v == 0.0 { 182 | v = 0.0; // normalize pos/neg zero 183 | } 184 | let u = v.to_bits(); 185 | let u = if v.is_sign_positive() { 186 | u | (1 << 63) 187 | } else { 188 | !u 189 | }; 190 | self.output.put_u64(u); 191 | Ok(()) 192 | } 193 | 194 | fn serialize_char(self, v: char) -> Result<()> { 195 | self.serialize_u32(v as u32) 196 | } 197 | 198 | fn serialize_str(self, v: &str) -> Result<()> { 199 | self.serialize_bytes(v.as_bytes()) 200 | } 201 | 202 | fn serialize_bytes(self, v: &[u8]) -> Result<()> { 203 | self.output.put_u8(!v.is_empty() as u8); 204 | let mut len = 0; 205 | for chunk in v.chunks(8) { 206 | self.output.put_slice(chunk); 207 | if chunk.len() != 8 { 208 | self.output.put_bytes(0, 8 - chunk.len()); 209 | } 210 | len += chunk.len(); 211 | // append an extra byte that signals the number of significant bytes in this chunk 212 | // 1-8: many bytes were significant and this group is the last group 213 | // 9: all 8 bytes were significant and there is more data to come 214 | let extra = if len == v.len() { chunk.len() as u8 } else { 9 }; 215 | self.output.put_u8(extra); 216 | } 217 | Ok(()) 218 | } 219 | 220 | fn serialize_none(self) -> Result<()> { 221 | self.serialize_u8(0) 222 | } 223 | 224 | fn serialize_some(self, value: &T) -> Result<()> 225 | where 226 | T: ?Sized + Serialize, 227 | { 228 | self.serialize_u8(1)?; 229 | value.serialize(self) 230 | } 231 | 232 | fn serialize_unit(self) -> Result<()> { 233 | Ok(()) 234 | } 235 | 236 | fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { 237 | self.serialize_unit() 238 | } 239 | 240 | fn serialize_unit_variant( 241 | self, 242 | _name: &'static str, 243 | variant_index: u32, 244 | _variant: &'static str, 245 | ) -> Result<()> { 246 | assert!(variant_index <= u8::MAX as u32, "too many variants"); 247 | self.serialize_u8(variant_index as u8) 248 | } 249 | 250 | fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> 251 | where 252 | T: ?Sized + Serialize, 253 | { 254 | value.serialize(self) 255 | } 256 | 257 | fn serialize_newtype_variant( 258 | self, 259 | _name: &'static str, 260 | variant_index: u32, 261 | _variant: &'static str, 262 | value: &T, 263 | ) -> Result<()> 264 | where 265 | T: ?Sized + Serialize, 266 | { 267 | assert!(variant_index <= u8::MAX as u32, "too many variants"); 268 | self.serialize_u8(variant_index as u8)?; 269 | value.serialize(&mut *self)?; 270 | Ok(()) 271 | } 272 | 273 | fn serialize_seq(self, _len: Option) -> Result { 274 | Ok(self) 275 | } 276 | 277 | fn serialize_tuple(self, _len: usize) -> Result { 278 | Ok(self) 279 | } 280 | 281 | fn serialize_tuple_struct( 282 | self, 283 | _name: &'static str, 284 | _len: usize, 285 | ) -> Result { 286 | Ok(self) 287 | } 288 | 289 | fn serialize_tuple_variant( 290 | self, 291 | _name: &'static str, 292 | variant_index: u32, 293 | _variant: &'static str, 294 | _len: usize, 295 | ) -> Result { 296 | assert!(variant_index <= u8::MAX as u32, "too many variants"); 297 | self.serialize_u8(variant_index as u8)?; 298 | Ok(self) 299 | } 300 | 301 | fn serialize_map(self, _len: Option) -> Result { 302 | Err(Error::NotSupported("map")) 303 | } 304 | 305 | fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { 306 | Ok(self) 307 | } 308 | 309 | fn serialize_struct_variant( 310 | self, 311 | _name: &'static str, 312 | variant_index: u32, 313 | _variant: &'static str, 314 | _len: usize, 315 | ) -> Result { 316 | assert!(variant_index <= u8::MAX as u32, "too many variants"); 317 | self.serialize_u8(variant_index as u8)?; 318 | Ok(self) 319 | } 320 | 321 | fn is_human_readable(&self) -> bool { 322 | false 323 | } 324 | } 325 | 326 | impl<'a, B: BufMut> ser::SerializeSeq for &'a mut Serializer { 327 | type Error = Error; 328 | type Ok = (); 329 | 330 | fn serialize_element(&mut self, value: &T) -> Result<()> 331 | where 332 | T: ?Sized + Serialize, 333 | { 334 | use serde::Serializer; 335 | self.serialize_u8(1)?; 336 | value.serialize(&mut **self) 337 | } 338 | 339 | fn end(self) -> Result<()> { 340 | use serde::Serializer; 341 | self.serialize_u8(0)?; 342 | Ok(()) 343 | } 344 | } 345 | 346 | impl<'a, B: BufMut> ser::SerializeTuple for &'a mut Serializer { 347 | type Error = Error; 348 | type Ok = (); 349 | 350 | fn serialize_element(&mut self, value: &T) -> Result<()> 351 | where 352 | T: ?Sized + Serialize, 353 | { 354 | value.serialize(&mut **self) 355 | } 356 | 357 | fn end(self) -> Result<()> { 358 | Ok(()) 359 | } 360 | } 361 | 362 | impl<'a, B: BufMut> ser::SerializeTupleStruct for &'a mut Serializer { 363 | type Error = Error; 364 | type Ok = (); 365 | 366 | fn serialize_field(&mut self, value: &T) -> Result<()> 367 | where 368 | T: ?Sized + Serialize, 369 | { 370 | value.serialize(&mut **self) 371 | } 372 | 373 | fn end(self) -> Result<()> { 374 | Ok(()) 375 | } 376 | } 377 | 378 | impl<'a, B: BufMut> ser::SerializeTupleVariant for &'a mut Serializer { 379 | type Error = Error; 380 | type Ok = (); 381 | 382 | fn serialize_field(&mut self, value: &T) -> Result<()> 383 | where 384 | T: ?Sized + Serialize, 385 | { 386 | value.serialize(&mut **self) 387 | } 388 | 389 | fn end(self) -> Result<()> { 390 | Ok(()) 391 | } 392 | } 393 | 394 | impl<'a, B: BufMut> ser::SerializeMap for &'a mut Serializer { 395 | type Error = Error; 396 | type Ok = (); 397 | 398 | fn serialize_key(&mut self, key: &T) -> Result<()> 399 | where 400 | T: ?Sized + Serialize, 401 | { 402 | key.serialize(&mut **self) 403 | } 404 | 405 | fn serialize_value(&mut self, value: &T) -> Result<()> 406 | where 407 | T: ?Sized + Serialize, 408 | { 409 | value.serialize(&mut **self) 410 | } 411 | 412 | fn end(self) -> Result<()> { 413 | Ok(()) 414 | } 415 | } 416 | 417 | impl<'a, B: BufMut> ser::SerializeStruct for &'a mut Serializer { 418 | type Error = Error; 419 | type Ok = (); 420 | 421 | fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> 422 | where 423 | T: ?Sized + Serialize, 424 | { 425 | value.serialize(&mut **self) 426 | } 427 | 428 | fn end(self) -> Result<()> { 429 | Ok(()) 430 | } 431 | } 432 | 433 | impl<'a, B: BufMut> ser::SerializeStructVariant for &'a mut Serializer { 434 | type Error = Error; 435 | type Ok = (); 436 | 437 | fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> 438 | where 439 | T: ?Sized + Serialize, 440 | { 441 | value.serialize(&mut **self) 442 | } 443 | 444 | fn end(self) -> Result<()> { 445 | Ok(()) 446 | } 447 | } 448 | 449 | impl Serializer { 450 | /// Serialize a decimal value. 451 | /// 452 | /// The encoding format follows `SQLite`: 453 | /// except that NaN is considered larger than +Infinity, to be consistent with f32 and f64. 454 | /// 455 | /// # Example 456 | /// ``` 457 | /// use memcomparable::Decimal; 458 | /// 459 | /// let d1 = Decimal::Normalized("12.34".parse().unwrap()); 460 | /// let d2 = Decimal::Inf; 461 | /// 462 | /// let mut ser = memcomparable::Serializer::new(vec![]); 463 | /// ser.serialize_decimal(d1.into()).unwrap(); 464 | /// ser.serialize_decimal(d2).unwrap(); 465 | /// ``` 466 | #[cfg(feature = "decimal")] 467 | #[cfg_attr(docsrs, doc(cfg(feature = "decimal")))] 468 | pub fn serialize_decimal(&mut self, decimal: Decimal) -> Result<()> { 469 | let decimal = match decimal { 470 | Decimal::NaN => { 471 | self.output.put_u8(0x24); 472 | return Ok(()); 473 | } 474 | Decimal::NegInf => { 475 | self.output.put_u8(0x07); 476 | return Ok(()); 477 | } 478 | Decimal::Inf => { 479 | self.output.put_u8(0x23); 480 | return Ok(()); 481 | } 482 | Decimal::Normalized(d) if d.is_zero() => { 483 | self.output.put_u8(0x15); 484 | return Ok(()); 485 | } 486 | Decimal::Normalized(d) => d, 487 | }; 488 | let (exponent, significand) = Self::decimal_e_m(decimal); 489 | if decimal.is_sign_positive() { 490 | match exponent { 491 | 11.. => { 492 | self.output.put_u8(0x22); 493 | self.output.put_u8(exponent as u8); 494 | } 495 | 0..=10 => { 496 | self.output.put_u8(0x17 + exponent as u8); 497 | } 498 | _ => { 499 | self.output.put_u8(0x16); 500 | self.output.put_u8(!(-exponent) as u8); 501 | } 502 | } 503 | self.output.put_slice(&significand); 504 | } else { 505 | match exponent { 506 | 11.. => { 507 | self.output.put_u8(0x8); 508 | self.output.put_u8(!exponent as u8); 509 | } 510 | 0..=10 => { 511 | self.output.put_u8(0x13 - exponent as u8); 512 | } 513 | _ => { 514 | self.output.put_u8(0x14); 515 | self.output.put_u8(-exponent as u8); 516 | } 517 | } 518 | for b in significand { 519 | self.output.put_u8(!b); 520 | } 521 | } 522 | Ok(()) 523 | } 524 | 525 | /// Get the exponent and significand mantissa from a decimal. 526 | #[cfg(feature = "decimal")] 527 | fn decimal_e_m(decimal: rust_decimal::Decimal) -> (i8, Vec) { 528 | if decimal.is_zero() { 529 | return (0, vec![]); 530 | } 531 | const POW10: [u128; 30] = [ 532 | 1, 533 | 10, 534 | 100, 535 | 1000, 536 | 10000, 537 | 100000, 538 | 1000000, 539 | 10000000, 540 | 100000000, 541 | 1000000000, 542 | 10000000000, 543 | 100000000000, 544 | 1000000000000, 545 | 10000000000000, 546 | 100000000000000, 547 | 1000000000000000, 548 | 10000000000000000, 549 | 100000000000000000, 550 | 1000000000000000000, 551 | 10000000000000000000, 552 | 100000000000000000000, 553 | 1000000000000000000000, 554 | 10000000000000000000000, 555 | 100000000000000000000000, 556 | 1000000000000000000000000, 557 | 10000000000000000000000000, 558 | 100000000000000000000000000, 559 | 1000000000000000000000000000, 560 | 10000000000000000000000000000, 561 | 100000000000000000000000000000, 562 | ]; 563 | let mut mantissa = decimal.mantissa().unsigned_abs(); 564 | let prec = POW10.as_slice().partition_point(|&p| p <= mantissa); 565 | 566 | let e10 = prec as i32 - decimal.scale() as i32; 567 | let e100 = if e10 >= 0 { (e10 + 1) / 2 } else { e10 / 2 }; 568 | // Maybe need to add a zero at the beginning. 569 | // e.g. 111.11 -> 2(exponent which is 100 based) + 0.011111(mantissa). 570 | // So, the `digit_num` of 111.11 will be 6. 571 | let mut digit_num = if e10 == 2 * e100 { prec } else { prec + 1 }; 572 | 573 | let mut byte_array = Vec::with_capacity(16); 574 | // Remove trailing zero. 575 | while mantissa % 10 == 0 && mantissa != 0 { 576 | mantissa /= 10; 577 | digit_num -= 1; 578 | } 579 | 580 | // Cases like: 0.12345, not 0.01111. 581 | if digit_num % 2 == 1 { 582 | mantissa *= 10; 583 | // digit_num += 1; 584 | } 585 | while mantissa >> 64 != 0 { 586 | let byte = (mantissa % 100) as u8 * 2 + 1; 587 | byte_array.push(byte); 588 | mantissa /= 100; 589 | } 590 | // optimize for division 591 | let mut mantissa = mantissa as u64; 592 | while mantissa != 0 { 593 | let byte = (mantissa % 100) as u8 * 2 + 1; 594 | byte_array.push(byte); 595 | mantissa /= 100; 596 | } 597 | byte_array[0] -= 1; 598 | byte_array.reverse(); 599 | 600 | (e100 as i8, byte_array) 601 | } 602 | } 603 | 604 | #[cfg(test)] 605 | mod tests { 606 | use rand::distributions::Alphanumeric; 607 | use rand::Rng; 608 | use serde::Serialize; 609 | 610 | use super::*; 611 | 612 | #[test] 613 | fn test_unit() { 614 | assert_eq!(to_vec(&()).unwrap(), []); 615 | 616 | #[derive(Serialize)] 617 | struct UnitStruct; 618 | assert_eq!(to_vec(&UnitStruct).unwrap(), []); 619 | } 620 | 621 | #[test] 622 | fn test_option() { 623 | assert_eq!(to_vec(&(None as Option)).unwrap(), [0]); 624 | assert_eq!(to_vec(&Some(0x12u8)).unwrap(), [1, 0x12]); 625 | } 626 | 627 | #[test] 628 | fn test_tuple() { 629 | let tuple: (i8, i16, i32, i64, i128) = ( 630 | 0x12, 631 | 0x1234, 632 | 0x12345678, 633 | 0x1234_5678_8765_4321, 634 | 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210, 635 | ); 636 | assert_eq!( 637 | to_vec(&tuple).unwrap(), 638 | [ 639 | 0x92, 0x92, 0x34, 0x92, 0x34, 0x56, 0x78, 0x92, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 640 | 0x21, 0x81, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 641 | 0x54, 0x32, 0x10, 642 | ] 643 | ); 644 | 645 | #[derive(Serialize)] 646 | struct TupleStruct(u8, u16, u32, u64, u128); 647 | let tuple = TupleStruct( 648 | 0x12, 649 | 0x1234, 650 | 0x12345678, 651 | 0x1234_5678_8765_4321, 652 | 0x0123_4567_89ab_cdef_fedc_ba98_7654_3210, 653 | ); 654 | assert_eq!( 655 | to_vec(&tuple).unwrap(), 656 | [ 657 | 0x12, 0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x12, 0x34, 0x56, 0x78, 0x87, 0x65, 0x43, 658 | 0x21, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 659 | 0x54, 0x32, 0x10, 660 | ] 661 | ); 662 | 663 | #[derive(Serialize)] 664 | struct NewTypeStruct(char); 665 | let tuple = NewTypeStruct('G'); 666 | assert_eq!(to_vec(&tuple).unwrap(), [0, 0, 0, b'G']); 667 | } 668 | 669 | #[test] 670 | fn test_vec() { 671 | let s: &[u8] = &[1, 2, 3]; 672 | assert_eq!(to_vec(&s).unwrap(), [1, 0x01, 1, 0x02, 1, 0x03, 0]); 673 | } 674 | 675 | #[test] 676 | fn test_enum() { 677 | #[derive(PartialEq, Eq, PartialOrd, Ord, Serialize)] 678 | enum TestEnum { 679 | Unit, 680 | NewType(u8), 681 | Tuple(u8, u8), 682 | Struct { a: u8, b: u8 }, 683 | } 684 | 685 | let test = TestEnum::Unit; 686 | assert_eq!(to_vec(&test).unwrap(), [0]); 687 | 688 | let test = TestEnum::NewType(0x12); 689 | assert_eq!(to_vec(&test).unwrap(), [1, 0x12]); 690 | 691 | let test = TestEnum::Tuple(0x12, 0x34); 692 | assert_eq!(to_vec(&test).unwrap(), [2, 0x12, 0x34]); 693 | 694 | let test = TestEnum::Struct { a: 0x12, b: 0x34 }; 695 | assert_eq!(to_vec(&test).unwrap(), [3, 0x12, 0x34]); 696 | } 697 | 698 | #[derive(PartialEq, PartialOrd, Serialize)] 699 | struct Test { 700 | a: bool, 701 | b: f32, 702 | c: f64, 703 | } 704 | 705 | #[test] 706 | fn test_struct() { 707 | let test = Test { 708 | a: true, 709 | b: 0.0, 710 | c: 0.0, 711 | }; 712 | assert_eq!( 713 | to_vec(&test).unwrap(), 714 | [1, 0x80, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0] 715 | ); 716 | } 717 | 718 | #[test] 719 | fn test_struct_order() { 720 | for _ in 0..1000 { 721 | let mut rng = rand::thread_rng(); 722 | let a = Test { 723 | a: rng.gen(), 724 | b: rng.gen(), 725 | c: rng.gen(), 726 | }; 727 | let b = Test { 728 | a: if rng.gen_bool(0.5) { a.a } else { rng.gen() }, 729 | b: if rng.gen_bool(0.5) { 730 | a.b 731 | } else { 732 | rng.gen_range(-1.0..1.0) 733 | }, 734 | c: if rng.gen_bool(0.5) { 735 | a.c 736 | } else { 737 | rng.gen_range(-1.0..1.0) 738 | }, 739 | }; 740 | let ea = to_vec(&a).unwrap(); 741 | let eb = to_vec(&b).unwrap(); 742 | assert_eq!(a.partial_cmp(&b), ea.partial_cmp(&eb)); 743 | } 744 | } 745 | 746 | #[test] 747 | fn test_string() { 748 | assert_eq!(to_vec(&"").unwrap(), [0]); 749 | assert_eq!( 750 | to_vec(&"123").unwrap(), 751 | [1, b'1', b'2', b'3', 0, 0, 0, 0, 0, 3] 752 | ); 753 | assert_eq!( 754 | to_vec(&"12345678").unwrap(), 755 | [1, b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', 8] 756 | ); 757 | assert_eq!( 758 | to_vec(&"1234567890").unwrap(), 759 | [ 760 | 1, b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', 9, b'9', b'0', 0, 0, 0, 0, 0, 0, 761 | 2 762 | ] 763 | ); 764 | } 765 | 766 | #[test] 767 | fn test_string_order() { 768 | fn to_vec_desc(s: &str) -> Vec { 769 | let mut ser = Serializer::new(vec![]); 770 | ser.set_reverse(true); 771 | s.serialize(&mut ser).unwrap(); 772 | ser.into_inner() 773 | } 774 | 775 | for _ in 0..1000 { 776 | let s = rand_string(0..16); 777 | let a = s.clone() + &rand_string(0..16); 778 | let b = s + &rand_string(0..16); 779 | 780 | let ea = to_vec(&a).unwrap(); 781 | let eb = to_vec(&b).unwrap(); 782 | assert_eq!(a.cmp(&b), ea.cmp(&eb)); 783 | 784 | let ra = to_vec_desc(&a); 785 | let rb = to_vec_desc(&b); 786 | assert_eq!(a.cmp(&b), ra.cmp(&rb).reverse()); 787 | } 788 | } 789 | 790 | fn rand_string(len_range: std::ops::Range) -> String { 791 | let mut rng = rand::thread_rng(); 792 | let len = rng.gen_range(len_range); 793 | rng.sample_iter(&Alphanumeric) 794 | .take(len) 795 | .map(char::from) 796 | .collect() 797 | } 798 | 799 | #[test] 800 | fn test_float_order() { 801 | let mut last_encoding = vec![]; 802 | for v in [ 803 | f32::NEG_INFINITY, 804 | f32::MIN, 805 | -0.0f32, 806 | f32::MIN_POSITIVE, 807 | f32::MAX, 808 | f32::INFINITY, 809 | -f32::NAN, 810 | ] { 811 | let encoding = to_vec(&v).unwrap(); 812 | assert!(encoding > last_encoding); 813 | last_encoding = encoding; 814 | } 815 | } 816 | 817 | #[test] 818 | #[cfg(feature = "decimal")] 819 | fn test_decimal_e_m() { 820 | // from: https://sqlite.org/src4/doc/trunk/www/key_encoding.wiki 821 | let cases = vec![ 822 | // (decimal, exponents, significand) 823 | ("1.0", 1, "02"), 824 | ("10.0", 1, "14"), 825 | ("10", 1, "14"), 826 | ("99.0", 1, "c6"), 827 | ("99.01", 1, "c7 02"), 828 | ("99.0001", 1, "c7 01 02"), 829 | ("100.0", 2, "02"), 830 | ("100.01", 2, "03 01 02"), 831 | ("100.1", 2, "03 01 14"), 832 | ("111.11", 2, "03 17 16"), 833 | ("1234", 2, "19 44"), 834 | ("9999", 2, "c7 c6"), 835 | ("9999.000001", 2, "c7 c7 01 01 02"), 836 | ("9999.000009", 2, "c7 c7 01 01 12"), 837 | ("9999.00001", 2, "c7 c7 01 01 14"), 838 | ("9999.00009", 2, "c7 c7 01 01 b4"), 839 | ("9999.000099", 2, "c7 c7 01 01 c6"), 840 | ("9999.0001", 2, "c7 c7 01 02"), 841 | ("9999.001", 2, "c7 c7 01 14"), 842 | ("9999.01", 2, "c7 c7 02"), 843 | ("9999.1", 2, "c7 c7 14"), 844 | ("10000", 3, "02"), 845 | ("10001", 3, "03 01 02"), 846 | ("12345", 3, "03 2f 5a"), 847 | ("123450", 3, "19 45 64"), 848 | ("1234.5", 2, "19 45 64"), 849 | ("12.345", 1, "19 45 64"), 850 | ("0.123", 0, "19 3c"), 851 | ("0.0123", 0, "03 2e"), 852 | ("0.00123", -1, "19 3c"), 853 | ("9223372036854775807", 10, "13 2d 43 91 07 89 6d 9b 75 0e"), 854 | ]; 855 | 856 | for (decimal, exponents, significand) in cases { 857 | let d = decimal.parse::().unwrap(); 858 | let (exp, sig) = Serializer::>::decimal_e_m(d); 859 | assert_eq!(exp, exponents, "wrong exponents for decimal: {decimal}"); 860 | assert_eq!( 861 | sig.iter() 862 | .map(|b| format!("{b:02x}")) 863 | .collect::>() 864 | .join(" "), 865 | significand, 866 | "wrong significand for decimal: {decimal}" 867 | ); 868 | } 869 | } 870 | 871 | #[test] 872 | fn test_reverse_order() { 873 | // Order: (ASC, DESC) 874 | let v1 = (0u8, 1i32); 875 | let v2 = (0u8, -1i32); 876 | let v3 = (1u8, -1i32); 877 | 878 | fn serialize(v: (u8, i32)) -> Vec { 879 | let mut ser = Serializer::new(vec![]); 880 | v.0.serialize(&mut ser).unwrap(); 881 | ser.set_reverse(true); 882 | v.1.serialize(&mut ser).unwrap(); 883 | ser.into_inner() 884 | } 885 | assert!(serialize(v1) < serialize(v2)); 886 | assert!(serialize(v2) < serialize(v3)); 887 | } 888 | } 889 | --------------------------------------------------------------------------------