├── .github └── workflows │ ├── CI.yml │ └── Release.yml ├── .gitignore ├── Cargo.toml ├── Readme.md ├── colfer-build ├── Cargo.toml └── src │ ├── ast.rs │ ├── config.rs │ ├── generator.rs │ ├── lib.rs │ └── parser.rs ├── example ├── Cargo.toml ├── bench.colf ├── bench.proto ├── build.rs ├── src │ ├── bench_colfer.rs │ ├── bench_pb.rs │ ├── gen.rs │ └── main.rs └── test.colf └── src ├── datetime.rs ├── lib.rs └── types.rs /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: {} 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v1 14 | - name: Build 15 | run: cargo build --all --verbose 16 | - name: Run tests 17 | run: cargo test --all --verbose 18 | -------------------------------------------------------------------------------- /.github/workflows/Release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - release 6 | paths: 7 | - '**/Cargo.toml' 8 | - '.github/workflows/release.yml' 9 | 10 | jobs: 11 | publish: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | fail-fast: false 15 | max-parallel: 1 16 | matrix: 17 | package: 18 | - name: colfer-rs 19 | registryName: colfer-rs 20 | path: . 21 | - name: colfer-build 22 | registryName: colfer-build 23 | path: colfer-build 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v2 27 | - name: get version 28 | working-directory: ${{ matrix.package.path }} 29 | run: echo PACKAGE_VERSION=$(sed -nE 's/^\s*version = "(.*?)"/\1/p' Cargo.toml) >> $GITHUB_ENV 30 | - name: check published version 31 | run: echo PUBLISHED_VERSION=$(cargo search ${{ matrix.package.registryName }} --limit 1 | sed -nE 's/^[^"]*"//; s/".*//1p' -) >> $GITHUB_ENV 32 | - name: cargo login 33 | if: env.PACKAGE_VERSION != env.PUBLISHED_VERSION 34 | run: cargo login ${{ secrets.CRATES_TOKEN }} 35 | - name: cargo package 36 | if: env.PACKAGE_VERSION != env.PUBLISHED_VERSION 37 | working-directory: ${{ matrix.package.path }} 38 | run: | 39 | cargo package 40 | echo "We will publish:" $PACKAGE_VERSION 41 | echo "This is current latest:" $PUBLISHED_VERSION 42 | - name: Publish ${{ matrix.package.name }} 43 | if: env.PACKAGE_VERSION != env.PUBLISHED_VERSION 44 | working-directory: ${{ matrix.package.path }} 45 | run: | 46 | echo "# Cargo Publish" 47 | cargo publish --no-verify 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "colfer" 3 | version = "0.1.0" 4 | authors = ["Sunli "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | byteorder = "1.3.4" 9 | chrono = { version = "0.4.19", optional = true } 10 | 11 | [dev-dependencies] 12 | colfer-build = { path = "colfer-build", version = "0.1.0" } 13 | 14 | [workspace] 15 | members = [ 16 | "colfer-build", 17 | "example" 18 | ] 19 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Colfer-rs 2 | 3 | `Colfer` is a binary serialization format optimized for speed and size, this crate is a Rust implementation of the [colfer](https://github.com/pascaldekloe/colfer). 4 | -------------------------------------------------------------------------------- /colfer-build/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "colfer-build" 3 | version = "0.1.0" 4 | authors = ["Sunli "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | nom = "6.0.1" 9 | case = "1.0.0" 10 | anyhow = "1.0.37" 11 | -------------------------------------------------------------------------------- /colfer-build/src/ast.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Eq, PartialEq)] 2 | pub enum FieldType { 3 | Bool, 4 | U8, 5 | U16, 6 | U32, 7 | U64, 8 | I32, 9 | I64, 10 | F32, 11 | F64, 12 | Timestamp, 13 | Text, 14 | Binary, 15 | Struct(String), 16 | ArrayF32, 17 | ArrayF64, 18 | ArrayText, 19 | ArrayBinary, 20 | ArrayStruct(String), 21 | } 22 | 23 | #[derive(Debug, Eq, PartialEq)] 24 | pub struct Field { 25 | pub name: String, 26 | pub ty: FieldType, 27 | } 28 | 29 | #[derive(Debug, Eq, PartialEq)] 30 | pub struct Struct { 31 | pub name: String, 32 | pub fields: Vec, 33 | } 34 | 35 | #[derive(Debug, Eq, PartialEq)] 36 | pub struct Colfer { 37 | pub package: String, 38 | pub structs: Vec, 39 | } 40 | 41 | impl Colfer { 42 | pub fn need_box(&self, start: &str, ty: &str) -> bool { 43 | if let Some(s) = self.structs.iter().find(|s| s.name == start) { 44 | for field in &s.fields { 45 | if let FieldType::Struct(struct_name) = &field.ty { 46 | if struct_name == start || self.need_box(&struct_name, ty) { 47 | return true; 48 | } 49 | } 50 | } 51 | } 52 | false 53 | } 54 | 55 | pub fn validate(&self) -> anyhow::Result<()> { 56 | self.validate_field_types()?; 57 | self.validate_fields_count()?; 58 | Ok(()) 59 | } 60 | 61 | fn validate_fields_count(&self) -> anyhow::Result<()> { 62 | for s in &self.structs { 63 | if s.fields.len() > 127 { 64 | anyhow::anyhow!("The maximum number of fields in a struct must be less than 128, but struct `{}` exceeds this limit.", s.name); 65 | } 66 | } 67 | Ok(()) 68 | } 69 | 70 | fn validate_field_types(&self) -> anyhow::Result<()> { 71 | for s in &self.structs { 72 | for f in &s.fields { 73 | if let FieldType::Struct(name) | FieldType::ArrayStruct(name) = &f.ty { 74 | if self.structs.iter().find(|s| &s.name == name).is_none() { 75 | anyhow::anyhow!("Struct `{}` is not defined.", name); 76 | } 77 | } 78 | } 79 | } 80 | Ok(()) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /colfer-build/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::path::{Path, PathBuf}; 2 | 3 | use case::CaseExt; 4 | 5 | use crate::generator::generate; 6 | use crate::parser::parse; 7 | 8 | pub struct Config { 9 | out_dir: PathBuf, 10 | } 11 | 12 | impl Default for Config { 13 | fn default() -> Self { 14 | Self { 15 | out_dir: std::env::var("OUT_DIR").unwrap().into(), 16 | } 17 | } 18 | } 19 | 20 | impl Config { 21 | pub fn out_dir(self, path: impl Into) -> Self { 22 | Self { 23 | out_dir: path.into(), 24 | } 25 | } 26 | 27 | pub fn compile>(self, files: &[P]) -> anyhow::Result<()> { 28 | for file in files { 29 | let source = std::fs::read_to_string(file)?; 30 | let colfer = parse(&source).map_err(|err| anyhow::anyhow!(err.to_string()))?; 31 | 32 | colfer.validate()?; 33 | std::fs::write( 34 | self.out_dir 35 | .join(colfer.package.to_snake()) 36 | .with_extension("rs"), 37 | generate(&colfer)?, 38 | )?; 39 | } 40 | 41 | Ok(()) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /colfer-build/src/generator.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Write; 2 | 3 | use crate::ast::{Colfer, FieldType}; 4 | 5 | pub fn generate(colfer: &Colfer) -> anyhow::Result { 6 | let mut code = String::new(); 7 | 8 | writeln!( 9 | &mut code, 10 | "#![allow(unused_variables, unused_assignments, unused_mut, unused_imports)]" 11 | )?; 12 | writeln!(&mut code)?; 13 | writeln!(&mut code, "use std::io::{{Write, Read, Result}};")?; 14 | writeln!(&mut code)?; 15 | writeln!(&mut code, "use colfer::{{Message, Type, DateTime}};")?; 16 | writeln!(&mut code)?; 17 | 18 | for s in &colfer.structs { 19 | writeln!(&mut code, "#[derive(Default, Clone, Debug, PartialEq)]")?; 20 | writeln!(&mut code, "pub struct {} {{", s.name)?; 21 | 22 | for f in &s.fields { 23 | write!(&mut code, "\tpub {}: ", f.name)?; 24 | 25 | match &f.ty { 26 | FieldType::Bool => write!(&mut code, "bool")?, 27 | FieldType::U8 => write!(&mut code, "u8")?, 28 | FieldType::U16 => write!(&mut code, "u16")?, 29 | FieldType::U32 => write!(&mut code, "u32")?, 30 | FieldType::U64 => write!(&mut code, "u64")?, 31 | FieldType::I32 => write!(&mut code, "i32")?, 32 | FieldType::I64 => write!(&mut code, "i64")?, 33 | FieldType::F32 => write!(&mut code, "f32")?, 34 | FieldType::F64 => write!(&mut code, "f64")?, 35 | FieldType::Timestamp => write!(&mut code, "DateTime")?, 36 | FieldType::Text => write!(&mut code, "String")?, 37 | FieldType::Binary => write!(&mut code, "Vec")?, 38 | FieldType::Struct(name) => { 39 | if colfer.need_box(&s.name, &name) { 40 | write!(&mut code, "Option>", name)?; 41 | } else { 42 | write!(&mut code, "Option<{}>", name)?; 43 | } 44 | } 45 | FieldType::ArrayF32 => write!(&mut code, "Vec")?, 46 | FieldType::ArrayF64 => write!(&mut code, "Vec")?, 47 | FieldType::ArrayText => write!(&mut code, "Vec")?, 48 | FieldType::ArrayBinary => write!(&mut code, "Vec>")?, 49 | FieldType::ArrayStruct(name) => write!(&mut code, "Vec<{}>", name)?, 50 | } 51 | 52 | writeln!(&mut code, ",")?; 53 | } 54 | 55 | writeln!(&mut code, "}}")?; 56 | 57 | writeln!(&mut code)?; 58 | writeln!(&mut code, "impl Message for {} {{", s.name)?; 59 | 60 | writeln!(&mut code, "\t#[inline]")?; 61 | writeln!( 62 | &mut code, 63 | "\tfn encode(&self, w: &mut W) -> Result<()> {{" 64 | )?; 65 | for (idx, f) in s.fields.iter().enumerate() { 66 | match &f.ty { 67 | FieldType::Struct(_) => writeln!( 68 | &mut code, 69 | "\t\tcolfer::encode_message(w, {}, self.{}.as_deref())?;", 70 | idx, f.name 71 | )?, 72 | FieldType::ArrayStruct(_) => writeln!( 73 | &mut code, 74 | "\t\tcolfer::encode_messages(w, {}, &self.{})?;", 75 | idx, f.name 76 | )?, 77 | _ => writeln!(&mut code, "\t\tself.{}.encode(w, {})?;", f.name, idx)?, 78 | } 79 | } 80 | writeln!(&mut code, "\t\tcolfer::write_end(w)?;",)?; 81 | writeln!(&mut code)?; 82 | writeln!(&mut code, "\t\tOk(())\n\t}}")?; 83 | writeln!(&mut code)?; 84 | 85 | writeln!(&mut code, "\t#[inline]")?; 86 | writeln!( 87 | &mut code, 88 | "\tfn decode(r: &mut R) -> Result {{" 89 | )?; 90 | writeln!(&mut code, "\t\tlet mut obj = Self::default();")?; 91 | writeln!( 92 | &mut code, 93 | "\t\tlet (mut id, mut flag) = colfer::read_header(r)?;" 94 | )?; 95 | for (idx, f) in s.fields.iter().enumerate() { 96 | writeln!(&mut code, "\t\tif id == {} {{", idx)?; 97 | match &f.ty { 98 | FieldType::Struct(_) => writeln!( 99 | &mut code, 100 | "\t\t\tobj.{} = colfer::decode_message(r)?;", 101 | f.name, 102 | )?, 103 | FieldType::ArrayStruct(_) => writeln!( 104 | &mut code, 105 | "\t\t\tobj.{} = colfer::decode_messages(r)?;", 106 | f.name, 107 | )?, 108 | _ => writeln!(&mut code, "\t\t\tobj.{} = Type::decode(r, flag)?;", f.name)?, 109 | } 110 | if idx < s.fields.len() - 1 { 111 | writeln!(&mut code, "\t\t\tlet next = colfer::read_header(r)?;")?; 112 | writeln!(&mut code, "\t\t\tid = next.0;")?; 113 | writeln!(&mut code, "\t\t\tflag = next.1;")?; 114 | } 115 | writeln!(&mut code, "\t\t}}")?; 116 | } 117 | writeln!(&mut code)?; 118 | 119 | writeln!(&mut code, "\t\tOk(obj)\n\t}}")?; 120 | 121 | writeln!(&mut code)?; 122 | writeln!(&mut code, "\t#[inline]")?; 123 | writeln!(&mut code, "\tfn size(&self) -> usize {{")?; 124 | writeln!(&mut code, "\t\tlet mut size = 0;")?; 125 | for f in &s.fields { 126 | match &f.ty { 127 | FieldType::Struct(_) => { 128 | writeln!( 129 | &mut code, 130 | "\t\tsize += self.{}.as_ref().map(|s| s.size()).unwrap_or_default();", 131 | f.name 132 | )?; 133 | } 134 | FieldType::ArrayStruct(_) => { 135 | writeln!( 136 | &mut code, 137 | "\t\tsize += self.{}.iter().map(|s| s.size()).sum::();", 138 | f.name 139 | )?; 140 | } 141 | _ => { 142 | writeln!(&mut code, "\t\tsize += self.{}.size();", f.name)?; 143 | } 144 | } 145 | } 146 | writeln!(&mut code, "\t\tsize")?; 147 | writeln!(&mut code, "\t}}")?; 148 | 149 | writeln!(&mut code, "}}")?; 150 | writeln!(&mut code)?; 151 | } 152 | 153 | Ok(code) 154 | } 155 | -------------------------------------------------------------------------------- /colfer-build/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod ast; 2 | mod config; 3 | mod generator; 4 | mod parser; 5 | 6 | pub use config::Config; 7 | -------------------------------------------------------------------------------- /colfer-build/src/parser.rs: -------------------------------------------------------------------------------- 1 | use case::CaseExt; 2 | use nom::branch::alt; 3 | use nom::bytes::complete::{is_not, tag}; 4 | use nom::character::complete::{alpha1, alphanumeric1, one_of}; 5 | use nom::combinator::{eof, map, recognize, value}; 6 | use nom::error::{context, ContextError}; 7 | use nom::error::{ParseError, VerboseError}; 8 | use nom::multi::{fold_many0, many0, many1}; 9 | use nom::sequence::{delimited, pair, preceded, terminated, tuple}; 10 | use nom::IResult; 11 | 12 | use crate::ast::{Colfer, Field, FieldType, Struct}; 13 | 14 | fn line_comment<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 15 | input: &'a str, 16 | ) -> IResult<&'a str, &'a str, E> { 17 | preceded(tag("//"), is_not("\n\r"))(input) 18 | } 19 | 20 | fn sp<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 21 | input: &'a str, 22 | ) -> IResult<&'a str, (), E> { 23 | fold_many0( 24 | alt((value((), one_of(" \t\n\r")), value((), line_comment))), 25 | (), 26 | |_, _| (), 27 | )(input) 28 | } 29 | 30 | fn ident<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 31 | input: &'a str, 32 | ) -> IResult<&'a str, &'a str, E> { 33 | context( 34 | "ident", 35 | recognize(pair( 36 | alt((alpha1, tag("_"))), 37 | many0(alt((alphanumeric1, tag("_")))), 38 | )), 39 | )(input) 40 | } 41 | 42 | fn package<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 43 | input: &'a str, 44 | ) -> IResult<&'a str, String, E> { 45 | context( 46 | "package", 47 | preceded( 48 | pair(tag("package"), sp), 49 | terminated(map(ident, |ident| ident.to_snake()), sp), 50 | ), 51 | )(input) 52 | } 53 | 54 | fn array_type<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 55 | input: &'a str, 56 | ) -> IResult<&'a str, FieldType, E> { 57 | let f32_ = map(tag("float32"), |_| FieldType::ArrayF32); 58 | let f64_ = map(tag("float64"), |_| FieldType::ArrayF64); 59 | let text_ = map(tag("text"), |_| FieldType::ArrayText); 60 | let binary_ = map(tag("binary"), |_| FieldType::ArrayBinary); 61 | let s = map(ident, |name| FieldType::ArrayStruct(name.to_camel())); 62 | 63 | context("array-type", alt((f32_, f64_, text_, binary_, s)))(input) 64 | } 65 | 66 | fn field_type<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 67 | input: &'a str, 68 | ) -> IResult<&'a str, FieldType, E> { 69 | let bool_ = map(tag("bool"), |_| FieldType::Bool); 70 | let u8_ = map(tag("uint8"), |_| FieldType::U8); 71 | let u16_ = map(tag("uint16"), |_| FieldType::U16); 72 | let u32_ = map(tag("uint32"), |_| FieldType::U32); 73 | let u64_ = map(tag("uint64"), |_| FieldType::U64); 74 | let i32_ = map(tag("int32"), |_| FieldType::I32); 75 | let i64_ = map(tag("int64"), |_| FieldType::I64); 76 | let f32_ = map(tag("float32"), |_| FieldType::F32); 77 | let f64_ = map(tag("float64"), |_| FieldType::F64); 78 | let timestamp_ = map(tag("timestamp"), |_| FieldType::Timestamp); 79 | let text_ = map(tag("text"), |_| FieldType::Text); 80 | let binary_ = map(tag("binary"), |_| FieldType::Binary); 81 | 82 | let array_ = preceded(tuple((tag("["), sp, tag("]"), sp)), array_type); 83 | let s = map(ident, |name| FieldType::Struct(name.to_camel())); 84 | 85 | context( 86 | "field-type", 87 | alt(( 88 | bool_, u8_, u16_, u32_, u64_, i32_, i64_, f32_, f64_, timestamp_, text_, binary_, 89 | array_, s, 90 | )), 91 | )(input) 92 | } 93 | 94 | fn type_struct<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 95 | input: &'a str, 96 | ) -> IResult<&'a str, String, E> { 97 | terminated( 98 | preceded(pair(tag("type"), sp), map(ident, |ident| ident.to_camel())), 99 | pair(sp, tag("struct")), 100 | )(input) 101 | } 102 | 103 | fn field_def<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 104 | input: &'a str, 105 | ) -> IResult<&'a str, Field, E> { 106 | delimited( 107 | sp, 108 | map(tuple((ident, sp, field_type)), |(name, _, ty)| Field { 109 | name: { 110 | let mut name = name.to_snake(); 111 | match name.as_str() { 112 | // 2015 strict keywords. 113 | | "as" | "break" | "const" | "continue" | "else" | "enum" | "false" 114 | | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" | "match" | "mod" | "move" | "mut" 115 | | "pub" | "ref" | "return" | "static" | "struct" | "trait" | "true" 116 | | "type" | "unsafe" | "use" | "where" | "while" 117 | // 2018 strict keywords. 118 | | "dyn" 119 | // 2015 reserved keywords. 120 | | "abstract" | "become" | "box" | "do" | "final" | "macro" | "override" | "priv" | "typeof" 121 | | "unsized" | "virtual" | "yield" 122 | // 2018 reserved keywords. 123 | | "async" | "await" | "try" => name.insert_str(0, "r#"), 124 | // the following keywords are not supported as raw identifiers and are therefore suffixed with an underscore. 125 | "self" | "super" | "extern" | "crate" => name += "_", 126 | _ => (), 127 | } 128 | name 129 | }, 130 | ty, 131 | }), 132 | sp, 133 | )(input) 134 | } 135 | 136 | fn struct_def<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 137 | input: &'a str, 138 | ) -> IResult<&'a str, Struct, E> { 139 | let fields = many1(delimited(sp, field_def, sp)); 140 | let body = delimited(tag("{"), fields, tag("}")); 141 | 142 | map(tuple((type_struct, sp, body)), |(name, _, fields)| Struct { 143 | name, 144 | fields, 145 | })(input) 146 | } 147 | 148 | fn colfer<'a, E: ParseError<&'a str> + ContextError<&'a str>>( 149 | input: &'a str, 150 | ) -> IResult<&'a str, Colfer, E> { 151 | let package = delimited(sp, package, sp); 152 | let structs = many1(delimited(sp, struct_def, sp)); 153 | context( 154 | "colfer", 155 | map(tuple((package, structs, eof)), |(package, structs, _)| { 156 | Colfer { package, structs } 157 | }), 158 | )(input) 159 | } 160 | 161 | pub fn parse(input: &str) -> Result>> { 162 | colfer::>(input).map(|(_, colfer)| colfer) 163 | } 164 | 165 | #[cfg(test)] 166 | mod tests { 167 | use super::*; 168 | use nom::error::VerboseError; 169 | 170 | #[test] 171 | fn test_package() { 172 | assert_eq!( 173 | package::>("package MyPkg"), 174 | Ok(("", "my_pkg".to_string())) 175 | ); 176 | assert_eq!( 177 | package::>("package MyPkg"), 178 | Ok(("", "my_pkg".to_string())) 179 | ); 180 | } 181 | 182 | #[test] 183 | fn test_field_type() { 184 | assert_eq!( 185 | field_type::>("bool"), 186 | Ok(("", FieldType::Bool)) 187 | ); 188 | assert_eq!( 189 | field_type::>("uint8"), 190 | Ok(("", FieldType::U8)) 191 | ); 192 | assert_eq!( 193 | field_type::>("uint16"), 194 | Ok(("", FieldType::U16)) 195 | ); 196 | assert_eq!( 197 | field_type::>("uint32"), 198 | Ok(("", FieldType::U32)) 199 | ); 200 | assert_eq!( 201 | field_type::>("uint64"), 202 | Ok(("", FieldType::U64)) 203 | ); 204 | assert_eq!( 205 | field_type::>("int32"), 206 | Ok(("", FieldType::I32)) 207 | ); 208 | assert_eq!( 209 | field_type::>("int64"), 210 | Ok(("", FieldType::I64)) 211 | ); 212 | assert_eq!( 213 | field_type::>("float32"), 214 | Ok(("", FieldType::F32)) 215 | ); 216 | assert_eq!( 217 | field_type::>("float64"), 218 | Ok(("", FieldType::F64)) 219 | ); 220 | assert_eq!( 221 | field_type::>("timestamp"), 222 | Ok(("", FieldType::Timestamp)) 223 | ); 224 | assert_eq!( 225 | field_type::>("text"), 226 | Ok(("", FieldType::Text)) 227 | ); 228 | assert_eq!( 229 | field_type::>("binary"), 230 | Ok(("", FieldType::Binary)) 231 | ); 232 | assert_eq!( 233 | field_type::>("abc"), 234 | Ok(("", FieldType::Struct("Abc".to_string()))) 235 | ); 236 | 237 | assert_eq!( 238 | field_type::>("[]float32"), 239 | Ok(("", FieldType::ArrayF32)) 240 | ); 241 | assert_eq!( 242 | field_type::>("[ ] float32"), 243 | Ok(("", FieldType::ArrayF32)) 244 | ); 245 | assert_eq!( 246 | field_type::>("[] float32"), 247 | Ok(("", FieldType::ArrayF32)) 248 | ); 249 | } 250 | 251 | #[test] 252 | fn test_type_struct() { 253 | assert_eq!( 254 | type_struct::>("type Abc struct"), 255 | Ok(("", "Abc".to_string())) 256 | ); 257 | assert_eq!( 258 | type_struct::>("type Abc struct"), 259 | Ok(("", "Abc".to_string())) 260 | ); 261 | } 262 | 263 | #[test] 264 | fn test_field_def() { 265 | assert_eq!( 266 | field_def::>("abc int32"), 267 | Ok(( 268 | "", 269 | Field { 270 | name: "abc".to_string(), 271 | ty: FieldType::I32 272 | } 273 | )) 274 | ); 275 | 276 | assert_eq!( 277 | field_def::>("abc int32"), 278 | Ok(( 279 | "", 280 | Field { 281 | name: "abc".to_string(), 282 | ty: FieldType::I32 283 | } 284 | )) 285 | ); 286 | } 287 | 288 | #[test] 289 | fn test_struct_def() { 290 | assert_eq!( 291 | struct_def::>( 292 | r#"type Abc struct { 293 | value1 int32 294 | value2 bool 295 | }"# 296 | ), 297 | Ok(( 298 | "", 299 | Struct { 300 | name: "Abc".to_string(), 301 | fields: vec![ 302 | Field { 303 | name: "value1".to_string(), 304 | ty: FieldType::I32 305 | }, 306 | Field { 307 | name: "value2".to_string(), 308 | ty: FieldType::Bool 309 | } 310 | ] 311 | } 312 | )) 313 | ) 314 | } 315 | 316 | #[test] 317 | fn test_comment() { 318 | assert_eq!(line_comment::>("//abc"), Ok(("", "abc"))); 319 | } 320 | 321 | #[test] 322 | fn test_sp() { 323 | assert_eq!(sp::>("//abc"), Ok(("", ()))); 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /example/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "example" 3 | version = "0.1.0" 4 | authors = ["Sunli "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | chrono = "0.4.19" 10 | colfer = { path = "..", version = "0.1.0" } 11 | prost = "0.7.0" 12 | 13 | [build-dependencies] 14 | colfer-build = { path = "../colfer-build", version = "0.1.0" } 15 | prost-build = "0.7.0" 16 | -------------------------------------------------------------------------------- /example/bench.colf: -------------------------------------------------------------------------------- 1 | package bench_colfer 2 | 3 | type Colfer struct { 4 | key int64 5 | host text 6 | port uint16 7 | size int64 8 | hash uint64 9 | ratio float64 10 | route bool 11 | } -------------------------------------------------------------------------------- /example/bench.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package bench_pb; 4 | 5 | message Colfer { 6 | int64 key = 1; 7 | string host = 2; 8 | uint32 port = 4; 9 | int64 size = 5; 10 | fixed64 hash = 6; 11 | double ratio = 7; 12 | bool route = 8; 13 | } -------------------------------------------------------------------------------- /example/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | colfer_build::Config::default() 3 | .out_dir("./src") 4 | .compile(&["test.colf", "bench.colf"]) 5 | .unwrap(); 6 | 7 | prost_build::Config::default() 8 | .out_dir("./src") 9 | .compile_protos(&["bench.proto"], &["./"]) 10 | .unwrap(); 11 | } 12 | -------------------------------------------------------------------------------- /example/src/bench_colfer.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_variables, unused_assignments, unused_mut, unused_imports)] 2 | 3 | use std::io::{Write, Read, Result}; 4 | 5 | use colfer::{Message, Type, DateTime}; 6 | 7 | #[derive(Default, Clone, Debug, PartialEq)] 8 | pub struct Colfer { 9 | pub key: i64, 10 | pub host: String, 11 | pub port: u16, 12 | pub size: i64, 13 | pub hash: u64, 14 | pub ratio: f64, 15 | pub route: bool, 16 | } 17 | 18 | impl Message for Colfer { 19 | #[inline] 20 | fn encode(&self, w: &mut W) -> Result<()> { 21 | self.key.encode(w, 0)?; 22 | self.host.encode(w, 1)?; 23 | self.port.encode(w, 2)?; 24 | self.size.encode(w, 3)?; 25 | self.hash.encode(w, 4)?; 26 | self.ratio.encode(w, 5)?; 27 | self.route.encode(w, 6)?; 28 | colfer::write_end(w)?; 29 | 30 | Ok(()) 31 | } 32 | 33 | #[inline] 34 | fn decode(r: &mut R) -> Result { 35 | let mut obj = Self::default(); 36 | let (mut id, mut flag) = colfer::read_header(r)?; 37 | if id == 0 { 38 | obj.key = Type::decode(r, flag)?; 39 | let next = colfer::read_header(r)?; 40 | id = next.0; 41 | flag = next.1; 42 | } 43 | if id == 1 { 44 | obj.host = Type::decode(r, flag)?; 45 | let next = colfer::read_header(r)?; 46 | id = next.0; 47 | flag = next.1; 48 | } 49 | if id == 2 { 50 | obj.port = Type::decode(r, flag)?; 51 | let next = colfer::read_header(r)?; 52 | id = next.0; 53 | flag = next.1; 54 | } 55 | if id == 3 { 56 | obj.size = Type::decode(r, flag)?; 57 | let next = colfer::read_header(r)?; 58 | id = next.0; 59 | flag = next.1; 60 | } 61 | if id == 4 { 62 | obj.hash = Type::decode(r, flag)?; 63 | let next = colfer::read_header(r)?; 64 | id = next.0; 65 | flag = next.1; 66 | } 67 | if id == 5 { 68 | obj.ratio = Type::decode(r, flag)?; 69 | let next = colfer::read_header(r)?; 70 | id = next.0; 71 | flag = next.1; 72 | } 73 | if id == 6 { 74 | obj.route = Type::decode(r, flag)?; 75 | } 76 | 77 | Ok(obj) 78 | } 79 | 80 | #[inline] 81 | fn size(&self) -> usize { 82 | let mut size = 0; 83 | size += self.key.size(); 84 | size += self.host.size(); 85 | size += self.port.size(); 86 | size += self.size.size(); 87 | size += self.hash.size(); 88 | size += self.ratio.size(); 89 | size += self.route.size(); 90 | size 91 | } 92 | } 93 | 94 | -------------------------------------------------------------------------------- /example/src/bench_pb.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone, PartialEq, ::prost::Message)] 2 | pub struct Colfer { 3 | #[prost(int64, tag="1")] 4 | pub key: i64, 5 | #[prost(string, tag="2")] 6 | pub host: ::prost::alloc::string::String, 7 | #[prost(uint32, tag="4")] 8 | pub port: u32, 9 | #[prost(int64, tag="5")] 10 | pub size: i64, 11 | #[prost(fixed64, tag="6")] 12 | pub hash: u64, 13 | #[prost(double, tag="7")] 14 | pub ratio: f64, 15 | #[prost(bool, tag="8")] 16 | pub route: bool, 17 | } 18 | -------------------------------------------------------------------------------- /example/src/gen.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_variables, unused_assignments, unused_mut, unused_imports)] 2 | 3 | use std::io::{Write, Read, Result}; 4 | 5 | use colfer::{Message, Type, DateTime}; 6 | 7 | #[derive(Default, Clone, Debug, PartialEq)] 8 | pub struct O { 9 | pub b: bool, 10 | pub u32: u32, 11 | pub u64: u64, 12 | pub i32: i32, 13 | pub i64: i64, 14 | pub f32: f32, 15 | pub f64: f64, 16 | pub t: DateTime, 17 | pub s: String, 18 | pub a: Vec, 19 | pub o: Option>, 20 | pub os: Vec, 21 | pub ss: Vec, 22 | pub r#as: Vec>, 23 | pub u8: u8, 24 | pub u16: u16, 25 | pub f32s: Vec, 26 | pub f64s: Vec, 27 | } 28 | 29 | impl Message for O { 30 | #[inline] 31 | fn encode(&self, w: &mut W) -> Result<()> { 32 | self.b.encode(w, 0)?; 33 | self.u32.encode(w, 1)?; 34 | self.u64.encode(w, 2)?; 35 | self.i32.encode(w, 3)?; 36 | self.i64.encode(w, 4)?; 37 | self.f32.encode(w, 5)?; 38 | self.f64.encode(w, 6)?; 39 | self.t.encode(w, 7)?; 40 | self.s.encode(w, 8)?; 41 | self.a.encode(w, 9)?; 42 | colfer::encode_message(w, 10, self.o.as_deref())?; 43 | colfer::encode_messages(w, 11, &self.os)?; 44 | self.ss.encode(w, 12)?; 45 | self.r#as.encode(w, 13)?; 46 | self.u8.encode(w, 14)?; 47 | self.u16.encode(w, 15)?; 48 | self.f32s.encode(w, 16)?; 49 | self.f64s.encode(w, 17)?; 50 | colfer::write_end(w)?; 51 | 52 | Ok(()) 53 | } 54 | 55 | #[inline] 56 | fn decode(r: &mut R) -> Result { 57 | let mut obj = Self::default(); 58 | let (mut id, mut flag) = colfer::read_header(r)?; 59 | if id == 0 { 60 | obj.b = Type::decode(r, flag)?; 61 | let next = colfer::read_header(r)?; 62 | id = next.0; 63 | flag = next.1; 64 | } 65 | if id == 1 { 66 | obj.u32 = Type::decode(r, flag)?; 67 | let next = colfer::read_header(r)?; 68 | id = next.0; 69 | flag = next.1; 70 | } 71 | if id == 2 { 72 | obj.u64 = Type::decode(r, flag)?; 73 | let next = colfer::read_header(r)?; 74 | id = next.0; 75 | flag = next.1; 76 | } 77 | if id == 3 { 78 | obj.i32 = Type::decode(r, flag)?; 79 | let next = colfer::read_header(r)?; 80 | id = next.0; 81 | flag = next.1; 82 | } 83 | if id == 4 { 84 | obj.i64 = Type::decode(r, flag)?; 85 | let next = colfer::read_header(r)?; 86 | id = next.0; 87 | flag = next.1; 88 | } 89 | if id == 5 { 90 | obj.f32 = Type::decode(r, flag)?; 91 | let next = colfer::read_header(r)?; 92 | id = next.0; 93 | flag = next.1; 94 | } 95 | if id == 6 { 96 | obj.f64 = Type::decode(r, flag)?; 97 | let next = colfer::read_header(r)?; 98 | id = next.0; 99 | flag = next.1; 100 | } 101 | if id == 7 { 102 | obj.t = Type::decode(r, flag)?; 103 | let next = colfer::read_header(r)?; 104 | id = next.0; 105 | flag = next.1; 106 | } 107 | if id == 8 { 108 | obj.s = Type::decode(r, flag)?; 109 | let next = colfer::read_header(r)?; 110 | id = next.0; 111 | flag = next.1; 112 | } 113 | if id == 9 { 114 | obj.a = Type::decode(r, flag)?; 115 | let next = colfer::read_header(r)?; 116 | id = next.0; 117 | flag = next.1; 118 | } 119 | if id == 10 { 120 | obj.o = colfer::decode_message(r)?; 121 | let next = colfer::read_header(r)?; 122 | id = next.0; 123 | flag = next.1; 124 | } 125 | if id == 11 { 126 | obj.os = colfer::decode_messages(r)?; 127 | let next = colfer::read_header(r)?; 128 | id = next.0; 129 | flag = next.1; 130 | } 131 | if id == 12 { 132 | obj.ss = Type::decode(r, flag)?; 133 | let next = colfer::read_header(r)?; 134 | id = next.0; 135 | flag = next.1; 136 | } 137 | if id == 13 { 138 | obj.r#as = Type::decode(r, flag)?; 139 | let next = colfer::read_header(r)?; 140 | id = next.0; 141 | flag = next.1; 142 | } 143 | if id == 14 { 144 | obj.u8 = Type::decode(r, flag)?; 145 | let next = colfer::read_header(r)?; 146 | id = next.0; 147 | flag = next.1; 148 | } 149 | if id == 15 { 150 | obj.u16 = Type::decode(r, flag)?; 151 | let next = colfer::read_header(r)?; 152 | id = next.0; 153 | flag = next.1; 154 | } 155 | if id == 16 { 156 | obj.f32s = Type::decode(r, flag)?; 157 | let next = colfer::read_header(r)?; 158 | id = next.0; 159 | flag = next.1; 160 | } 161 | if id == 17 { 162 | obj.f64s = Type::decode(r, flag)?; 163 | } 164 | 165 | Ok(obj) 166 | } 167 | 168 | #[inline] 169 | fn size(&self) -> usize { 170 | let mut size = 0; 171 | size += self.b.size(); 172 | size += self.u32.size(); 173 | size += self.u64.size(); 174 | size += self.i32.size(); 175 | size += self.i64.size(); 176 | size += self.f32.size(); 177 | size += self.f64.size(); 178 | size += self.t.size(); 179 | size += self.s.size(); 180 | size += self.a.size(); 181 | size += self.o.as_ref().map(|s| s.size()).unwrap_or_default(); 182 | size += self.os.iter().map(|s| s.size()).sum::(); 183 | size += self.ss.size(); 184 | size += self.r#as.size(); 185 | size += self.u8.size(); 186 | size += self.u16.size(); 187 | size += self.f32s.size(); 188 | size += self.f64s.size(); 189 | size 190 | } 191 | } 192 | 193 | #[derive(Default, Clone, Debug, PartialEq)] 194 | pub struct DromedaryCase { 195 | pub pascal_case: String, 196 | } 197 | 198 | impl Message for DromedaryCase { 199 | #[inline] 200 | fn encode(&self, w: &mut W) -> Result<()> { 201 | self.pascal_case.encode(w, 0)?; 202 | colfer::write_end(w)?; 203 | 204 | Ok(()) 205 | } 206 | 207 | #[inline] 208 | fn decode(r: &mut R) -> Result { 209 | let mut obj = Self::default(); 210 | let (mut id, mut flag) = colfer::read_header(r)?; 211 | if id == 0 { 212 | obj.pascal_case = Type::decode(r, flag)?; 213 | } 214 | 215 | Ok(obj) 216 | } 217 | 218 | #[inline] 219 | fn size(&self) -> usize { 220 | let mut size = 0; 221 | size += self.pascal_case.size(); 222 | size 223 | } 224 | } 225 | 226 | #[derive(Default, Clone, Debug, PartialEq)] 227 | pub struct EmbedO { 228 | pub inner: Option>, 229 | } 230 | 231 | impl Message for EmbedO { 232 | #[inline] 233 | fn encode(&self, w: &mut W) -> Result<()> { 234 | colfer::encode_message(w, 0, self.inner.as_deref())?; 235 | colfer::write_end(w)?; 236 | 237 | Ok(()) 238 | } 239 | 240 | #[inline] 241 | fn decode(r: &mut R) -> Result { 242 | let mut obj = Self::default(); 243 | let (mut id, mut flag) = colfer::read_header(r)?; 244 | if id == 0 { 245 | obj.inner = colfer::decode_message(r)?; 246 | } 247 | 248 | Ok(obj) 249 | } 250 | 251 | #[inline] 252 | fn size(&self) -> usize { 253 | let mut size = 0; 254 | size += self.inner.as_ref().map(|s| s.size()).unwrap_or_default(); 255 | size 256 | } 257 | } 258 | 259 | -------------------------------------------------------------------------------- /example/src/main.rs: -------------------------------------------------------------------------------- 1 | use colfer::Message; 2 | 3 | mod bench_colfer; 4 | mod bench_pb; 5 | mod gen; 6 | 7 | use prost::bytes::Bytes; 8 | use std::io::Cursor; 9 | use std::time::Instant; 10 | 11 | fn bench_colfer() { 12 | use bench_colfer::Colfer; 13 | 14 | let test_data = vec![ 15 | Colfer { 16 | key: 1234567890, 17 | host: "db003lz12".to_string(), 18 | port: 389, 19 | size: 452, 20 | hash: 0x488b5c2428488918, 21 | ratio: 0.99, 22 | route: true, 23 | }, 24 | Colfer { 25 | key: 1234567891, 26 | host: "localhost".to_string(), 27 | port: 22, 28 | size: 4096, 29 | hash: 0x243048899c24c824, 30 | ratio: 0.20, 31 | route: false, 32 | }, 33 | Colfer { 34 | key: 1234567892, 35 | host: "kdc.local".to_string(), 36 | port: 88, 37 | size: 1984, 38 | hash: 0x000048891c24485c, 39 | ratio: 0.06, 40 | route: false, 41 | }, 42 | Colfer { 43 | key: 1234567893, 44 | host: "vhost8.dmz.example.com".to_string(), 45 | port: 27017, 46 | size: 59741, 47 | hash: 0x5c2408488b9c2489, 48 | ratio: 0.0, 49 | route: true, 50 | }, 51 | ]; 52 | 53 | let s = Instant::now(); 54 | let count = 1000000; 55 | let mut data = Vec::new(); 56 | 57 | for _ in 0..count { 58 | for c in &test_data { 59 | c.encode(&mut data).unwrap(); 60 | } 61 | } 62 | println!( 63 | "COLFER encode: {:.03}s size: {}", 64 | (Instant::now() - s).as_secs_f32(), 65 | data.len() 66 | ); 67 | 68 | let s = Instant::now(); 69 | let mut r = Cursor::new(data); 70 | for _ in 0..count { 71 | for _ in 0..4 { 72 | Colfer::decode(&mut r).unwrap(); 73 | } 74 | } 75 | println!("COLFER decode: {:.03}s", (Instant::now() - s).as_secs_f32()); 76 | } 77 | 78 | fn bench_pb() { 79 | use bench_pb::Colfer; 80 | use prost::Message; 81 | 82 | let test_data = vec![ 83 | Colfer { 84 | key: 1234567890, 85 | host: "db003lz12".to_string(), 86 | port: 389, 87 | size: 452, 88 | hash: 0x488b5c2428488918, 89 | ratio: 0.99, 90 | route: true, 91 | }, 92 | Colfer { 93 | key: 1234567891, 94 | host: "localhost".to_string(), 95 | port: 22, 96 | size: 4096, 97 | hash: 0x243048899c24c824, 98 | ratio: 0.20, 99 | route: false, 100 | }, 101 | Colfer { 102 | key: 1234567892, 103 | host: "kdc.local".to_string(), 104 | port: 88, 105 | size: 1984, 106 | hash: 0x000048891c24485c, 107 | ratio: 0.06, 108 | route: false, 109 | }, 110 | Colfer { 111 | key: 1234567893, 112 | host: "vhost8.dmz.example.com".to_string(), 113 | port: 27017, 114 | size: 59741, 115 | hash: 0x5c2408488b9c2489, 116 | ratio: 0.0, 117 | route: true, 118 | }, 119 | ]; 120 | 121 | let s = Instant::now(); 122 | let count = 1000000; 123 | let mut data: Vec = Vec::new(); 124 | 125 | for _ in 0..count { 126 | for c in &test_data { 127 | c.encode(&mut data).unwrap(); 128 | } 129 | } 130 | println!( 131 | "Protobuf encode: {:.03}s size: {}", 132 | (Instant::now() - s).as_secs_f32(), 133 | data.len() 134 | ); 135 | 136 | let s = Instant::now(); 137 | let mut buf = Bytes::from(data); 138 | for _ in 0..count { 139 | for _ in 0..4 { 140 | Colfer::decode(&mut buf).unwrap(); 141 | } 142 | } 143 | println!( 144 | "Protobuf decode: {:.03}s", 145 | (Instant::now() - s).as_secs_f32() 146 | ); 147 | } 148 | 149 | fn main() { 150 | bench_colfer(); 151 | bench_pb(); 152 | } 153 | -------------------------------------------------------------------------------- /example/test.colf: -------------------------------------------------------------------------------- 1 | // Package gen tests all field mapping options. 2 | package gen 3 | 4 | // O contains all supported data types. 5 | type o struct { 6 | // B tests booleans. 7 | b bool 8 | // U32 tests unsigned 32-bit integers. 9 | u32 uint32 10 | // U64 tests unsigned 64-bit integers. 11 | u64 uint64 12 | // I32 tests signed 32-bit integers. 13 | i32 int32 14 | // I64 tests signed 64-bit integers. 15 | i64 int64 16 | // F32 tests 32-bit floating points. 17 | f32 float32 18 | // F64 tests 64-bit floating points. 19 | f64 float64 20 | // T tests timestamps. 21 | t timestamp 22 | // S tests text. 23 | s text 24 | // A tests binaries. 25 | a binary 26 | // O tests nested data structures. 27 | o o 28 | // Os tests data structure lists. 29 | os []o 30 | // Ss tests text lists. 31 | ss []text 32 | // As tests binary lists. 33 | as []binary 34 | // U8 tests unsigned 8-bit integers. 35 | u8 uint8 36 | // U16 tests unsigned 16-bit integers. 37 | u16 uint16 38 | // F32s tests 32-bit floating point lists. 39 | f32s []float32 40 | // F64s tests 64-bit floating point lists. 41 | f64s []float64 42 | } 43 | 44 | // DromedaryCase oposes name casings. 45 | type dromedaryCase struct { 46 | PascalCase text 47 | } 48 | 49 | // EmbedO has an inner object only. 50 | // Covers regression of issue #66. 51 | type EmbedO struct { 52 | inner o 53 | } -------------------------------------------------------------------------------- /src/datetime.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "chrono")] 2 | use chrono::{DateTime as ChronoDateTime, FixedOffset, Local, TimeZone, Utc}; 3 | 4 | /// A datetime type. 5 | /// 6 | /// If the `chrono` feature is enabled, then `From>` and `Into>` are implemented for it. 7 | #[derive(Default, Copy, Clone, Eq, PartialEq, Debug)] 8 | pub struct DateTime { 9 | /// Number of non-leap-milliseconds since January 1, 1970 UTC. 10 | pub seconds: i64, 11 | 12 | /// Number of nanoseconds since the last second boundary. 13 | pub nano_seconds: u32, 14 | } 15 | 16 | #[cfg(feature = "chrono")] 17 | impl From for ChronoDateTime { 18 | fn from(t: DateTime) -> Self { 19 | Utc.timestamp(t.seconds, t.nano_seconds) 20 | } 21 | } 22 | 23 | #[cfg(feature = "chrono")] 24 | impl From> for DateTime { 25 | fn from(t: ChronoDateTime) -> Self { 26 | Self { 27 | seconds: t.timestamp_millis(), 28 | nano_seconds: t.timestamp_subsec_nanos(), 29 | } 30 | } 31 | } 32 | 33 | #[cfg(feature = "chrono")] 34 | impl From for ChronoDateTime { 35 | fn from(t: DateTime) -> Self { 36 | Local.timestamp(t.seconds, t.nano_seconds) 37 | } 38 | } 39 | 40 | #[cfg(feature = "chrono")] 41 | impl From> for DateTime { 42 | fn from(t: ChronoDateTime) -> Self { 43 | Self { 44 | seconds: t.timestamp_millis(), 45 | nano_seconds: t.timestamp_subsec_nanos(), 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! `Colfer` is a binary serialization format optimized for speed and size, this crate 2 | //! is a Rust implementation of the [colfer](https://github.com/pascaldekloe/colfer). 3 | 4 | #![warn(missing_docs)] 5 | #![forbid(unsafe_code)] 6 | 7 | mod datetime; 8 | mod types; 9 | 10 | use std::io::{Cursor, Read, Result, Write}; 11 | 12 | use byteorder::{ReadBytesExt, WriteBytesExt}; 13 | pub use datetime::DateTime; 14 | pub use types::{decode_message, decode_messages, encode_message, encode_messages, Type}; 15 | 16 | /// `MAX_SIZE` is the upper limit for serial byte sizes. 17 | pub const MAX_SIZE: usize = 16 * 1024 * 1024; 18 | 19 | /// `MAX_LIST_SIZE` is the upper limit for the number of elements in a list. 20 | pub const MAX_LIST_SIZE: usize = 64 * 1024; 21 | 22 | /// A colfer message. 23 | pub trait Message: Sized { 24 | /// Encodes the message to writer `W`. 25 | fn encode(&self, w: &mut W) -> Result<()>; 26 | 27 | /// Decodes an instance of the message from reader `R`. 28 | fn decode(r: &mut R) -> Result; 29 | 30 | /// Returns the encoded length of the message. 31 | fn size(&self) -> usize; 32 | 33 | /// Encodes the message to `Vec`. 34 | fn to_vec(&self) -> Result> { 35 | let mut data = Vec::new(); 36 | self.encode(&mut data)?; 37 | Ok(data) 38 | } 39 | 40 | /// Decodes an instance of the message from `Vec`. 41 | fn from_bytes(data: &[u8]) -> Result { 42 | Self::decode(&mut Cursor::new(data)) 43 | } 44 | } 45 | 46 | #[inline] 47 | #[doc(hidden)] 48 | pub fn read_header(r: &mut R) -> Result<(u8, bool)> { 49 | let d = r.read_u8()?; 50 | Ok((d & 0x7f, d & 0x80 > 0)) 51 | } 52 | 53 | #[inline] 54 | #[doc(hidden)] 55 | pub fn write_end(w: &mut W) -> Result<()> { 56 | w.write_u8(0x7f) 57 | } 58 | -------------------------------------------------------------------------------- /src/types.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Error, ErrorKind, Read, Result, Write}; 2 | use std::ops::Deref; 3 | 4 | use byteorder::{ReadBytesExt, WriteBytesExt, BE}; 5 | 6 | use crate::{DateTime, Message}; 7 | 8 | #[inline] 9 | fn write_uint(w: &mut W, mut x: u64) -> Result<()> { 10 | while x >= 0x80 { 11 | w.write_u8((x | 0x80) as u8)?; 12 | x >>= 7; 13 | } 14 | w.write_u8(x as u8)?; 15 | Ok(()) 16 | } 17 | 18 | #[inline] 19 | fn read_uint(r: &mut R) -> Result { 20 | let mut x = r.read_u8()? as u64; 21 | if x >= 0x80 { 22 | x &= 0x7f; 23 | let mut shift = 7; 24 | loop { 25 | let b = r.read_u8()? as u64; 26 | if b < 0x80 || shift == 56 { 27 | x |= b << shift; 28 | break; 29 | } 30 | x |= (b & 0x7f) << shift; 31 | shift += 7; 32 | } 33 | } 34 | Ok(x) 35 | } 36 | 37 | #[inline] 38 | fn uint_size(mut x: u64) -> usize { 39 | let mut l = 1; 40 | while x >= 0x80 { 41 | x >>= 7; 42 | l += 1; 43 | } 44 | l 45 | } 46 | 47 | #[doc(hidden)] 48 | pub trait Type: Sized { 49 | fn encode(&self, w: &mut W, id: u8) -> Result<()>; 50 | 51 | fn decode(r: &mut R, flag: bool) -> Result; 52 | 53 | fn size(&self) -> usize; 54 | } 55 | 56 | impl Type for bool { 57 | #[inline] 58 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 59 | if *self { 60 | w.write_u8(id)?; 61 | } 62 | Ok(()) 63 | } 64 | 65 | #[inline] 66 | fn decode(_r: &mut R, _flag: bool) -> Result { 67 | Ok(true) 68 | } 69 | 70 | #[inline] 71 | fn size(&self) -> usize { 72 | if *self { 73 | 1 74 | } else { 75 | 0 76 | } 77 | } 78 | } 79 | 80 | impl Type for u32 { 81 | #[inline] 82 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 83 | if *self >= 1 << 21 { 84 | w.write_u8(id | 0x80)?; 85 | w.write_u32::(*self)?; 86 | } else if *self != 0 { 87 | w.write_u8(id)?; 88 | write_uint(w, *self as u64)?; 89 | } 90 | Ok(()) 91 | } 92 | 93 | #[inline] 94 | fn decode(r: &mut R, flag: bool) -> Result { 95 | if !flag { 96 | Ok(read_uint(r)? as u32) 97 | } else { 98 | r.read_u32::() 99 | } 100 | } 101 | 102 | #[inline] 103 | fn size(&self) -> usize { 104 | if *self >= 1 << 21 { 105 | 5 106 | } else if *self != 0 { 107 | 1 + uint_size(*self as u64) 108 | } else { 109 | 0 110 | } 111 | } 112 | } 113 | 114 | impl Type for u64 { 115 | #[inline] 116 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 117 | if *self >= 1 << 49 { 118 | w.write_u8(id | 0x80)?; 119 | w.write_u64::(*self)?; 120 | } else if *self != 0 { 121 | w.write_u8(id)?; 122 | write_uint(w, *self as u64)?; 123 | } 124 | Ok(()) 125 | } 126 | 127 | #[inline] 128 | fn decode(r: &mut R, flag: bool) -> Result { 129 | if !flag { 130 | read_uint(r) 131 | } else { 132 | r.read_u64::() 133 | } 134 | } 135 | 136 | #[inline] 137 | fn size(&self) -> usize { 138 | if *self >= 1 << 49 { 139 | 9 140 | } else { 141 | 1 + uint_size(*self) 142 | } 143 | } 144 | } 145 | 146 | impl Type for i32 { 147 | #[inline] 148 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 149 | (*self as i64).encode(w, id) 150 | } 151 | 152 | #[inline] 153 | fn decode(r: &mut R, flag: bool) -> Result { 154 | Ok(i64::decode(r, flag)? as i32) 155 | } 156 | 157 | #[inline] 158 | fn size(&self) -> usize { 159 | (*self as i64).size() 160 | } 161 | } 162 | 163 | impl Type for i64 { 164 | #[inline] 165 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 166 | if *self != 0 { 167 | let mut x = *self as u64; 168 | if *self >= 0 { 169 | w.write_u8(id)?; 170 | } else { 171 | x = !x + 1; 172 | w.write_u8(id | 0x80)?; 173 | } 174 | write_uint(w, x as u64)?; 175 | } 176 | Ok(()) 177 | } 178 | 179 | #[inline] 180 | fn decode(r: &mut R, flag: bool) -> Result { 181 | if !flag { 182 | Ok(read_uint(r)? as i64) 183 | } else { 184 | Ok((!read_uint(r)? + 1) as i64) 185 | } 186 | } 187 | 188 | #[inline] 189 | fn size(&self) -> usize { 190 | if *self >= 0 { 191 | 1 + uint_size(*self as u64) 192 | } else { 193 | 1 + uint_size((!*self + 1) as u64) 194 | } 195 | } 196 | } 197 | 198 | impl Type for f32 { 199 | #[inline] 200 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 201 | if *self != 0.0 { 202 | w.write_u8(id)?; 203 | w.write_u32::(self.to_bits())?; 204 | } 205 | Ok(()) 206 | } 207 | 208 | #[inline] 209 | fn decode(r: &mut R, _flag: bool) -> Result { 210 | Ok(f32::from_bits(r.read_u32::()?)) 211 | } 212 | 213 | #[inline] 214 | fn size(&self) -> usize { 215 | if *self != 0.0 { 216 | 1 + 4 217 | } else { 218 | 0 219 | } 220 | } 221 | } 222 | 223 | impl Type for f64 { 224 | #[inline] 225 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 226 | if *self != 0.0 { 227 | w.write_u8(id)?; 228 | w.write_u64::(self.to_bits())?; 229 | } 230 | Ok(()) 231 | } 232 | 233 | #[inline] 234 | fn decode(r: &mut R, _flag: bool) -> Result { 235 | Ok(f64::from_bits(r.read_u64::()?)) 236 | } 237 | 238 | #[inline] 239 | fn size(&self) -> usize { 240 | if *self != 0.0 { 241 | 1 + 8 242 | } else { 243 | 0 244 | } 245 | } 246 | } 247 | 248 | impl Type for DateTime { 249 | #[inline] 250 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 251 | let DateTime { 252 | seconds: s, 253 | nano_seconds: ns, 254 | } = *self; 255 | if s != 0 || ns != 0 { 256 | if s < 1 << 32 { 257 | w.write_u8(id)?; 258 | w.write_u32::(s as u32)?; 259 | } else { 260 | w.write_u8(id | 0x80)?; 261 | w.write_u64::(s as u64)?; 262 | } 263 | w.write_u32::(ns)?; 264 | } 265 | Ok(()) 266 | } 267 | 268 | #[inline] 269 | fn decode(r: &mut R, flag: bool) -> Result { 270 | if !flag { 271 | let s = r.read_u32::()?; 272 | let ns = r.read_u32::()?; 273 | Ok(DateTime { 274 | seconds: s as i64, 275 | nano_seconds: ns, 276 | }) 277 | } else { 278 | let s = r.read_u64::()?; 279 | let ns = r.read_u32::()?; 280 | Ok(DateTime { 281 | seconds: s as i64, 282 | nano_seconds: ns, 283 | }) 284 | } 285 | } 286 | 287 | #[inline] 288 | fn size(&self) -> usize { 289 | let DateTime { 290 | seconds: s, 291 | nano_seconds: ns, 292 | } = *self; 293 | if s != 0 || ns != 0 { 294 | if s < 1 << 32 { 295 | 1 + 8 296 | } else { 297 | 1 + 12 298 | } 299 | } else { 300 | 0 301 | } 302 | } 303 | } 304 | 305 | impl Type for String { 306 | #[inline] 307 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 308 | if !self.is_empty() { 309 | w.write_u8(id)?; 310 | write_uint(w, self.len() as u64)?; 311 | w.write_all(self.as_bytes())?; 312 | } 313 | Ok(()) 314 | } 315 | 316 | #[inline] 317 | fn decode(r: &mut R, _flag: bool) -> Result { 318 | let l = read_uint(r)?; 319 | let mut s = vec![0; l as usize]; 320 | r.read_exact(&mut s)?; 321 | Ok(String::from_utf8(s).map_err(|err| Error::new(ErrorKind::Other, err))?) 322 | } 323 | 324 | #[inline] 325 | fn size(&self) -> usize { 326 | if !self.is_empty() { 327 | 1 + uint_size(self.len() as u64) + self.len() 328 | } else { 329 | 0 330 | } 331 | } 332 | } 333 | 334 | impl Type for Vec { 335 | #[inline] 336 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 337 | if !self.is_empty() { 338 | w.write_u8(id)?; 339 | write_uint(w, self.len() as u64)?; 340 | w.write_all(self)?; 341 | } 342 | Ok(()) 343 | } 344 | 345 | #[inline] 346 | fn decode(r: &mut R, _flag: bool) -> Result { 347 | let l = read_uint(r)?; 348 | let mut s = vec![0; l as usize]; 349 | r.read_exact(&mut s)?; 350 | Ok(s) 351 | } 352 | 353 | #[inline] 354 | fn size(&self) -> usize { 355 | if !self.is_empty() { 356 | 1 + uint_size(self.len() as u64) + self.len() 357 | } else { 358 | 0 359 | } 360 | } 361 | } 362 | 363 | #[doc(hidden)] 364 | #[inline] 365 | pub fn encode_message(w: &mut W, id: u8, message: Option<&T>) -> Result<()> { 366 | if let Some(message) = message { 367 | w.write_u8(id)?; 368 | message.encode(w)?; 369 | } 370 | Ok(()) 371 | } 372 | 373 | #[doc(hidden)] 374 | #[inline] 375 | pub fn decode_message + From>( 376 | r: &mut R, 377 | ) -> Result> { 378 | Ok(Some(T::from(M::decode(r)?))) 379 | } 380 | 381 | #[doc(hidden)] 382 | #[inline] 383 | pub fn encode_messages(w: &mut W, id: u8, messages: &[T]) -> Result<()> { 384 | if !messages.is_empty() { 385 | w.write_u8(id)?; 386 | write_uint(w, messages.len() as u64)?; 387 | for s in messages { 388 | s.encode(w)?; 389 | } 390 | } 391 | Ok(()) 392 | } 393 | 394 | #[doc(hidden)] 395 | #[inline] 396 | pub fn decode_messages(r: &mut R) -> Result> { 397 | let l = read_uint(r)?; 398 | let mut s = Vec::with_capacity(l as usize); 399 | for _ in 0..l { 400 | s.push(T::decode(r)?); 401 | } 402 | Ok(s) 403 | } 404 | 405 | impl Type for Vec { 406 | #[inline] 407 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 408 | if !self.is_empty() { 409 | w.write_u8(id)?; 410 | write_uint(w, self.len() as u64)?; 411 | for s in self { 412 | write_uint(w, self.len() as u64)?; 413 | w.write_all(s.as_bytes())?; 414 | } 415 | } 416 | Ok(()) 417 | } 418 | 419 | #[inline] 420 | fn decode(r: &mut R, _flag: bool) -> Result { 421 | let l = read_uint(r)?; 422 | let mut s = Vec::with_capacity(l as usize); 423 | for _ in 0..l { 424 | let sz = read_uint(r)?; 425 | let mut d = vec![0; sz as usize]; 426 | r.read_exact(&mut d)?; 427 | s.push(String::from_utf8(d).map_err(|err| Error::new(ErrorKind::Other, err))?); 428 | } 429 | Ok(s) 430 | } 431 | 432 | #[inline] 433 | fn size(&self) -> usize { 434 | if !self.is_empty() { 435 | 1 + uint_size(self.len() as u64) 436 | + self 437 | .iter() 438 | .map(|s| uint_size(s.len() as u64) + s.len()) 439 | .sum::() 440 | } else { 441 | 0 442 | } 443 | } 444 | } 445 | 446 | impl Type for Vec> { 447 | #[inline] 448 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 449 | if !self.is_empty() { 450 | w.write_u8(id)?; 451 | write_uint(w, self.len() as u64)?; 452 | for s in self { 453 | write_uint(w, self.len() as u64)?; 454 | w.write_all(s)?; 455 | } 456 | } 457 | Ok(()) 458 | } 459 | 460 | #[inline] 461 | fn decode(r: &mut R, _flag: bool) -> Result { 462 | let l = read_uint(r)?; 463 | let mut s = Vec::with_capacity(l as usize); 464 | for _ in 0..l { 465 | let sz = read_uint(r)?; 466 | let mut d = vec![0; sz as usize]; 467 | r.read_exact(&mut d)?; 468 | s.push(d); 469 | } 470 | Ok(s) 471 | } 472 | 473 | #[inline] 474 | fn size(&self) -> usize { 475 | if !self.is_empty() { 476 | 1 + uint_size(self.len() as u64) 477 | + self 478 | .iter() 479 | .map(|s| uint_size(s.len() as u64) + s.len()) 480 | .sum::() 481 | } else { 482 | 0 483 | } 484 | } 485 | } 486 | 487 | impl Type for u8 { 488 | #[inline] 489 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 490 | if *self != 0 { 491 | w.write_u8(id)?; 492 | w.write_u8(*self)?; 493 | } 494 | Ok(()) 495 | } 496 | 497 | #[inline] 498 | fn decode(r: &mut R, _flag: bool) -> Result { 499 | r.read_u8() 500 | } 501 | 502 | #[inline] 503 | fn size(&self) -> usize { 504 | if *self != 0 { 505 | 1 + 1 506 | } else { 507 | 0 508 | } 509 | } 510 | } 511 | 512 | impl Type for u16 { 513 | #[inline] 514 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 515 | if *self >= 1 << 8 { 516 | w.write_u8(id)?; 517 | w.write_u16::(*self)?; 518 | } else if *self != 0 { 519 | w.write_u8(id | 0x80)?; 520 | w.write_u8(*self as u8)?; 521 | } 522 | Ok(()) 523 | } 524 | 525 | #[inline] 526 | fn decode(r: &mut R, flag: bool) -> Result { 527 | if !flag { 528 | r.read_u16::() 529 | } else { 530 | Ok(r.read_u8()? as u16) 531 | } 532 | } 533 | 534 | #[inline] 535 | fn size(&self) -> usize { 536 | if *self >= 1 << 8 { 537 | 3 538 | } else if *self != 0 { 539 | 2 540 | } else { 541 | 0 542 | } 543 | } 544 | } 545 | 546 | impl Type for Vec { 547 | #[inline] 548 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 549 | if !self.is_empty() { 550 | w.write_u8(id)?; 551 | write_uint(w, self.len() as u64)?; 552 | for s in self { 553 | w.write_u32::(s.to_bits())?; 554 | } 555 | } 556 | Ok(()) 557 | } 558 | 559 | #[inline] 560 | fn decode(r: &mut R, _flag: bool) -> Result { 561 | let l = read_uint(r)?; 562 | let mut s = Vec::with_capacity(l as usize); 563 | for _ in 0..l { 564 | s.push(f32::from_bits(r.read_u32::()?)); 565 | } 566 | Ok(s) 567 | } 568 | 569 | #[inline] 570 | fn size(&self) -> usize { 571 | if !self.is_empty() { 572 | 1 + uint_size(self.len() as u64) 573 | + self 574 | .iter() 575 | .map(|n| uint_size(n.to_bits() as u64)) 576 | .sum::() 577 | } else { 578 | 0 579 | } 580 | } 581 | } 582 | 583 | impl Type for Vec { 584 | #[inline] 585 | fn encode(&self, w: &mut W, id: u8) -> Result<()> { 586 | if !self.is_empty() { 587 | w.write_u8(id)?; 588 | write_uint(w, self.len() as u64)?; 589 | for s in self { 590 | w.write_u64::(s.to_bits())?; 591 | } 592 | } 593 | Ok(()) 594 | } 595 | 596 | #[inline] 597 | fn decode(r: &mut R, _flag: bool) -> Result { 598 | let l = read_uint(r)?; 599 | let mut s = Vec::with_capacity(l as usize); 600 | for _ in 0..l { 601 | s.push(f64::from_bits(r.read_u64::()?)); 602 | } 603 | Ok(s) 604 | } 605 | 606 | #[inline] 607 | fn size(&self) -> usize { 608 | if !self.is_empty() { 609 | 1 + uint_size(self.len() as u64) 610 | + self.iter().map(|n| uint_size(n.to_bits())).sum::() 611 | } else { 612 | 0 613 | } 614 | } 615 | } 616 | 617 | #[cfg(test)] 618 | mod tests { 619 | use std::fmt::Debug; 620 | use std::io::Cursor; 621 | 622 | use crate::*; 623 | 624 | fn do_test(value: T) { 625 | let mut data = Vec::new(); 626 | value.encode(&mut data, 10).unwrap(); 627 | 628 | let mut r = Cursor::new(&data); 629 | if data.is_empty() { 630 | assert_eq!(T::default(), value); 631 | } else { 632 | let (id, flag) = read_header(&mut r).unwrap(); 633 | assert_eq!(id, 10); 634 | assert_eq!(T::decode(&mut r, flag).unwrap(), value); 635 | } 636 | } 637 | 638 | #[test] 639 | fn test_i32() { 640 | do_test(i32::MAX); 641 | do_test(i32::MIN); 642 | do_test(0i32); 643 | } 644 | 645 | #[test] 646 | fn test_i64() { 647 | do_test(i64::MAX); 648 | do_test(i64::MIN); 649 | } 650 | 651 | #[test] 652 | fn test_f32() { 653 | do_test(f32::MAX); 654 | do_test(f32::MIN); 655 | } 656 | 657 | #[test] 658 | fn test_f64() { 659 | do_test(f64::MAX); 660 | do_test(f64::MIN); 661 | } 662 | } 663 | --------------------------------------------------------------------------------