├── .github └── workflows │ └── test.yml ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE ├── README.md ├── benches └── main.rs ├── coverage.sh ├── examples ├── encode_varint_from_stdin.rs └── read_write_file.rs └── src ├── fixed.rs ├── fixed_tests.rs ├── lib.rs ├── reader.rs ├── varint.rs ├── varint_tests.rs └── writer.rs /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: Actions CI 4 | 5 | jobs: 6 | build_and_test: 7 | name: integer-encoding-rs 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | features: ["tokio_async", "futures_async", ""] 12 | platform: [ubuntu-latest, windows-latest, macos-latest] 13 | runs-on: ${{ matrix.platform }} 14 | steps: 15 | - uses: actions/checkout@v2 16 | - uses: actions-rs/toolchain@v1 17 | with: 18 | toolchain: stable 19 | - uses: actions-rs/cargo@v1 20 | with: 21 | command: check 22 | args: --no-default-features --features=${{ matrix.features }} 23 | - uses: actions-rs/cargo@v1 24 | with: 25 | command: test 26 | args: --no-default-features --features=${{ matrix.features }} 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | 3 | os: 4 | - linux 5 | - osx 6 | - windows 7 | dist: focal 8 | rust: 9 | - stable 10 | - nightly 11 | 12 | # Test on the latest versions of all channels. 13 | script: 14 | - cargo test --features tokio_async --verbose 15 | - cargo test --features futures_async --verbose 16 | - cargo test --verbose 17 | - cargo build --examples 18 | # Run this build on the "container-based infrastructure" 19 | # See http://docs.travis-ci.com/user/workers/container-based-infrastructure/. 20 | sudo: false 21 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "integer-encoding" 3 | version = "4.0.2" 4 | authors = ["Lewin Bormann "] 5 | description = "varint+zigzag and fixedint integer encoding/decoding (https://developers.google.com/protocol-buffers/docs/encoding)" 6 | repository = "https://github.com/dermesser/integer-encoding-rs" 7 | documentation = "https://docs.rs/integer-encoding/" 8 | license = "MIT" 9 | keywords = ["integer", "varint", "zigzag", "protobuf", "serialize"] 10 | edition = "2018" 11 | 12 | [dependencies] 13 | async-trait = { version = "0.1", optional = true } 14 | tokio = { version = "1.0", features = ["io-util"], optional = true } 15 | futures-util = { version = "0.3", optional = true, features = ["io"] } 16 | 17 | [dev-dependencies] 18 | tokio = { version = "1.0", features = ["fs", "rt-multi-thread", "macros"] } 19 | bencher = "~0.1" 20 | 21 | [[example]] 22 | name = "encode_varint_from_stdin" 23 | required-features = ["tokio_async"] 24 | 25 | [[example]] 26 | name = "read_write_file" 27 | required-features = ["tokio_async"] 28 | 29 | [[bench]] 30 | name = "main" 31 | harness = false 32 | 33 | [features] 34 | # Enable one of these features if you want to use the AsyncRead/AsyncWrite traits from 35 | # the futures crate instead of those from tokio. 36 | tokio_async = ["tokio", "async-trait"] 37 | futures_async = ["futures-util", "async-trait"] 38 | 39 | [package.metadata.docs.rs] 40 | features = ["tokio_async"] 41 | 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Google Inc. (lewinb@google.com) -- though not an official 4 | Google product or in any way related! 5 | Copyright (c) 2018-2020 Lewin Bormann (lbo@spheniscida.de) 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to 9 | deal in the Software without restriction, including without limitation the 10 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | sell copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 | IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # integer-encoding-rs 2 | 3 | [![crates.io](https://img.shields.io/crates/v/integer-encoding.svg)](https://crates.io/crates/integer-encoding) 4 | [![Actions CI](https://github.com/dermesser/integer-encoding-rs/workflows/Actions%20CI/badge.svg)](https://github.com/dermesser/integer-encoding-rs/actions) 5 | 6 | [full documentation](https://docs.rs/integer-encoding/) 7 | 8 | This crate provides encoding and decoding of integers to and from bytestring 9 | representations. 10 | 11 | The format is described here: [Google's protobuf integer encoding technique](https://developers.google.com/protocol-buffers/docs/encoding). 12 | 13 | Please feel free to use `cargo bench` to determine the rate at which your 14 | machine can encode and decode varints and fixedints. Note that one iteration 15 | comprises each eight rounds of encoding (or decoding) a signed and an unsigned 16 | integer each -- divide the resulting benchmark time by 16 in order to have a 17 | rough estimate of time per operation. The integers are very large, so the 18 | results represent the worst case. 19 | 20 | ## Crate 21 | 22 | If you use Tokio v0.2 and you use the asynchronous types in this crate (feature 23 | `tokio_async`), you may be interested in the `v2.0` branch. It is still 24 | maintained with the occasional fix for edge cases and depends on Tokio v0.2. 25 | 26 | ## FixedInt 27 | 28 | `FixedInt` casts integers to bytes by either copying the underlying memory or 29 | performing a transmutation. The encoded values use are little-endian. 30 | 31 | However, a trait method is implemented for all integer types allowing convenient conversion between 32 | little and big endian. That is, if you receive a big-endian on the wire and decode it, it will first 33 | be interpreted as little-endian; converting will recover the correct value. 34 | 35 | ## VarInt 36 | 37 | `VarInt` encodes integers in blocks of 7 bits; the MSB is set for every byte but 38 | the last, in which it is cleared. 39 | 40 | Signed values are first converted to an unsigned representation using zigzag 41 | encoding (also described on the page linked above), and then encoded as every 42 | other unsigned number. 43 | 44 | -------------------------------------------------------------------------------- /benches/main.rs: -------------------------------------------------------------------------------- 1 | use bencher::Bencher; 2 | 3 | use integer_encoding::*; 4 | 5 | fn encode_v(b: &mut Bencher) { 6 | let my_u64s = [ 7 | 9494929199119074561, 8 | 3823923198123425321, 9 | 2595862268225688522, 10 | 1231230009321245673, 11 | 2909812083312547546, 12 | 3492011001874124465, 13 | 4848848884210156473, 14 | 4012941340125654654, 15 | ] as [u64; 8]; 16 | let my_i64s = [ 17 | -122193043711204545, 18 | 2446312246543212452, 19 | -445854216865433664, 20 | 3242135654513135465, 21 | -543122132545464312, 22 | 3613543123031434343, 23 | -431231254654543211, 24 | 7854615463131234543, 25 | ] as [i64; 8]; 26 | 27 | let mut dst = [0 as u8; 10]; 28 | 29 | b.iter(|| { 30 | // 8x each. 31 | my_u64s[0].encode_var(&mut dst); 32 | my_u64s[1].encode_var(&mut dst); 33 | my_u64s[2].encode_var(&mut dst); 34 | my_u64s[3].encode_var(&mut dst); 35 | my_u64s[4].encode_var(&mut dst); 36 | my_u64s[5].encode_var(&mut dst); 37 | my_u64s[6].encode_var(&mut dst); 38 | my_u64s[7].encode_var(&mut dst); 39 | 40 | my_i64s[0].encode_var(&mut dst); 41 | my_i64s[1].encode_var(&mut dst); 42 | my_i64s[2].encode_var(&mut dst); 43 | my_i64s[3].encode_var(&mut dst); 44 | my_i64s[4].encode_var(&mut dst); 45 | my_i64s[5].encode_var(&mut dst); 46 | my_i64s[6].encode_var(&mut dst); 47 | my_i64s[7].encode_var(&mut dst); 48 | }); 49 | } 50 | 51 | fn decode_v(b: &mut Bencher) { 52 | let my_u64s = [ 53 | 9494929199119074561, 54 | 3823923198123425321, 55 | 2595862268225688522, 56 | 1231230009321245673, 57 | 2909812083312547546, 58 | 3492011001874124465, 59 | 4848848884210156473, 60 | 4012941340125654654, 61 | ] as [u64; 8]; 62 | let my_i64s = [ 63 | -122193043711204545, 64 | 2446312246543212452, 65 | -445854216865433664, 66 | 3242135654513135465, 67 | -543122132545464312, 68 | 3613543123031434343, 69 | -431231254654543211, 70 | 7854615463131234543, 71 | ] as [i64; 8]; 72 | 73 | let u64_src = [ 74 | my_u64s[0].encode_var_vec(), 75 | my_u64s[1].encode_var_vec(), 76 | my_u64s[2].encode_var_vec(), 77 | my_u64s[3].encode_var_vec(), 78 | my_u64s[4].encode_var_vec(), 79 | my_u64s[5].encode_var_vec(), 80 | my_u64s[6].encode_var_vec(), 81 | my_u64s[7].encode_var_vec(), 82 | ] as [Vec; 8]; 83 | let i64_src = [ 84 | my_i64s[0].encode_var_vec(), 85 | my_i64s[1].encode_var_vec(), 86 | my_i64s[2].encode_var_vec(), 87 | my_i64s[3].encode_var_vec(), 88 | my_i64s[4].encode_var_vec(), 89 | my_i64s[5].encode_var_vec(), 90 | my_i64s[6].encode_var_vec(), 91 | my_i64s[7].encode_var_vec(), 92 | ] as [Vec; 8]; 93 | 94 | b.iter(|| { 95 | // 8x each. 96 | u64::decode_var(&u64_src[0]).unwrap(); 97 | u64::decode_var(&u64_src[1]).unwrap(); 98 | u64::decode_var(&u64_src[2]).unwrap(); 99 | u64::decode_var(&u64_src[3]).unwrap(); 100 | u64::decode_var(&u64_src[4]).unwrap(); 101 | u64::decode_var(&u64_src[5]).unwrap(); 102 | u64::decode_var(&u64_src[6]).unwrap(); 103 | u64::decode_var(&u64_src[7]).unwrap(); 104 | 105 | i64::decode_var(&i64_src[0]).unwrap(); 106 | i64::decode_var(&i64_src[1]).unwrap(); 107 | i64::decode_var(&i64_src[2]).unwrap(); 108 | i64::decode_var(&i64_src[3]).unwrap(); 109 | i64::decode_var(&i64_src[4]).unwrap(); 110 | i64::decode_var(&i64_src[5]).unwrap(); 111 | i64::decode_var(&i64_src[6]).unwrap(); 112 | i64::decode_var(&i64_src[7]).unwrap(); 113 | }); 114 | } 115 | 116 | bencher::benchmark_group!(varint_benches, encode_v, decode_v); 117 | 118 | fn encode_f(b: &mut Bencher) { 119 | let my_u64 = 94949291991190 as u64; 120 | let my_i64 = -12219304371120 as i64; 121 | 122 | let mut dst = [0 as u8; 8]; 123 | 124 | b.iter(|| { 125 | // 8x each. 126 | my_u64.encode_fixed(&mut dst); 127 | my_u64.encode_fixed(&mut dst); 128 | my_u64.encode_fixed(&mut dst); 129 | my_u64.encode_fixed(&mut dst); 130 | my_u64.encode_fixed(&mut dst); 131 | my_u64.encode_fixed(&mut dst); 132 | my_u64.encode_fixed(&mut dst); 133 | 134 | my_i64.encode_fixed(&mut dst); 135 | my_i64.encode_fixed(&mut dst); 136 | my_i64.encode_fixed(&mut dst); 137 | my_i64.encode_fixed(&mut dst); 138 | my_i64.encode_fixed(&mut dst); 139 | my_i64.encode_fixed(&mut dst); 140 | my_i64.encode_fixed(&mut dst); 141 | my_i64.encode_fixed(&mut dst); 142 | }); 143 | } 144 | 145 | fn decode_f(b: &mut Bencher) { 146 | let my_u64 = 94949291991190 as u64; 147 | let my_i64 = -12219304371120 as i64; 148 | 149 | let u64_src = my_u64.encode_fixed_vec(); 150 | let i64_src = my_i64.encode_fixed_vec(); 151 | 152 | b.iter(|| { 153 | // 8x each. 154 | u64::decode_fixed(&u64_src); 155 | u64::decode_fixed(&u64_src); 156 | u64::decode_fixed(&u64_src); 157 | u64::decode_fixed(&u64_src); 158 | u64::decode_fixed(&u64_src); 159 | u64::decode_fixed(&u64_src); 160 | u64::decode_fixed(&u64_src); 161 | 162 | i64::decode_fixed(&i64_src); 163 | i64::decode_fixed(&i64_src); 164 | i64::decode_fixed(&i64_src); 165 | i64::decode_fixed(&i64_src); 166 | i64::decode_fixed(&i64_src); 167 | i64::decode_fixed(&i64_src); 168 | i64::decode_fixed(&i64_src); 169 | i64::decode_fixed(&i64_src); 170 | }); 171 | } 172 | 173 | bencher::benchmark_group!(fixedint_benches, encode_f, decode_f); 174 | 175 | bencher::benchmark_main!(varint_benches, fixedint_benches); 176 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | KCOV=kcov 6 | KCOV_OPTS="--exclude-pattern=/.cargo,/glibc,/usr/lib,/usr/include" 7 | KCOV_OUT="./kcov-out/" 8 | 9 | export RUSTFLAGS="-C link-dead-code" 10 | 11 | TEST_BIN=$(cargo test 2>&1 > /dev/null | awk '/^ Running target\/debug\/deps\// { print $2 }') 12 | 13 | echo $TEST_BIN 14 | ${KCOV} ${KCOV_OPTS} ${KCOV_OUT} ${TEST_BIN} && xdg-open ${KCOV_OUT}/index.html 15 | -------------------------------------------------------------------------------- /examples/encode_varint_from_stdin.rs: -------------------------------------------------------------------------------- 1 | use integer_encoding::VarInt; 2 | 3 | use std::io::{self, BufRead}; 4 | use std::str::FromStr; 5 | 6 | fn binencode(b: &[u8]) -> String { 7 | let mut s = String::new(); 8 | for byte in b { 9 | s.push_str(&format!("{:08b} ", byte)); 10 | } 11 | s 12 | } 13 | 14 | fn main() { 15 | let stdin = io::BufReader::new(io::stdin()); 16 | 17 | println!("Enter decimal numbers here:\n"); 18 | for l in stdin.lines() { 19 | if l.is_err() { 20 | break; 21 | } 22 | let l = l.unwrap(); 23 | match i64::from_str(&l) { 24 | Ok(i) => println!( 25 | "fixed: {:b} encoded (unsigned): {} encoded (signed): {}", 26 | i, 27 | if i >= 0 { 28 | binencode(&(i as u64).encode_var_vec()) 29 | } else { 30 | "-".to_string() 31 | }, 32 | binencode(&i.encode_var_vec()) 33 | ), 34 | Err(e) => println!("{:?}", e), 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/read_write_file.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::io; 3 | 4 | use integer_encoding::*; 5 | 6 | async fn write_test_files() -> io::Result<()> { 7 | let _ = fs::remove_file("/tmp/varintbytes"); 8 | let mut f = tokio::fs::File::create("/tmp/varintbytes").await?; 9 | f.write_varint_async(30 as u32).await?; 10 | f.write_varint_async(60 as u32).await?; 11 | f.write_varint_async(90 as u32).await?; 12 | f.write_varint_async(9000000 as u32).await?; 13 | 14 | let _ = fs::remove_file("/tmp/fixedintbytes"); 15 | let mut f = tokio::fs::File::create("/tmp/fixedintbytes").await?; 16 | f.write_fixedint_async(30 as u32).await?; 17 | f.write_fixedint_async(60 as u32).await?; 18 | f.write_fixedint_async(90 as u32).await?; 19 | f.write_fixedint_async(9000000 as u32).await?; 20 | Ok(()) 21 | } 22 | 23 | async fn read_and_verify_varints() -> io::Result<()> { 24 | let f = tokio::fs::File::open("/tmp/varintbytes").await?; 25 | let mut f = tokio::io::BufReader::new(f); 26 | let i1: u32 = f.read_varint_async().await?; 27 | let i2: u32 = f.read_varint_async().await?; 28 | let i3: u32 = f.read_varint_async().await?; 29 | let i4: u32 = f.read_varint_async().await?; 30 | assert!(f.read_varint_async::().await.is_err()); 31 | println!("{:?}", (i1, i2, i3, i4)); 32 | assert!(i2 == 2 * i1 && i3 == 3 * i1); 33 | Ok(()) 34 | } 35 | 36 | async fn read_and_verify_fixedints() -> io::Result<()> { 37 | let f = tokio::fs::File::open("/tmp/fixedintbytes").await?; 38 | let mut f = tokio::io::BufReader::new(f); 39 | let i1: u32 = f.read_fixedint_async().await?; 40 | let i2: u32 = f.read_fixedint_async().await?; 41 | let i3: u32 = f.read_fixedint_async().await?; 42 | let i4: u32 = f.read_fixedint_async().await?; 43 | assert!(f.read_fixedint_async::().await.is_err()); 44 | println!("{:?}", (i1, i2, i3, i4)); 45 | assert!(i2 == 2 * i1 && i3 == 3 * i1); 46 | Ok(()) 47 | } 48 | 49 | #[tokio::main] 50 | async fn main() { 51 | write_test_files().await.unwrap(); 52 | 53 | read_and_verify_varints().await.unwrap(); 54 | read_and_verify_fixedints().await.unwrap(); 55 | } 56 | -------------------------------------------------------------------------------- /src/fixed.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryInto; 2 | use std::mem::size_of; 3 | 4 | /// `FixedInt` provides encoding/decoding to and from fixed int representations. Note that current 5 | /// Rust versions already provide this functionality via the `to_le_bytes()` and `to_be_bytes()` 6 | /// methods. 7 | /// 8 | /// The emitted bytestring contains the bytes of the integer in machine endianness. 9 | pub trait FixedInt: Sized + Copy { 10 | type Bytes: AsRef<[u8]>; 11 | const ENCODED_SIZE: usize = size_of::(); 12 | 13 | /// Encode a value into the given slice using little-endian. Returns `None` if `dst` 14 | /// doesn't provide enough space to encode this integer. 15 | /// 16 | /// Use `switch_endianness()` if machine endianness doesn't match the desired target encoding. 17 | fn encode_fixed(self, dst: &mut [u8]) -> Option<()>; 18 | /// Returns the representation of [`FixedInt`] as [`Bytes`], the little-endian representation 19 | /// of self in the stack. 20 | fn encode_fixed_light(self) -> Self::Bytes; 21 | 22 | /// Decode a value from the given slice assuming little-endian. Use `switch_endianness()` on 23 | /// the returned value if the source was not encoded in little-endian. 24 | fn decode_fixed(src: &[u8]) -> Option; 25 | 26 | /// Helper: Encode the value and return a Vec. 27 | fn encode_fixed_vec(self) -> Vec { 28 | self.encode_fixed_light().as_ref().to_vec() 29 | } 30 | 31 | /// integer-encoding-rs always emits and receives little-endian integers (converting implicitly 32 | /// on big-endian machines). If you receive a big-endian integer, and would like it to be 33 | /// treated correctly, use this helper method to convert between endiannesses. 34 | fn switch_endianness(self) -> Self; 35 | } 36 | 37 | macro_rules! impl_fixedint { 38 | ($t:ty) => { 39 | impl FixedInt for $t { 40 | type Bytes = [u8; size_of::<$t>()]; 41 | 42 | fn encode_fixed(self, dst: &mut [u8]) -> Option<()> { 43 | if dst.len() == size_of::() { 44 | dst.clone_from_slice(&self.to_le_bytes()); 45 | Some(()) 46 | } else { 47 | None 48 | } 49 | } 50 | 51 | fn encode_fixed_light(self) -> Self::Bytes { 52 | self.to_le_bytes() 53 | } 54 | 55 | fn decode_fixed(src: &[u8]) -> Option { 56 | if src.len() == size_of::() { 57 | Some(Self::from_le_bytes(src.try_into().unwrap())) 58 | } else { 59 | None 60 | } 61 | } 62 | 63 | fn switch_endianness(self) -> Self { 64 | Self::from_le_bytes(self.to_be_bytes()) 65 | } 66 | } 67 | }; 68 | } 69 | 70 | impl_fixedint!(usize); 71 | impl_fixedint!(u64); 72 | impl_fixedint!(u32); 73 | impl_fixedint!(u16); 74 | impl_fixedint!(u8); 75 | impl_fixedint!(isize); 76 | impl_fixedint!(i64); 77 | impl_fixedint!(i32); 78 | impl_fixedint!(i16); 79 | impl_fixedint!(i8); 80 | -------------------------------------------------------------------------------- /src/fixed_tests.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use crate::fixed::FixedInt; 4 | 5 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 6 | use crate::reader::FixedIntAsyncReader; 7 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 8 | use crate::writer::FixedIntAsyncWriter; 9 | 10 | use crate::reader::FixedIntReader; 11 | use crate::writer::FixedIntWriter; 12 | 13 | #[test] 14 | fn test_u32_enc() { 15 | let result = (32 as u32).encode_fixed_vec(); 16 | assert_eq!(result, vec![32, 0, 0, 0]); 17 | } 18 | #[test] 19 | fn test_u16_enc() { 20 | let result = (256 as u16).encode_fixed_vec(); 21 | assert_eq!(result, vec![0, 1]); 22 | } 23 | #[test] 24 | fn test_u16_endian() { 25 | let le = 1 as u16; 26 | let be = le.switch_endianness(); 27 | assert_eq!(be, 256); 28 | } 29 | #[test] 30 | fn test_u32_endian() { 31 | let le = 1 as u32; 32 | let be = le.switch_endianness(); 33 | assert_eq!(be, 1 << 24); 34 | assert_eq!(be.switch_endianness(), 1); 35 | } 36 | #[test] 37 | fn test_allones_endian() { 38 | assert_eq!(u64::MAX.switch_endianness(), u64::MAX); 39 | } 40 | #[test] 41 | fn test_signed_endian() { 42 | // x86: two's complement, LE. 43 | let le = -2 as i16; 44 | let be = le.switch_endianness(); 45 | assert_eq!(be, -257); 46 | } 47 | #[test] 48 | fn test_u8_enc() { 49 | let result = (255 as u8).encode_fixed_vec(); 50 | assert_eq!(result, vec![255]); 51 | } 52 | #[test] 53 | fn test_i8_enc() { 54 | let result = (-1 as i8).encode_fixed_vec(); 55 | assert_eq!(result, vec![255]); 56 | } 57 | #[test] 58 | fn test_i16_enc() { 59 | let result = (-32768 as i16).encode_fixed_vec(); 60 | assert_eq!(result, vec![0, 128]); 61 | } 62 | #[test] 63 | fn test_i32_enc() { 64 | let result = (-32767 as i32).encode_fixed_vec(); 65 | assert_eq!(result, vec![1, 128, 255, 255]); 66 | } 67 | 68 | // This must fail to compile: 69 | /* 70 | fn test() -> &'static [u8] { 71 | let int = -32767 as i32; 72 | let result = int.encode_fixed_light(); 73 | assert_eq!(result, &[1, 128, 255, 255]); 74 | result 75 | } 76 | */ 77 | 78 | #[test] 79 | fn test_i32_enc_light() { 80 | let int = -32767 as i32; 81 | let result = int.encode_fixed_light(); 82 | assert_eq!(result, [1, 128, 255, 255]); 83 | } 84 | #[test] 85 | fn test_all_identity() { 86 | let a: u8 = 17; 87 | let b: u16 = 17; 88 | let c: u32 = 17; 89 | let d: u64 = 17; 90 | let e: i8 = -17; 91 | let f: i16 = -17; 92 | let g: i32 = -17; 93 | let h: i64 = -17; 94 | 95 | assert_eq!(a, FixedInt::decode_fixed(&a.encode_fixed_light()).unwrap()); 96 | assert_eq!(b, FixedInt::decode_fixed(&b.encode_fixed_light()).unwrap()); 97 | assert_eq!(c, FixedInt::decode_fixed(&c.encode_fixed_light()).unwrap()); 98 | assert_eq!(d, FixedInt::decode_fixed(&d.encode_fixed_light()).unwrap()); 99 | assert_eq!(e, FixedInt::decode_fixed(&e.encode_fixed_light()).unwrap()); 100 | assert_eq!(f, FixedInt::decode_fixed(&f.encode_fixed_light()).unwrap()); 101 | assert_eq!(g, FixedInt::decode_fixed(&g.encode_fixed_light()).unwrap()); 102 | assert_eq!(h, FixedInt::decode_fixed(&h.encode_fixed_light()).unwrap()); 103 | } 104 | 105 | #[test] 106 | fn test_reader_writer() { 107 | let mut buf = Vec::with_capacity(128); 108 | 109 | let i1: u32 = 123; 110 | let i2: u32 = 124; 111 | let i3: u32 = 125; 112 | 113 | assert!(buf.write_fixedint(i1).is_ok()); 114 | assert!(buf.write_fixedint(i2).is_ok()); 115 | assert!(buf.write_fixedint(i3).is_ok()); 116 | 117 | assert_eq!(3 * 4, buf.len()); 118 | 119 | let mut reader: &[u8] = buf.as_ref(); 120 | 121 | let i1_res = reader.read_fixedint().unwrap(); 122 | let i2_res = reader.read_fixedint().unwrap(); 123 | let i3_res = reader.read_fixedint().unwrap(); 124 | 125 | assert_eq!(i1, i1_res); 126 | assert_eq!(i2, i2_res); 127 | assert_eq!(i3, i3_res); 128 | 129 | assert!(reader.read_fixedint::().is_err()); 130 | } 131 | 132 | #[should_panic] 133 | #[test] 134 | fn test_invalid_decode_size() { 135 | assert_eq!(33, u64::decode_fixed(&[1, 0, 0, 0, 0, 1]).unwrap()); 136 | } 137 | #[should_panic] 138 | #[test] 139 | fn test_invalid_encode_size() { 140 | let mut buf = [0 as u8; 4]; 141 | (11 as u64).encode_fixed(&mut buf).unwrap(); 142 | } 143 | 144 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 145 | #[tokio::test] 146 | async fn test_async_reader() { 147 | let mut buf = Vec::with_capacity(128); 148 | 149 | let i1: u32 = 1; 150 | let i2: u32 = 65532; 151 | let i3: u32 = 4200123456; 152 | let i4: i64 = i3 as i64 * 1000; 153 | let i5: i32 = -32456; 154 | let i6: i8 = -128; 155 | let i7: u8 = 255; 156 | 157 | buf.write_fixedint_async(i1).await.unwrap(); 158 | buf.write_fixedint_async(i2).await.unwrap(); 159 | buf.write_fixedint_async(i3).await.unwrap(); 160 | buf.write_fixedint_async(i4).await.unwrap(); 161 | buf.write_fixedint_async(i5).await.unwrap(); 162 | buf.write_fixedint_async(i6).await.unwrap(); 163 | buf.write_fixedint_async(i7).await.unwrap(); 164 | 165 | let mut reader: &[u8] = buf.as_ref(); 166 | 167 | assert_eq!(i1, reader.read_fixedint_async().await.unwrap()); 168 | assert_eq!(i2, reader.read_fixedint_async().await.unwrap()); 169 | assert_eq!(i3, reader.read_fixedint_async().await.unwrap()); 170 | assert_eq!(i4, reader.read_fixedint_async().await.unwrap()); 171 | assert_eq!(i5, reader.read_fixedint_async().await.unwrap()); 172 | assert_eq!(i6, reader.read_fixedint_async().await.unwrap()); 173 | assert_eq!(i7, reader.read_fixedint_async().await.unwrap()); 174 | assert!(reader.read_fixedint_async::().await.is_err()); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Fast serialization of integers. 2 | //! 3 | //! This crate implements encoding and decoding of integer types to and from `FixedInt` (i.e. a 4 | //! representation of integers similar or equal to how they are stored in memory) as well as 5 | //! `VarInt` (encoding integers so that they only use as much memory as needed to represent their 6 | //! magnitude). 7 | //! 8 | //! This is useful when (de)serializing data from and to binary representations. For example, 9 | //! Protocol Buffers (by Google) use these kinds of encoding. 10 | //! 11 | //! ``` 12 | //! use integer_encoding::*; 13 | //! 14 | //! fn main() { 15 | //! let a: u32 = 344; 16 | //! let encoded_byte_slice = a.encode_fixed_light(); 17 | //! assert_eq!(Some(a), u32::decode_fixed(&encoded_byte_slice)); 18 | //! assert_eq!(4, encoded_byte_slice.len()); 19 | //! 20 | //! let b: i32 = -111; 21 | //! let encoded_byte_vec = b.encode_var_vec(); 22 | //! assert_eq!(Some((b, 2)), i32::decode_var(&encoded_byte_vec)); 23 | //! } 24 | //! ``` 25 | #[forbid(unsafe_code)] 26 | mod fixed; 27 | mod fixed_tests; 28 | 29 | mod varint; 30 | mod varint_tests; 31 | 32 | mod reader; 33 | mod writer; 34 | 35 | pub use fixed::FixedInt; 36 | pub use varint::VarInt; 37 | 38 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 39 | pub use reader::FixedIntAsyncReader; 40 | pub use reader::FixedIntReader; 41 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 42 | pub use reader::VarIntAsyncReader; 43 | pub use reader::VarIntReader; 44 | 45 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 46 | pub use writer::FixedIntAsyncWriter; 47 | pub use writer::FixedIntWriter; 48 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 49 | pub use writer::VarIntAsyncWriter; 50 | pub use writer::VarIntWriter; 51 | -------------------------------------------------------------------------------- /src/reader.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::io::{Read, Result}; 3 | 4 | use crate::fixed::FixedInt; 5 | use crate::varint::{VarInt, VarIntMaxSize, MSB}; 6 | 7 | #[cfg(feature = "tokio_async")] 8 | use tokio::io::{AsyncRead, AsyncReadExt}; 9 | 10 | #[cfg(feature = "futures_async")] 11 | use futures_util::{io::AsyncRead, io::AsyncReadExt}; 12 | 13 | /// A trait for reading VarInts from any other `Reader`. 14 | /// 15 | /// It's recommended to use a buffered reader, as many small reads will happen. 16 | pub trait VarIntReader { 17 | /// Returns either the decoded integer, or an error. 18 | /// 19 | /// In general, this always reads a whole varint. If the encoded varint's value is bigger 20 | /// than the valid value range of `VI`, then the value is truncated. 21 | /// 22 | /// On EOF, an io::Error with io::ErrorKind::UnexpectedEof is returned. 23 | fn read_varint(&mut self) -> Result; 24 | } 25 | 26 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 27 | /// Like a VarIntReader, but returns a future. 28 | #[async_trait::async_trait(?Send)] 29 | pub trait VarIntAsyncReader { 30 | async fn read_varint_async(&mut self) -> Result; 31 | } 32 | 33 | /// VarIntProcessor encapsulates the logic for decoding a VarInt byte-by-byte. 34 | #[derive(Default)] 35 | pub struct VarIntProcessor { 36 | buf: [u8; 10], 37 | maxsize: usize, 38 | i: usize, 39 | } 40 | 41 | impl VarIntProcessor { 42 | fn new() -> VarIntProcessor { 43 | VarIntProcessor { 44 | maxsize: VI::varint_max_size(), 45 | ..VarIntProcessor::default() 46 | } 47 | } 48 | fn push(&mut self, b: u8) -> Result<()> { 49 | if self.i >= self.maxsize { 50 | return Err(io::Error::new( 51 | io::ErrorKind::InvalidData, 52 | "Unterminated varint", 53 | )); 54 | } 55 | self.buf[self.i] = b; 56 | self.i += 1; 57 | Ok(()) 58 | } 59 | fn finished(&self) -> bool { 60 | self.i > 0 && (self.buf[self.i - 1] & MSB == 0) 61 | } 62 | fn decode(&self) -> Option { 63 | Some(VI::decode_var(&self.buf[0..self.i])?.0) 64 | } 65 | } 66 | 67 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 68 | #[async_trait::async_trait(?Send)] 69 | impl VarIntAsyncReader for AR { 70 | async fn read_varint_async(&mut self) -> Result { 71 | let mut buf = [0_u8; 1]; 72 | let mut p = VarIntProcessor::new::(); 73 | 74 | while !p.finished() { 75 | let read = self.read(&mut buf).await?; 76 | 77 | // EOF 78 | if read == 0 && p.i == 0 { 79 | return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Reached EOF")); 80 | } 81 | if read == 0 { 82 | break; 83 | } 84 | 85 | p.push(buf[0])?; 86 | } 87 | 88 | p.decode() 89 | .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Reached EOF")) 90 | } 91 | } 92 | 93 | impl VarIntReader for R { 94 | fn read_varint(&mut self) -> Result { 95 | let mut buf = [0_u8; 1]; 96 | let mut p = VarIntProcessor::new::(); 97 | 98 | while !p.finished() { 99 | let read = self.read(&mut buf)?; 100 | 101 | // EOF 102 | if read == 0 && p.i == 0 { 103 | return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Reached EOF")); 104 | } 105 | if read == 0 { 106 | break; 107 | } 108 | 109 | p.push(buf[0])?; 110 | } 111 | 112 | p.decode() 113 | .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "Reached EOF")) 114 | } 115 | } 116 | 117 | /// A trait for reading FixedInts from any other `Reader`. 118 | pub trait FixedIntReader { 119 | /// Read a fixed integer from a reader. How many bytes are read depends on `FI`. 120 | /// 121 | /// On EOF, an io::Error with io::ErrorKind::UnexpectedEof is returned. 122 | fn read_fixedint(&mut self) -> Result; 123 | } 124 | 125 | /// Like FixedIntReader, but returns a future. 126 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 127 | #[async_trait::async_trait(?Send)] 128 | pub trait FixedIntAsyncReader { 129 | async fn read_fixedint_async(&mut self) -> Result; 130 | } 131 | 132 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 133 | #[async_trait::async_trait(?Send)] 134 | impl FixedIntAsyncReader for AR { 135 | async fn read_fixedint_async(&mut self) -> Result { 136 | let mut buf = [0_u8; 8]; 137 | self.read_exact(&mut buf[0..std::mem::size_of::()]) 138 | .await?; 139 | Ok(FI::decode_fixed(&buf[0..std::mem::size_of::()]).unwrap()) 140 | } 141 | } 142 | 143 | impl FixedIntReader for R { 144 | fn read_fixedint(&mut self) -> Result { 145 | let mut buf = [0_u8; 8]; 146 | self.read_exact(&mut buf[0..std::mem::size_of::()])?; 147 | Ok(FI::decode_fixed(&buf[0..std::mem::size_of::()]).unwrap()) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/varint.rs: -------------------------------------------------------------------------------- 1 | use std::mem::size_of; 2 | 3 | /// Most-significant byte, == 0x80 4 | pub const MSB: u8 = 0b1000_0000; 5 | /// All bits except for the most significant. Can be used as bitmask to drop the most-signficant 6 | /// bit using `&` (binary-and). 7 | const DROP_MSB: u8 = 0b0111_1111; 8 | 9 | /// How many bytes an integer uses when being encoded as a VarInt. 10 | #[inline] 11 | fn required_encoded_space_unsigned(mut v: u64) -> usize { 12 | if v == 0 { 13 | return 1; 14 | } 15 | 16 | let mut logcounter = 0; 17 | while v > 0 { 18 | logcounter += 1; 19 | v >>= 7; 20 | } 21 | logcounter 22 | } 23 | 24 | /// How many bytes an integer uses when being encoded as a VarInt. 25 | #[inline] 26 | fn required_encoded_space_signed(v: i64) -> usize { 27 | required_encoded_space_unsigned(zigzag_encode(v)) 28 | } 29 | 30 | /// Varint (variable length integer) encoding, as described in 31 | /// https://developers.google.com/protocol-buffers/docs/encoding. 32 | /// 33 | /// Uses zigzag encoding (also described there) for signed integer representation. 34 | pub trait VarInt: Sized + Copy { 35 | /// Returns the number of bytes this number needs in its encoded form. Note: This varies 36 | /// depending on the actual number you want to encode. 37 | fn required_space(self) -> usize; 38 | /// Decode a value from the slice. Returns the value and the number of bytes read from the 39 | /// slice (can be used to read several consecutive values from a big slice) 40 | /// return None if all bytes has MSB set. 41 | fn decode_var(src: &[u8]) -> Option<(Self, usize)>; 42 | /// Encode a value into the slice. The slice must be at least `required_space()` bytes long. 43 | /// The number of bytes taken by the encoded integer is returned. 44 | fn encode_var(self, src: &mut [u8]) -> usize; 45 | 46 | /// Helper: Encode a value and return the encoded form as Vec. The Vec must be at least 47 | /// `required_space()` bytes long. 48 | fn encode_var_vec(self) -> Vec { 49 | let mut v = vec![0; self.required_space()]; 50 | self.encode_var(&mut v); 51 | v 52 | } 53 | } 54 | 55 | #[inline] 56 | fn zigzag_encode(from: i64) -> u64 { 57 | ((from << 1) ^ (from >> 63)) as u64 58 | } 59 | 60 | // see: http://stackoverflow.com/a/2211086/56332 61 | // casting required because operations like unary negation 62 | // cannot be performed on unsigned integers 63 | #[inline] 64 | fn zigzag_decode(from: u64) -> i64 { 65 | ((from >> 1) ^ (-((from & 1) as i64)) as u64) as i64 66 | } 67 | 68 | pub(crate) trait VarIntMaxSize { 69 | fn varint_max_size() -> usize; 70 | } 71 | 72 | impl VarIntMaxSize for VI { 73 | fn varint_max_size() -> usize { 74 | (size_of::() * 8 + 7) / 7 75 | } 76 | } 77 | 78 | macro_rules! impl_varint { 79 | ($t:ty, unsigned) => { 80 | impl VarInt for $t { 81 | fn required_space(self) -> usize { 82 | required_encoded_space_unsigned(self as u64) 83 | } 84 | 85 | fn decode_var(src: &[u8]) -> Option<(Self, usize)> { 86 | let (n, s) = u64::decode_var(src)?; 87 | Some((n as Self, s)) 88 | } 89 | 90 | fn encode_var(self, dst: &mut [u8]) -> usize { 91 | (self as u64).encode_var(dst) 92 | } 93 | } 94 | }; 95 | ($t:ty, signed) => { 96 | impl VarInt for $t { 97 | fn required_space(self) -> usize { 98 | required_encoded_space_signed(self as i64) 99 | } 100 | 101 | fn decode_var(src: &[u8]) -> Option<(Self, usize)> { 102 | let (n, s) = i64::decode_var(src)?; 103 | Some((n as Self, s)) 104 | } 105 | 106 | fn encode_var(self, dst: &mut [u8]) -> usize { 107 | (self as i64).encode_var(dst) 108 | } 109 | } 110 | }; 111 | } 112 | 113 | impl_varint!(usize, unsigned); 114 | impl_varint!(u32, unsigned); 115 | impl_varint!(u16, unsigned); 116 | impl_varint!(u8, unsigned); 117 | 118 | impl_varint!(isize, signed); 119 | impl_varint!(i32, signed); 120 | impl_varint!(i16, signed); 121 | impl_varint!(i8, signed); 122 | 123 | // Below are the "base implementations" doing the actual encodings; all other integer types are 124 | // first cast to these biggest types before being encoded. 125 | 126 | impl VarInt for u64 { 127 | fn required_space(self) -> usize { 128 | required_encoded_space_unsigned(self) 129 | } 130 | 131 | #[inline] 132 | fn decode_var(src: &[u8]) -> Option<(Self, usize)> { 133 | let mut result: u64 = 0; 134 | let mut shift = 0; 135 | 136 | let mut success = false; 137 | for b in src.iter() { 138 | let msb_dropped = b & DROP_MSB; 139 | result |= (msb_dropped as u64) << shift; 140 | shift += 7; 141 | 142 | if b & MSB == 0 || shift > (9 * 7) { 143 | success = b & MSB == 0; 144 | break; 145 | } 146 | } 147 | 148 | if success { 149 | Some((result, shift / 7)) 150 | } else { 151 | None 152 | } 153 | } 154 | 155 | #[inline] 156 | fn encode_var(self, dst: &mut [u8]) -> usize { 157 | debug_assert!(dst.len() >= self.required_space()); 158 | let mut n = self; 159 | let mut i = 0; 160 | 161 | while n >= 0x80 { 162 | dst[i] = MSB | (n as u8); 163 | i += 1; 164 | n >>= 7; 165 | } 166 | 167 | dst[i] = n as u8; 168 | i + 1 169 | } 170 | } 171 | 172 | impl VarInt for i64 { 173 | fn required_space(self) -> usize { 174 | required_encoded_space_signed(self) 175 | } 176 | 177 | #[inline] 178 | fn decode_var(src: &[u8]) -> Option<(Self, usize)> { 179 | if let Some((result, size)) = u64::decode_var(src) { 180 | Some((zigzag_decode(result) as Self, size)) 181 | } else { 182 | None 183 | } 184 | } 185 | 186 | #[inline] 187 | fn encode_var(self, dst: &mut [u8]) -> usize { 188 | debug_assert!(dst.len() >= self.required_space()); 189 | let mut n: u64 = zigzag_encode(self); 190 | let mut i = 0; 191 | 192 | while n >= 0x80 { 193 | dst[i] = MSB | (n as u8); 194 | i += 1; 195 | n >>= 7; 196 | } 197 | 198 | dst[i] = n as u8; 199 | i + 1 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /src/varint_tests.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 4 | use crate::reader::VarIntAsyncReader; 5 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 6 | use crate::writer::VarIntAsyncWriter; 7 | 8 | use crate::reader::VarIntReader; 9 | use crate::varint::VarInt; 10 | use crate::writer::VarIntWriter; 11 | 12 | #[test] 13 | fn test_required_space() { 14 | assert_eq!((0 as u32).required_space(), 1); 15 | assert_eq!((1 as u32).required_space(), 1); 16 | assert_eq!((128 as u32).required_space(), 2); 17 | assert_eq!((16384 as u32).required_space(), 3); 18 | assert_eq!((2097151 as u32).required_space(), 3); 19 | assert_eq!((2097152 as u32).required_space(), 4); 20 | } 21 | 22 | #[test] 23 | fn test_encode_u64() { 24 | assert_eq!((0 as u32).encode_var_vec(), vec![0b00000000]); 25 | assert_eq!((300 as u32).encode_var_vec(), vec![0b10101100, 0b00000010]); 26 | } 27 | 28 | #[test] 29 | fn test_identity_u64() { 30 | for i in 1 as u64..100 { 31 | assert_eq!( 32 | u64::decode_var(i.encode_var_vec().as_slice()).unwrap(), 33 | (i, 1) 34 | ); 35 | } 36 | for i in 16400 as u64..16500 { 37 | assert_eq!( 38 | u64::decode_var(i.encode_var_vec().as_slice()).unwrap(), 39 | (i, 3) 40 | ); 41 | } 42 | } 43 | 44 | #[test] 45 | fn test_decode_max_u64() { 46 | let max_vec_encoded = vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01]; 47 | assert_eq!( 48 | u64::decode_var(max_vec_encoded.as_slice()).unwrap().0, 49 | u64::max_value() 50 | ); 51 | } 52 | 53 | #[test] 54 | fn test_encode_i64() { 55 | assert_eq!((0 as i64).encode_var_vec(), (0 as u32).encode_var_vec()); 56 | assert_eq!((150 as i64).encode_var_vec(), (300 as u32).encode_var_vec()); 57 | assert_eq!( 58 | (-150 as i64).encode_var_vec(), 59 | (299 as u32).encode_var_vec() 60 | ); 61 | assert_eq!( 62 | (-2147483648 as i64).encode_var_vec(), 63 | (4294967295 as u64).encode_var_vec() 64 | ); 65 | assert_eq!( 66 | (i64::max_value() as i64).encode_var_vec(), 67 | &[0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01] 68 | ); 69 | assert_eq!( 70 | (i64::min_value() as i64).encode_var_vec(), 71 | &[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01] 72 | ); 73 | } 74 | 75 | #[test] 76 | fn test_decode_min_i64() { 77 | let min_vec_encoded = vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01]; 78 | assert_eq!( 79 | i64::decode_var(min_vec_encoded.as_slice()).unwrap().0, 80 | i64::min_value() 81 | ); 82 | } 83 | 84 | #[test] 85 | fn test_decode_max_i64() { 86 | let max_vec_encoded = vec![0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01]; 87 | assert_eq!( 88 | i64::decode_var(max_vec_encoded.as_slice()).unwrap().0, 89 | i64::max_value() 90 | ); 91 | } 92 | 93 | #[test] 94 | fn test_encode_i16() { 95 | assert_eq!((150 as i16).encode_var_vec(), (300 as u32).encode_var_vec()); 96 | assert_eq!( 97 | (-150 as i16).encode_var_vec(), 98 | (299 as u32).encode_var_vec() 99 | ); 100 | } 101 | 102 | #[test] 103 | fn test_reader_writer() { 104 | let mut buf = Vec::with_capacity(128); 105 | 106 | let i1: u32 = 1; 107 | let i2: u32 = 65532; 108 | let i3: u32 = 4200123456; 109 | let i4: i64 = i3 as i64 * 1000; 110 | let i5: i32 = -32456; 111 | 112 | assert!(buf.write_varint(i1).is_ok()); 113 | assert!(buf.write_varint(i2).is_ok()); 114 | assert!(buf.write_varint(i3).is_ok()); 115 | assert!(buf.write_varint(i4).is_ok()); 116 | assert!(buf.write_varint(i5).is_ok()); 117 | 118 | let mut reader: &[u8] = buf.as_ref(); 119 | 120 | assert_eq!(i1, reader.read_varint().unwrap()); 121 | assert_eq!(i2, reader.read_varint().unwrap()); 122 | assert_eq!(i3, reader.read_varint().unwrap()); 123 | assert_eq!(i4, reader.read_varint().unwrap()); 124 | assert_eq!(i5, reader.read_varint().unwrap()); 125 | 126 | assert!(reader.read_varint::().is_err()); 127 | } 128 | 129 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 130 | #[tokio::test] 131 | async fn test_async_reader() { 132 | let mut buf = Vec::with_capacity(128); 133 | 134 | let i1: u32 = 1; 135 | let i2: u32 = 65532; 136 | let i3: u32 = 4200123456; 137 | let i4: i64 = i3 as i64 * 1000; 138 | let i5: i32 = -32456; 139 | 140 | buf.write_varint_async(i1).await.unwrap(); 141 | buf.write_varint_async(i2).await.unwrap(); 142 | buf.write_varint_async(i3).await.unwrap(); 143 | buf.write_varint_async(i4).await.unwrap(); 144 | buf.write_varint_async(i5).await.unwrap(); 145 | 146 | let mut reader: &[u8] = buf.as_ref(); 147 | 148 | assert_eq!(i1, reader.read_varint_async().await.unwrap()); 149 | assert_eq!(i2, reader.read_varint_async().await.unwrap()); 150 | assert_eq!(i3, reader.read_varint_async().await.unwrap()); 151 | assert_eq!(i4, reader.read_varint_async().await.unwrap()); 152 | assert_eq!(i5, reader.read_varint_async().await.unwrap()); 153 | assert!(reader.read_varint_async::().await.is_err()); 154 | } 155 | 156 | #[test] 157 | fn test_unterminated_varint() { 158 | let buf = vec![0xff as u8; 12]; 159 | let mut read = buf.as_slice(); 160 | assert!(read.read_varint::().is_err()); 161 | } 162 | 163 | #[test] 164 | fn test_unterminated_varint_2() { 165 | let buf = [0xff, 0xff]; 166 | let mut read = &buf[..]; 167 | assert!(read.read_varint::().is_err()); 168 | } 169 | 170 | #[test] 171 | fn test_decode_extra_bytes_u64() { 172 | let mut encoded = 0x12345u64.encode_var_vec(); 173 | assert_eq!(u64::decode_var(&encoded[..]), Some((0x12345, 3))); 174 | 175 | encoded.push(0x99); 176 | assert_eq!(u64::decode_var(&encoded[..]), Some((0x12345, 3))); 177 | 178 | let encoded = [0xFF, 0xFF, 0xFF]; 179 | assert_eq!(u64::decode_var(&encoded[..]), None); 180 | 181 | // Overflow 182 | let mut encoded = vec![0xFF; 64]; 183 | encoded.push(0x00); 184 | assert_eq!(u64::decode_var(&encoded[..]), None); 185 | } 186 | 187 | #[test] 188 | fn test_decode_extra_bytes_i64() { 189 | let mut encoded = (-0x12345i64).encode_var_vec(); 190 | assert_eq!(i64::decode_var(&encoded[..]), Some((-0x12345, 3))); 191 | 192 | encoded.push(0x99); 193 | assert_eq!(i64::decode_var(&encoded[..]), Some((-0x12345, 3))); 194 | 195 | let encoded = [0xFF, 0xFF, 0xFF]; 196 | assert_eq!(i64::decode_var(&encoded[..]), None); 197 | 198 | // Overflow 199 | let mut encoded = vec![0xFF; 64]; 200 | encoded.push(0x00); 201 | assert_eq!(i64::decode_var(&encoded[..]), None); 202 | } 203 | 204 | #[test] 205 | fn test_regression_22() { 206 | let encoded: Vec = (0x112233 as u64).encode_var_vec(); 207 | assert_eq!( 208 | encoded.as_slice().read_varint::().unwrap_err().kind(), 209 | std::io::ErrorKind::InvalidData 210 | ); 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/writer.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Result, Write}; 2 | 3 | use crate::fixed::FixedInt; 4 | use crate::varint::VarInt; 5 | 6 | #[cfg(feature = "tokio_async")] 7 | use tokio::io::{AsyncWrite, AsyncWriteExt}; 8 | 9 | #[cfg(feature = "futures_async")] 10 | use futures_util::{io::AsyncWrite, io::AsyncWriteExt}; 11 | 12 | /// A trait for writing integers in VarInt encoding to any `Write` type. This packs encoding and 13 | /// writing into one step. 14 | pub trait VarIntWriter { 15 | fn write_varint(&mut self, n: VI) -> Result; 16 | } 17 | 18 | /// Like VarIntWriter, but asynchronous. 19 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 20 | #[async_trait::async_trait(?Send)] 21 | pub trait VarIntAsyncWriter { 22 | /// Write a VarInt integer to an asynchronous writer. 23 | async fn write_varint_async(&mut self, n: VI) -> Result; 24 | } 25 | 26 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 27 | #[async_trait::async_trait(?Send)] 28 | impl VarIntAsyncWriter for AW { 29 | async fn write_varint_async(&mut self, n: VI) -> Result { 30 | let mut buf = [0_u8; 10]; 31 | let b = n.encode_var(&mut buf); 32 | self.write_all(&buf[0..b]).await?; 33 | Ok(b) 34 | } 35 | } 36 | 37 | impl VarIntWriter for Inner { 38 | fn write_varint(&mut self, n: VI) -> Result { 39 | let mut buf = [0_u8; 10]; 40 | let used = n.encode_var(&mut buf[..]); 41 | 42 | self.write_all(&buf[0..used])?; 43 | Ok(used) 44 | } 45 | } 46 | 47 | /// A trait for writing integers without encoding (i.e. `FixedInt`) to any `Write` type. 48 | pub trait FixedIntWriter { 49 | fn write_fixedint(&mut self, n: FI) -> Result; 50 | } 51 | 52 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 53 | #[async_trait::async_trait(?Send)] 54 | pub trait FixedIntAsyncWriter { 55 | async fn write_fixedint_async(&mut self, n: FI) -> Result; 56 | } 57 | 58 | #[cfg(any(feature = "tokio_async", feature = "futures_async"))] 59 | #[async_trait::async_trait(?Send)] 60 | impl FixedIntAsyncWriter for AW { 61 | async fn write_fixedint_async(&mut self, n: FI) -> Result { 62 | let mut buf = [0_u8; 8]; 63 | n.encode_fixed(&mut buf[..std::mem::size_of::()]); 64 | self.write_all(&buf[..std::mem::size_of::()]).await?; 65 | Ok(std::mem::size_of::()) 66 | } 67 | } 68 | 69 | impl FixedIntWriter for W { 70 | fn write_fixedint(&mut self, n: FI) -> Result { 71 | let mut buf = [0_u8; 8]; 72 | n.encode_fixed(&mut buf[..std::mem::size_of::()]); 73 | 74 | self.write_all(&buf[..std::mem::size_of::()])?; 75 | Ok(std::mem::size_of::()) 76 | } 77 | } 78 | --------------------------------------------------------------------------------