├── .gitignore ├── .travis.yml ├── src ├── codec │ ├── mod.rs │ ├── bytes.rs │ ├── lines.rs │ ├── length.rs │ ├── json.rs │ └── cbor.rs ├── encoder.rs ├── lib.rs ├── fuse.rs ├── decoder.rs ├── framed.rs ├── framed_read.rs └── framed_write.rs ├── tests ├── bytes.rs ├── lines.rs ├── length_delimited.rs ├── framed_read.rs └── framed_write.rs ├── README.md ├── LICENSE ├── Cargo.toml └── benches └── lines.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - stable 4 | - beta 5 | - nightly 6 | -------------------------------------------------------------------------------- /src/codec/mod.rs: -------------------------------------------------------------------------------- 1 | mod bytes; 2 | pub use self::bytes::BytesCodec; 3 | 4 | mod length; 5 | pub use self::length::LengthCodec; 6 | 7 | mod lines; 8 | pub use self::lines::LinesCodec; 9 | 10 | #[cfg(feature = "json")] 11 | mod json; 12 | #[cfg(feature = "json")] 13 | pub use self::json::{JsonCodec, JsonCodecError}; 14 | 15 | #[cfg(feature = "cbor")] 16 | mod cbor; 17 | #[cfg(feature = "cbor")] 18 | pub use self::cbor::{CborCodec, CborCodecError}; 19 | -------------------------------------------------------------------------------- /tests/bytes.rs: -------------------------------------------------------------------------------- 1 | use futures::io::Cursor; 2 | use futures::{executor, TryStreamExt}; 3 | use futures_codec::{BytesCodec, Framed}; 4 | 5 | #[test] 6 | fn decodes() { 7 | let mut buf = [0u8; 32]; 8 | let expected = buf; 9 | let cur = Cursor::new(&mut buf[..]); 10 | let mut framed = Framed::new(cur, BytesCodec {}); 11 | 12 | let read = executor::block_on(framed.try_next()).unwrap().unwrap(); 13 | assert_eq!(&read[..], &expected[..]); 14 | 15 | assert!(executor::block_on(framed.try_next()).unwrap().is_none()); 16 | } 17 | -------------------------------------------------------------------------------- /tests/lines.rs: -------------------------------------------------------------------------------- 1 | use futures::io::Cursor; 2 | use futures::{executor, TryStreamExt}; 3 | use futures_codec::{FramedRead, LinesCodec}; 4 | 5 | #[test] 6 | fn it_works() { 7 | let buf = "Hello\nWorld\nError".to_owned(); 8 | let cur = Cursor::new(buf); 9 | 10 | let mut framed = FramedRead::new(cur, LinesCodec {}); 11 | let next = executor::block_on(framed.try_next()).unwrap().unwrap(); 12 | assert_eq!(next, "Hello\n"); 13 | let next = executor::block_on(framed.try_next()).unwrap().unwrap(); 14 | assert_eq!(next, "World\n"); 15 | 16 | assert!(executor::block_on(framed.try_next()).is_err()); 17 | } 18 | -------------------------------------------------------------------------------- /src/encoder.rs: -------------------------------------------------------------------------------- 1 | use super::fuse::Fuse; 2 | use bytes::BytesMut; 3 | use std::io::Error; 4 | 5 | /// Encoding of messages as bytes, for use with `FramedWrite`. 6 | pub trait Encoder { 7 | /// The type of items consumed by `encode` 8 | type Item; 9 | /// The type of encoding errors. 10 | type Error: From; 11 | 12 | /// Encodes an item into the `BytesMut` provided by dst. 13 | fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error>; 14 | } 15 | 16 | impl Encoder for Fuse { 17 | type Item = U::Item; 18 | type Error = U::Error; 19 | 20 | fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> { 21 | self.u.encode(item, dst) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # futures_codec 2 | 3 | Utilities for encoding and decoding frames using async/await. 4 | 5 | Contains adapters to go from streams of bytes, `AsyncRead` and `AsyncWrite`, 6 | to framed streams implementing `Sink` and `Stream`. Framed streams are also known as transports. 7 | 8 | [![Latest Version](https://img.shields.io/crates/v/futures-codec.svg)](https://crates.io/crates/futures-codec) 9 | [![Rust Documentation](https://img.shields.io/badge/api-rustdoc-blue.svg)](https://docs.rs/futures-codec) 10 | [![Build Status](https://travis-ci.com/matthunz/futures-codec.svg)](https://travis-ci.com/matthunz/futures-codec) 11 | ![LICENSE](https://img.shields.io/badge/license-MIT-blue.svg) 12 | 13 | 14 | ### Example 15 | ```rust 16 | use futures_codec::{LinesCodec, Framed}; 17 | 18 | async fn main() { 19 | // let stream = ... 20 | let mut framed = Framed::new(stream, LinesCodec {}); 21 | 22 | while let Some(line) = framed.try_next().await.unwrap() { 23 | println!("{:?}", line); 24 | } 25 | } 26 | ``` 27 | -------------------------------------------------------------------------------- /tests/length_delimited.rs: -------------------------------------------------------------------------------- 1 | use futures::io::Cursor; 2 | use futures::{executor, SinkExt, StreamExt}; 3 | use futures_codec::{Bytes, Framed, LengthCodec}; 4 | 5 | #[test] 6 | fn same_msgs_are_received_as_were_sent() { 7 | let cur = Cursor::new(vec![0; 256]); 8 | let mut framed = Framed::new(cur, LengthCodec {}); 9 | 10 | let send_msgs = async { 11 | framed.send(Bytes::from("msg1")).await.unwrap(); 12 | framed.send(Bytes::from("msg2")).await.unwrap(); 13 | framed.send(Bytes::from("msg3")).await.unwrap(); 14 | }; 15 | executor::block_on(send_msgs); 16 | 17 | let (mut cur, _) = framed.release(); 18 | cur.set_position(0); 19 | let framed = Framed::new(cur, LengthCodec {}); 20 | 21 | let recv_msgs = framed 22 | .take(3) 23 | .map(|res| res.unwrap()) 24 | .map(|buf| String::from_utf8(buf.to_vec()).unwrap()) 25 | .collect::>(); 26 | let msgs: Vec = executor::block_on(recv_msgs); 27 | 28 | assert!(msgs == vec!["msg1", "msg2", "msg3"]); 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Matt Hunzinger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "futures_codec" 3 | edition = "2018" 4 | version = "0.4.1" 5 | authors = ["Matt Hunzinger "] 6 | description = "Utilities for encoding and decoding frames using `async/await`" 7 | license = "MIT" 8 | readme = "README.md" 9 | repository = "https://github.com/matthunz/futures-codec" 10 | homepage = "https://github.com/matthunz/futures-codec" 11 | documentation = "https://docs.rs/crate/futures_codec" 12 | keywords = ["future", "futures", "async", "codec"] 13 | categories = ["asynchronous", "network-programming"] 14 | 15 | [features] 16 | default = [] 17 | json = [ "serde", "serde_json" ] 18 | cbor = [ "serde", "serde_cbor" ] 19 | 20 | [dependencies] 21 | bytes = "0.5.4" 22 | futures-sink = "0.3.7" 23 | futures-util = { version = "0.3.7", features = ["io"] } 24 | memchr = "2.3.4" 25 | pin-project-lite = "0.1.11" 26 | 27 | [dev-dependencies] 28 | futures = "0.3.7" 29 | 30 | [dependencies.serde] 31 | version = '1.0.113' 32 | optional = true 33 | features = [ "derive" ] 34 | 35 | [dependencies.serde_json] 36 | version = '1.0.55' 37 | optional = true 38 | 39 | [dependencies.serde_cbor] 40 | version = '0.11.1' 41 | optional = true 42 | 43 | [package.metadata.docs.rs] 44 | all-features = true 45 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! Utilities for encoding and decoding frames using `async/await`. 3 | //! 4 | //! Contains adapters to go from streams of bytes, [`AsyncRead`](futures::io::AsyncRead) 5 | //! and [`AsyncWrite`](futures::io::AsyncWrite), to framed streams implementing [`Sink`](futures::Sink) and [`Stream`](futures::Stream). 6 | //! Framed streams are also known as `transports`. 7 | //! 8 | //! ``` 9 | //! # futures::executor::block_on(async move { 10 | //! use futures::TryStreamExt; 11 | //! use futures::io::Cursor; 12 | //! use futures_codec::{LinesCodec, Framed}; 13 | //! 14 | //! let io = Cursor::new(Vec::new()); 15 | //! let mut framed = Framed::new(io, LinesCodec); 16 | //! 17 | //! while let Some(line) = framed.try_next().await? { 18 | //! dbg!(line); 19 | //! } 20 | //! # Ok::<_, std::io::Error>(()) 21 | //! # }).unwrap(); 22 | //! ``` 23 | 24 | mod codec; 25 | pub use bytes::{Bytes, BytesMut}; 26 | pub use codec::{BytesCodec, LengthCodec, LinesCodec}; 27 | 28 | #[cfg(feature = "cbor")] 29 | pub use codec::{CborCodec, CborCodecError}; 30 | #[cfg(feature = "json")] 31 | pub use codec::{JsonCodec, JsonCodecError}; 32 | 33 | mod decoder; 34 | pub use decoder::Decoder; 35 | 36 | mod encoder; 37 | pub use encoder::Encoder; 38 | 39 | mod framed; 40 | pub use framed::Framed; 41 | 42 | mod framed_read; 43 | pub use framed_read::FramedRead; 44 | 45 | mod framed_write; 46 | pub use framed_write::FramedWrite; 47 | 48 | mod fuse; 49 | -------------------------------------------------------------------------------- /src/codec/bytes.rs: -------------------------------------------------------------------------------- 1 | use crate::{Decoder, Encoder}; 2 | use bytes::{Bytes, BytesMut}; 3 | use std::io::Error; 4 | 5 | /// A simple codec that ships bytes around 6 | /// 7 | /// # Example 8 | /// 9 | /// ``` 10 | /// # futures::executor::block_on(async move { 11 | /// use bytes::Bytes; 12 | /// use futures::{SinkExt, TryStreamExt}; 13 | /// use futures::io::Cursor; 14 | /// use futures_codec::{BytesCodec, Framed}; 15 | /// 16 | /// let mut buf = vec![]; 17 | /// // Cursor implements AsyncRead and AsyncWrite 18 | /// let cur = Cursor::new(&mut buf); 19 | /// let mut framed = Framed::new(cur, BytesCodec); 20 | /// 21 | /// framed.send(Bytes::from("Hello World!")).await?; 22 | /// 23 | /// while let Some(bytes) = framed.try_next().await? { 24 | /// dbg!(bytes); 25 | /// } 26 | /// # Ok::<_, std::io::Error>(()) 27 | /// # }).unwrap(); 28 | /// ``` 29 | pub struct BytesCodec; 30 | 31 | impl Encoder for BytesCodec { 32 | type Item = Bytes; 33 | type Error = Error; 34 | 35 | fn encode(&mut self, src: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> { 36 | dst.extend_from_slice(&src); 37 | Ok(()) 38 | } 39 | } 40 | 41 | impl Decoder for BytesCodec { 42 | type Item = Bytes; 43 | type Error = Error; 44 | 45 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 46 | let len = src.len(); 47 | if len > 0 { 48 | Ok(Some(src.split_to(len).freeze())) 49 | } else { 50 | Ok(None) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/codec/lines.rs: -------------------------------------------------------------------------------- 1 | use crate::{Decoder, Encoder}; 2 | use bytes::{BufMut, BytesMut}; 3 | use memchr::memchr; 4 | use std::io::{Error, ErrorKind}; 5 | 6 | /// A simple `Codec` implementation that splits up data into lines. 7 | /// 8 | /// ```rust 9 | /// # futures::executor::block_on(async move { 10 | /// use futures::stream::TryStreamExt; // for lines.try_next() 11 | /// use futures_codec::{FramedRead, LinesCodec}; 12 | /// 13 | /// let input = "hello\nworld\nthis\nis\ndog\n".as_bytes(); 14 | /// let mut lines = FramedRead::new(input, LinesCodec); 15 | /// while let Some(line) = lines.try_next().await? { 16 | /// println!("{}", line); 17 | /// } 18 | /// # Ok::<_, std::io::Error>(()) 19 | /// # }).unwrap(); 20 | /// ``` 21 | pub struct LinesCodec; 22 | 23 | impl Encoder for LinesCodec { 24 | type Item = String; 25 | type Error = Error; 26 | 27 | fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> { 28 | dst.reserve(item.len()); 29 | dst.put(item.as_bytes()); 30 | Ok(()) 31 | } 32 | } 33 | 34 | impl Decoder for LinesCodec { 35 | type Item = String; 36 | type Error = Error; 37 | 38 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 39 | match memchr(b'\n', src) { 40 | Some(pos) => { 41 | let buf = src.split_to(pos + 1); 42 | String::from_utf8(buf.to_vec()) 43 | .map(Some) 44 | .map_err(|e| Error::new(ErrorKind::InvalidData, e)) 45 | } 46 | _ => Ok(None), 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/fuse.rs: -------------------------------------------------------------------------------- 1 | use futures_util::io::{AsyncRead, AsyncWrite}; 2 | use pin_project_lite::pin_project; 3 | use std::io::Error; 4 | use std::marker::Unpin; 5 | use std::ops::{Deref, DerefMut}; 6 | use std::pin::Pin; 7 | use std::task::{Context, Poll}; 8 | 9 | pin_project! { 10 | #[derive(Debug)] 11 | pub(crate) struct Fuse { 12 | #[pin] 13 | pub t: T, 14 | pub u: U, 15 | } 16 | } 17 | 18 | impl Fuse { 19 | pub(crate) fn new(t: T, u: U) -> Self { 20 | Self { t, u } 21 | } 22 | } 23 | 24 | impl Deref for Fuse { 25 | type Target = T; 26 | 27 | fn deref(&self) -> &T { 28 | &self.t 29 | } 30 | } 31 | 32 | impl DerefMut for Fuse { 33 | fn deref_mut(&mut self) -> &mut T { 34 | &mut self.t 35 | } 36 | } 37 | 38 | impl AsyncRead for Fuse { 39 | fn poll_read( 40 | self: Pin<&mut Self>, 41 | cx: &mut Context<'_>, 42 | buf: &mut [u8], 43 | ) -> Poll> { 44 | self.project().t.poll_read(cx, buf) 45 | } 46 | } 47 | 48 | impl AsyncWrite for Fuse { 49 | fn poll_write( 50 | self: Pin<&mut Self>, 51 | cx: &mut Context, 52 | buf: &[u8], 53 | ) -> Poll> { 54 | self.project().t.poll_write(cx, buf) 55 | } 56 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 57 | self.project().t.poll_flush(cx) 58 | } 59 | fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 60 | self.project().t.poll_close(cx) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/decoder.rs: -------------------------------------------------------------------------------- 1 | use super::framed_write::FramedWrite2; 2 | use super::fuse::Fuse; 3 | use bytes::BytesMut; 4 | use std::io::Error; 5 | 6 | /// Decoding of frames via buffers, for use with `FramedRead`. 7 | pub trait Decoder { 8 | /// The type of items returned by `decode` 9 | type Item; 10 | /// The type of decoding errors. 11 | type Error: From; 12 | 13 | /// Decode an item from the src `BytesMut` into an item 14 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error>; 15 | 16 | /// Called when the input stream reaches EOF, signaling a last attempt to decode 17 | /// 18 | /// # Notes 19 | /// 20 | /// The default implementation of this method invokes the `Decoder::decode` method. 21 | fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 22 | self.decode(src) 23 | } 24 | } 25 | 26 | impl Decoder for Fuse { 27 | type Item = U::Item; 28 | type Error = U::Error; 29 | 30 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 31 | self.u.decode(src) 32 | } 33 | 34 | fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 35 | self.u.decode_eof(src) 36 | } 37 | } 38 | 39 | impl Decoder for FramedWrite2 { 40 | type Item = T::Item; 41 | type Error = T::Error; 42 | 43 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 44 | self.inner.decode(src) 45 | } 46 | 47 | fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 48 | self.inner.decode_eof(src) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /benches/lines.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | 3 | extern crate test; 4 | 5 | use futures::{executor, io::Cursor, TryStreamExt}; 6 | use futures_codec::{FramedRead, LinesCodec}; 7 | 8 | #[bench] 9 | fn short(b: &mut test::Bencher) { 10 | let data = [ 11 | ["a"; 16].join("b"), 12 | ["b"; 16].join("c"), 13 | ["c"; 16].join("d"), 14 | ] 15 | .join("\n"); 16 | b.iter(|| { 17 | executor::block_on(async { 18 | let read = Cursor::new(test::black_box(&data)); 19 | let mut framed = FramedRead::new(read, LinesCodec {}); 20 | 21 | framed.try_next().await.unwrap(); 22 | framed.try_next().await.unwrap(); 23 | framed.try_next().await.is_ok() 24 | }) 25 | }) 26 | } 27 | 28 | #[bench] 29 | fn medium(b: &mut test::Bencher) { 30 | let data = [ 31 | ["a"; 128].join("b"), 32 | ["b"; 128].join("c"), 33 | ["c"; 128].join("d"), 34 | ] 35 | .join("\n"); 36 | b.iter(|| { 37 | executor::block_on(async { 38 | let read = Cursor::new(test::black_box(&data)); 39 | let mut framed = FramedRead::new(read, LinesCodec {}); 40 | 41 | framed.try_next().await.unwrap(); 42 | framed.try_next().await.unwrap(); 43 | framed.try_next().await.is_ok() 44 | }) 45 | }) 46 | } 47 | 48 | #[bench] 49 | fn long(b: &mut test::Bencher) { 50 | let data = [ 51 | ["a"; 2048].join("b"), 52 | ["b"; 2048].join("c"), 53 | ["c"; 2048].join("d"), 54 | ] 55 | .join("\n"); 56 | b.iter(|| { 57 | executor::block_on(async { 58 | let read = Cursor::new(test::black_box(&data)); 59 | let mut framed = FramedRead::new(read, LinesCodec {}); 60 | 61 | framed.try_next().await.unwrap(); 62 | framed.try_next().await.unwrap(); 63 | framed.try_next().await.is_ok() 64 | }) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /tests/framed_read.rs: -------------------------------------------------------------------------------- 1 | use futures::executor; 2 | use futures::stream::StreamExt; 3 | use futures::AsyncRead; 4 | use futures_codec::{BytesMut, Decoder, FramedRead, LinesCodec}; 5 | use std::io; 6 | use std::pin::Pin; 7 | use std::task::{Context, Poll}; 8 | 9 | // Sends two lines at once, then nothing else forever 10 | struct MockBurstySender { 11 | sent: bool, 12 | } 13 | impl AsyncRead for MockBurstySender { 14 | fn poll_read( 15 | mut self: Pin<&mut Self>, 16 | _cx: &mut Context<'_>, 17 | buf: &mut [u8], 18 | ) -> Poll> { 19 | const MESSAGES: &[u8] = b"one\ntwo\n"; 20 | if !self.sent && buf.len() >= MESSAGES.len() { 21 | self.sent = true; 22 | buf[0..MESSAGES.len()].clone_from_slice(MESSAGES); 23 | Poll::Ready(Ok(MESSAGES.len())) 24 | } else { 25 | Poll::Pending 26 | } 27 | } 28 | } 29 | 30 | #[test] 31 | fn line_read_multi() { 32 | let io = MockBurstySender { sent: false }; 33 | let mut framed = FramedRead::new(io, LinesCodec {}); 34 | let one = executor::block_on(framed.next()).unwrap().unwrap(); 35 | assert_eq!(one, "one\n"); 36 | let two = executor::block_on(framed.next()).unwrap().unwrap(); 37 | assert_eq!(two, "two\n"); 38 | } 39 | 40 | struct OneByteAtATime<'a> { 41 | input: &'a [u8], 42 | } 43 | impl AsyncRead for OneByteAtATime<'_> { 44 | fn poll_read( 45 | mut self: Pin<&mut Self>, 46 | _cx: &mut Context<'_>, 47 | buf: &mut [u8], 48 | ) -> Poll> { 49 | if self.input.is_empty() { 50 | Poll::Ready(Ok(0)) 51 | } else { 52 | buf[0] = self.input[0]; 53 | self.input = &self.input[1..]; 54 | Poll::Ready(Ok(1)) 55 | } 56 | } 57 | } 58 | 59 | /// A decoder that only returns `a` characters from the input. 60 | struct AllTheAs; 61 | 62 | impl Decoder for AllTheAs { 63 | type Item = char; 64 | type Error = io::Error; 65 | 66 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 67 | while !src.is_empty() { 68 | let buf = src.split_to(1); 69 | let c = char::from(buf[0]); 70 | if c == 'a' { 71 | return Ok(Some(c)); 72 | } 73 | } 74 | Ok(None) 75 | } 76 | } 77 | 78 | #[test] 79 | fn read_few_messages() { 80 | let string: &[u8] = b"aabbbabbbabbbabb"; 81 | let input = OneByteAtATime { input: string }; 82 | let mut framed = FramedRead::new(input, AllTheAs); 83 | for _ in 0..5 { 84 | let item = executor::block_on(framed.next()).unwrap().unwrap(); 85 | assert_eq!(item, 'a'); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/codec/length.rs: -------------------------------------------------------------------------------- 1 | use crate::{Decoder, Encoder}; 2 | use bytes::{Buf, BufMut, Bytes, BytesMut}; 3 | use std::io::Error; 4 | 5 | const U64_LENGTH: usize = std::mem::size_of::(); 6 | 7 | /// A simple `Codec` implementation sending your data by prefixing it by its length. 8 | /// 9 | /// # Example 10 | /// 11 | /// This codec will most likely be used wrapped in another codec like so. 12 | /// 13 | /// ``` 14 | /// use futures_codec::{Decoder, Encoder, LengthCodec}; 15 | /// use bytes::{Bytes, BytesMut}; 16 | /// use std::io::{Error, ErrorKind}; 17 | /// 18 | /// pub struct MyStringCodec(LengthCodec); 19 | /// 20 | /// impl Encoder for MyStringCodec { 21 | /// type Item = String; 22 | /// type Error = Error; 23 | /// 24 | /// fn encode(&mut self, src: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> { 25 | /// let bytes = Bytes::from(src); 26 | /// self.0.encode(bytes, dst) 27 | /// } 28 | /// } 29 | /// 30 | /// impl Decoder for MyStringCodec { 31 | /// type Item = String; 32 | /// type Error = Error; 33 | /// 34 | /// fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 35 | /// match self.0.decode(src)? { 36 | /// Some(bytes) => { 37 | /// match String::from_utf8(bytes.to_vec()) { 38 | /// Ok(string) => Ok(Some(string)), 39 | /// Err(e) => Err(Error::new(ErrorKind::InvalidData, e)) 40 | /// } 41 | /// }, 42 | /// None => Ok(None), 43 | /// } 44 | /// } 45 | /// } 46 | /// ``` 47 | pub struct LengthCodec; 48 | 49 | impl Encoder for LengthCodec { 50 | type Item = Bytes; 51 | type Error = Error; 52 | 53 | fn encode(&mut self, src: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> { 54 | dst.reserve(U64_LENGTH + src.len()); 55 | dst.put_u64(src.len() as u64); 56 | dst.extend_from_slice(&src); 57 | Ok(()) 58 | } 59 | } 60 | 61 | impl Decoder for LengthCodec { 62 | type Item = Bytes; 63 | type Error = Error; 64 | 65 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 66 | if src.len() < U64_LENGTH { 67 | return Ok(None); 68 | } 69 | 70 | let mut len_bytes = [0u8; U64_LENGTH]; 71 | len_bytes.copy_from_slice(&src[..U64_LENGTH]); 72 | let len = u64::from_be_bytes(len_bytes) as usize; 73 | 74 | if src.len() - U64_LENGTH >= len { 75 | // Skip the length header we already read. 76 | src.advance(U64_LENGTH); 77 | Ok(Some(src.split_to(len).freeze())) 78 | } else { 79 | Ok(None) 80 | } 81 | } 82 | } 83 | 84 | #[cfg(test)] 85 | mod tests { 86 | use super::*; 87 | 88 | mod decode { 89 | use super::*; 90 | 91 | #[test] 92 | fn it_returns_bytes_withouth_length_header() { 93 | let mut codec = LengthCodec {}; 94 | 95 | let mut src = BytesMut::with_capacity(5); 96 | src.put(&[0, 0, 0, 0, 0, 0, 0, 3u8, 1, 2, 3, 4][..]); 97 | let item = codec.decode(&mut src).unwrap(); 98 | 99 | assert!(item == Some(Bytes::from(&[1u8, 2, 3][..]))); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /tests/framed_write.rs: -------------------------------------------------------------------------------- 1 | use core::iter::Iterator; 2 | use futures::io::{AsyncWrite, Cursor}; 3 | use futures::sink::SinkExt; 4 | use futures::{executor, stream, stream::StreamExt}; 5 | use futures_codec::{Bytes, BytesCodec, FramedWrite, LinesCodec}; 6 | use std::pin::Pin; 7 | use std::task::{Context, Poll}; 8 | 9 | // An iterator which outputs a single zero byte up to limit times 10 | struct ZeroBytes { 11 | pub count: usize, 12 | pub limit: usize, 13 | } 14 | impl Iterator for ZeroBytes { 15 | type Item = Bytes; 16 | fn next(&mut self) -> Option { 17 | if self.count >= self.limit { 18 | None 19 | } else { 20 | self.count += 1; 21 | Some(Bytes::from_static(b"\0")) 22 | } 23 | } 24 | } 25 | 26 | // An AsyncWrite which is always ready and just consumes the data 27 | struct AsyncWriteNull { 28 | // number of poll_write calls 29 | pub num_poll_write: usize, 30 | 31 | // size of the last poll_write 32 | pub last_write_size: usize, 33 | } 34 | impl AsyncWrite for AsyncWriteNull { 35 | fn poll_write( 36 | mut self: Pin<&mut Self>, 37 | _cx: &mut Context<'_>, 38 | buf: &[u8], 39 | ) -> Poll> { 40 | self.num_poll_write += 1; 41 | self.last_write_size = buf.len(); 42 | Poll::Ready(Ok(buf.len())) 43 | } 44 | 45 | fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 46 | Poll::Ready(Ok(())) 47 | } 48 | 49 | fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 50 | Poll::Ready(Ok(())) 51 | } 52 | } 53 | 54 | #[test] 55 | fn line_write() { 56 | let curs = Cursor::new(vec![0u8; 16]); 57 | let mut framer = FramedWrite::new(curs, LinesCodec {}); 58 | executor::block_on(framer.send("Hello\n".to_owned())).unwrap(); 59 | executor::block_on(framer.send("World\n".to_owned())).unwrap(); 60 | let (curs, _) = framer.release(); 61 | assert_eq!(&curs.get_ref()[0..12], b"Hello\nWorld\n"); 62 | assert_eq!(curs.position(), 12); 63 | } 64 | 65 | #[test] 66 | fn line_write_to_eof() { 67 | let mut buf = [0u8; 16]; 68 | let curs = Cursor::new(&mut buf[..]); 69 | let mut framer = FramedWrite::new(curs, LinesCodec {}); 70 | let _err = 71 | executor::block_on(framer.send("This will fill up the buffer\n".to_owned())).unwrap_err(); 72 | let (curs, _) = framer.release(); 73 | assert_eq!(curs.position(), 16); 74 | assert_eq!(&curs.get_ref()[0..16], b"This will fill u"); 75 | } 76 | 77 | #[test] 78 | fn send_high_water_mark() { 79 | // stream will output 999 bytes, 1 at at a time, and will always be ready 80 | let mut stream = stream::iter(ZeroBytes { 81 | count: 0, 82 | limit: 999, 83 | }) 84 | .map(Ok); 85 | 86 | // sink will eat whatever it receives 87 | let io = AsyncWriteNull { 88 | num_poll_write: 0, 89 | last_write_size: 0, 90 | }; 91 | 92 | // expect two sends 93 | let mut framer = FramedWrite::new(io, BytesCodec {}); 94 | framer.set_send_high_water_mark(500); 95 | executor::block_on(framer.send_all(&mut stream)).unwrap(); 96 | let (io, _) = framer.release(); 97 | assert_eq!(io.num_poll_write, 2); 98 | assert_eq!(io.last_write_size, 499); 99 | } 100 | -------------------------------------------------------------------------------- /src/framed.rs: -------------------------------------------------------------------------------- 1 | use super::framed_read::{framed_read_2, FramedRead2}; 2 | use super::framed_write::{framed_write_2, FramedWrite2}; 3 | use super::fuse::Fuse; 4 | use super::{Decoder, Encoder}; 5 | use bytes::BytesMut; 6 | use futures_sink::Sink; 7 | use futures_util::io::{AsyncRead, AsyncWrite}; 8 | use futures_util::stream::{Stream, TryStreamExt}; 9 | use pin_project_lite::pin_project; 10 | use std::marker::Unpin; 11 | use std::ops::{Deref, DerefMut}; 12 | use std::pin::Pin; 13 | use std::task::{Context, Poll}; 14 | 15 | pin_project! { 16 | /// A unified `Stream` and `Sink` interface to an underlying I/O object, 17 | /// using the `Encoder` and `Decoder` traits to encode and decode frames. 18 | /// 19 | /// # Example 20 | /// ``` 21 | /// use bytes::Bytes; 22 | /// use futures::{SinkExt, TryStreamExt}; 23 | /// use futures::io::Cursor; 24 | /// use futures_codec::{BytesCodec, Framed}; 25 | /// 26 | /// # futures::executor::block_on(async move { 27 | /// let cur = Cursor::new(vec![0u8; 12]); 28 | /// let mut framed = Framed::new(cur, BytesCodec {}); 29 | /// 30 | /// // Send bytes to `buf` through the `BytesCodec` 31 | /// let bytes = Bytes::from("Hello world!"); 32 | /// framed.send(bytes).await?; 33 | /// 34 | /// // Release the I/O and codec 35 | /// let (cur, _) = framed.release(); 36 | /// assert_eq!(cur.get_ref(), b"Hello world!"); 37 | /// # Ok::<_, std::io::Error>(()) 38 | /// # }).unwrap(); 39 | /// ``` 40 | #[derive(Debug)] 41 | pub struct Framed { 42 | #[pin] 43 | inner: FramedRead2>>, 44 | } 45 | } 46 | 47 | impl Deref for Framed { 48 | type Target = T; 49 | 50 | fn deref(&self) -> &T { 51 | &self.inner 52 | } 53 | } 54 | 55 | impl DerefMut for Framed { 56 | fn deref_mut(&mut self) -> &mut T { 57 | &mut self.inner 58 | } 59 | } 60 | 61 | impl Framed 62 | where 63 | T: AsyncRead + AsyncWrite, 64 | U: Decoder + Encoder, 65 | { 66 | /// Creates a new `Framed` transport with the given codec. 67 | /// A codec is a type which implements `Decoder` and `Encoder`. 68 | pub fn new(inner: T, codec: U) -> Self { 69 | Self { 70 | inner: framed_read_2(framed_write_2(Fuse::new(inner, codec))), 71 | } 72 | } 73 | 74 | /// Release the I/O and Codec 75 | pub fn release(self) -> (T, U) { 76 | let fuse = self.inner.release().release(); 77 | (fuse.t, fuse.u) 78 | } 79 | 80 | /// Consumes the `Framed`, returning its underlying I/O stream. 81 | /// 82 | /// Note that care should be taken to not tamper with the underlying stream 83 | /// of data coming in as it may corrupt the stream of frames otherwise 84 | /// being worked with. 85 | pub fn into_inner(self) -> T { 86 | self.release().0 87 | } 88 | 89 | /// Returns a reference to the underlying codec wrapped by 90 | /// `Framed`. 91 | /// 92 | /// Note that care should be taken to not tamper with the underlying codec 93 | /// as it may corrupt the stream of frames otherwise being worked with. 94 | pub fn codec(&self) -> &U { 95 | &self.inner.u 96 | } 97 | 98 | /// Returns a mutable reference to the underlying codec wrapped by 99 | /// `Framed`. 100 | /// 101 | /// Note that care should be taken to not tamper with the underlying codec 102 | /// as it may corrupt the stream of frames otherwise being worked with. 103 | pub fn codec_mut(&mut self) -> &mut U { 104 | &mut self.inner.u 105 | } 106 | 107 | /// Returns a reference to the read buffer. 108 | pub fn read_buffer(&self) -> &BytesMut { 109 | self.inner.buffer() 110 | } 111 | } 112 | 113 | impl Stream for Framed 114 | where 115 | T: AsyncRead + Unpin, 116 | U: Decoder, 117 | { 118 | type Item = Result; 119 | 120 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 121 | self.inner.try_poll_next_unpin(cx) 122 | } 123 | } 124 | 125 | impl Sink for Framed 126 | where 127 | T: AsyncWrite + Unpin, 128 | U: Encoder, 129 | { 130 | type Error = U::Error; 131 | 132 | fn poll_ready(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 133 | self.project().inner.poll_ready(cx) 134 | } 135 | fn start_send(self: Pin<&mut Self>, item: U::Item) -> Result<(), Self::Error> { 136 | self.project().inner.start_send(item) 137 | } 138 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 139 | self.project().inner.poll_flush(cx) 140 | } 141 | fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 142 | self.project().inner.poll_close(cx) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/codec/json.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | 3 | use crate::{Decoder, Encoder}; 4 | use bytes::{Buf, BufMut, BytesMut}; 5 | 6 | use serde::{Deserialize, Serialize}; 7 | 8 | /// A codec for JSON encoding and decoding using serde_json 9 | /// Enc is the type to encode, Dec is the type to decode 10 | /// ``` 11 | /// # use futures::{executor, SinkExt, TryStreamExt}; 12 | /// # use futures::io::Cursor; 13 | /// use serde::{Serialize, Deserialize}; 14 | /// use futures_codec::{JsonCodec, Framed}; 15 | /// 16 | /// #[derive(Serialize, Deserialize)] 17 | /// struct Something { 18 | /// pub data: u16, 19 | /// } 20 | /// 21 | /// async move { 22 | /// # let mut buf = vec![]; 23 | /// # let stream = Cursor::new(&mut buf); 24 | /// // let stream = ... 25 | /// let codec = JsonCodec::::new(); 26 | /// let mut framed = Framed::new(stream, codec); 27 | /// 28 | /// while let Some(s) = framed.try_next().await.unwrap() { 29 | /// println!("{:?}", s.data); 30 | /// } 31 | /// }; 32 | /// ``` 33 | #[derive(Debug, PartialEq)] 34 | pub struct JsonCodec { 35 | enc: PhantomData, 36 | dec: PhantomData, 37 | } 38 | 39 | /// JSON Codec error enumeration 40 | #[derive(Debug)] 41 | pub enum JsonCodecError { 42 | /// IO error 43 | Io(std::io::Error), 44 | /// JSON error 45 | Json(serde_json::Error), 46 | } 47 | 48 | impl std::fmt::Display for JsonCodecError { 49 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 50 | match self { 51 | JsonCodecError::Io(e) => write!(f, "I/O error: {}", e), 52 | JsonCodecError::Json(e) => write!(f, "JSON error: {}", e), 53 | } 54 | } 55 | } 56 | 57 | impl std::error::Error for JsonCodecError { 58 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 59 | match self { 60 | JsonCodecError::Io(ref e) => Some(e), 61 | JsonCodecError::Json(ref e) => Some(e), 62 | } 63 | } 64 | } 65 | 66 | impl From for JsonCodecError { 67 | fn from(e: std::io::Error) -> JsonCodecError { 68 | JsonCodecError::Io(e) 69 | } 70 | } 71 | 72 | impl From for JsonCodecError { 73 | fn from(e: serde_json::Error) -> JsonCodecError { 74 | JsonCodecError::Json(e) 75 | } 76 | } 77 | 78 | impl JsonCodec 79 | where 80 | for<'de> Dec: Deserialize<'de> + 'static, 81 | for<'de> Enc: Serialize + 'static, 82 | { 83 | /// Creates a new `JsonCodec` with the associated types 84 | pub fn new() -> JsonCodec { 85 | JsonCodec { 86 | enc: PhantomData, 87 | dec: PhantomData, 88 | } 89 | } 90 | } 91 | 92 | impl Clone for JsonCodec 93 | where 94 | for<'de> Dec: Deserialize<'de> + 'static, 95 | for<'de> Enc: Serialize + 'static, 96 | { 97 | /// Clone creates a new instance of the `JsonCodec` 98 | fn clone(&self) -> JsonCodec { 99 | JsonCodec::new() 100 | } 101 | } 102 | 103 | /// Decoder impl parses json objects from bytes 104 | impl Decoder for JsonCodec 105 | where 106 | for<'de> Dec: Deserialize<'de> + 'static, 107 | for<'de> Enc: Serialize + 'static, 108 | { 109 | type Item = Dec; 110 | type Error = JsonCodecError; 111 | 112 | fn decode(&mut self, buf: &mut BytesMut) -> Result, Self::Error> { 113 | // Build streaming JSON iterator over data 114 | let de = serde_json::Deserializer::from_slice(&buf); 115 | let mut iter = de.into_iter::(); 116 | 117 | // Attempt to fetch an item and generate response 118 | let res = match iter.next() { 119 | Some(Ok(v)) => Ok(Some(v)), 120 | Some(Err(ref e)) if e.is_eof() => Ok(None), 121 | Some(Err(e)) => Err(e.into()), 122 | None => Ok(None), 123 | }; 124 | 125 | // Update offset from iterator 126 | let offset = iter.byte_offset(); 127 | 128 | // Advance buffer 129 | buf.advance(offset); 130 | 131 | res 132 | } 133 | } 134 | 135 | /// Encoder impl encodes object streams to bytes 136 | impl Encoder for JsonCodec 137 | where 138 | for<'de> Dec: Deserialize<'de> + 'static, 139 | for<'de> Enc: Serialize + 'static, 140 | { 141 | type Item = Enc; 142 | type Error = JsonCodecError; 143 | 144 | fn encode(&mut self, data: Self::Item, buf: &mut BytesMut) -> Result<(), Self::Error> { 145 | // Encode json 146 | let j = serde_json::to_string(&data)?; 147 | 148 | // Write to buffer 149 | buf.reserve(j.len()); 150 | buf.put_slice(&j.as_bytes()); 151 | 152 | Ok(()) 153 | } 154 | } 155 | 156 | 157 | impl Default for JsonCodec 158 | where 159 | for<'de> Dec: Deserialize<'de> + 'static, 160 | for<'de> Enc: Serialize + 'static, 161 | { 162 | fn default() -> Self { 163 | Self::new() 164 | } 165 | } 166 | 167 | 168 | #[cfg(test)] 169 | mod test { 170 | use bytes::BytesMut; 171 | use serde::{Deserialize, Serialize}; 172 | 173 | use super::JsonCodec; 174 | use crate::{Decoder, Encoder}; 175 | 176 | #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] 177 | struct TestStruct { 178 | pub name: String, 179 | pub data: u16, 180 | } 181 | 182 | #[test] 183 | fn json_codec_encode_decode() { 184 | let mut codec = JsonCodec::::new(); 185 | let mut buff = BytesMut::new(); 186 | 187 | let item1 = TestStruct { 188 | name: "Test name".to_owned(), 189 | data: 16, 190 | }; 191 | codec.encode(item1.clone(), &mut buff).unwrap(); 192 | 193 | let item2 = codec.decode(&mut buff).unwrap().unwrap(); 194 | assert_eq!(item1, item2); 195 | 196 | assert_eq!(codec.decode(&mut buff).unwrap(), None); 197 | 198 | assert_eq!(buff.len(), 0); 199 | } 200 | 201 | #[test] 202 | fn json_codec_partial_decode() { 203 | let mut codec = JsonCodec::::new(); 204 | let mut buff = BytesMut::new(); 205 | 206 | let item1 = TestStruct { 207 | name: "Test name".to_owned(), 208 | data: 34, 209 | }; 210 | codec.encode(item1, &mut buff).unwrap(); 211 | 212 | let mut start = buff.clone().split_to(4); 213 | assert_eq!(codec.decode(&mut start).unwrap(), None); 214 | 215 | codec.decode(&mut buff).unwrap().unwrap(); 216 | 217 | assert_eq!(buff.len(), 0); 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/codec/cbor.rs: -------------------------------------------------------------------------------- 1 | use std::io::Error as IoError; 2 | use std::marker::PhantomData; 3 | 4 | use crate::{Decoder, Encoder}; 5 | use bytes::{Buf, BufMut, BytesMut}; 6 | 7 | use serde::{Deserialize, Serialize}; 8 | use serde_cbor::Error as CborError; 9 | 10 | /// A codec for JSON encoding and decoding using serde_cbor 11 | /// Enc is the type to encode, Dec is the type to decode 12 | /// ``` 13 | /// # use futures::{executor, SinkExt, TryStreamExt}; 14 | /// # use futures::io::Cursor; 15 | /// use serde::{Serialize, Deserialize}; 16 | /// use futures_codec::{CborCodec, Framed}; 17 | /// 18 | /// #[derive(Serialize, Deserialize)] 19 | /// struct Something { 20 | /// pub data: u16, 21 | /// } 22 | /// 23 | /// async move { 24 | /// # let mut buf = vec![]; 25 | /// # let stream = Cursor::new(&mut buf); 26 | /// // let stream = ... 27 | /// let codec = CborCodec::::new(); 28 | /// let mut framed = Framed::new(stream, codec); 29 | /// 30 | /// while let Some(s) = framed.try_next().await.unwrap() { 31 | /// println!("{:?}", s.data); 32 | /// } 33 | /// }; 34 | /// ``` 35 | #[derive(Debug, PartialEq)] 36 | pub struct CborCodec { 37 | enc: PhantomData, 38 | dec: PhantomData, 39 | } 40 | 41 | /// JSON Codec error enumeration 42 | #[derive(Debug)] 43 | pub enum CborCodecError { 44 | /// IO error 45 | Io(IoError), 46 | /// JSON error 47 | Cbor(CborError), 48 | } 49 | 50 | impl std::fmt::Display for CborCodecError { 51 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 52 | match self { 53 | CborCodecError::Io(e) => write!(f, "I/O error: {}", e), 54 | CborCodecError::Cbor(e) => write!(f, "CBOR error: {}", e), 55 | } 56 | } 57 | } 58 | 59 | impl std::error::Error for CborCodecError { 60 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 61 | match self { 62 | CborCodecError::Io(ref e) => Some(e), 63 | CborCodecError::Cbor(ref e) => Some(e), 64 | } 65 | } 66 | } 67 | 68 | impl From for CborCodecError { 69 | fn from(e: IoError) -> CborCodecError { 70 | CborCodecError::Io(e) 71 | } 72 | } 73 | 74 | impl From for CborCodecError { 75 | fn from(e: CborError) -> CborCodecError { 76 | CborCodecError::Cbor(e) 77 | } 78 | } 79 | 80 | impl CborCodec 81 | where 82 | for<'de> Dec: Deserialize<'de> + 'static, 83 | for<'de> Enc: Serialize + 'static, 84 | { 85 | /// Creates a new `CborCodec` with the associated types 86 | pub fn new() -> CborCodec { 87 | CborCodec { 88 | enc: PhantomData, 89 | dec: PhantomData, 90 | } 91 | } 92 | } 93 | 94 | impl Clone for CborCodec 95 | where 96 | for<'de> Dec: Deserialize<'de> + 'static, 97 | for<'de> Enc: Serialize + 'static, 98 | { 99 | /// Clone creates a new instance of the `CborCodec` 100 | fn clone(&self) -> CborCodec { 101 | CborCodec::new() 102 | } 103 | } 104 | 105 | /// Decoder impl parses cbor objects from bytes 106 | impl Decoder for CborCodec 107 | where 108 | for<'de> Dec: Deserialize<'de> + 'static, 109 | for<'de> Enc: Serialize + 'static, 110 | { 111 | type Item = Dec; 112 | type Error = CborCodecError; 113 | 114 | fn decode(&mut self, buf: &mut BytesMut) -> Result, Self::Error> { 115 | // Build deserializer 116 | let mut de = serde_cbor::Deserializer::from_slice(&buf); 117 | 118 | // Attempt deserialization 119 | let res: Result = serde::de::Deserialize::deserialize(&mut de); 120 | 121 | // If we ran out before parsing, return none and try again later 122 | let res = match res { 123 | Ok(v) => Ok(Some(v)), 124 | Err(e) if e.is_eof() => Ok(None), 125 | Err(e) => Err(e.into()), 126 | }; 127 | 128 | // Update offset from iterator 129 | let offset = de.byte_offset(); 130 | 131 | // Advance buffer 132 | buf.advance(offset); 133 | 134 | res 135 | } 136 | } 137 | 138 | /// Encoder impl encodes object streams to bytes 139 | impl Encoder for CborCodec 140 | where 141 | for<'de> Dec: Deserialize<'de> + 'static, 142 | for<'de> Enc: Serialize + 'static, 143 | { 144 | type Item = Enc; 145 | type Error = CborCodecError; 146 | 147 | fn encode(&mut self, data: Self::Item, buf: &mut BytesMut) -> Result<(), Self::Error> { 148 | // Encode cbor 149 | let j = serde_cbor::to_vec(&data)?; 150 | 151 | // Write to buffer 152 | buf.reserve(j.len()); 153 | buf.put_slice(&j); 154 | 155 | Ok(()) 156 | } 157 | } 158 | 159 | impl Default for CborCodec 160 | where 161 | for<'de> Dec: Deserialize<'de> + 'static, 162 | for<'de> Enc: Serialize + 'static, 163 | { 164 | fn default() -> Self { 165 | Self::new() 166 | } 167 | } 168 | 169 | #[cfg(test)] 170 | mod test { 171 | use bytes::BytesMut; 172 | use serde::{Deserialize, Serialize}; 173 | 174 | use super::CborCodec; 175 | use crate::{Decoder, Encoder}; 176 | 177 | #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] 178 | struct TestStruct { 179 | pub name: String, 180 | pub data: u16, 181 | } 182 | 183 | #[test] 184 | fn cbor_codec_encode_decode() { 185 | let mut codec = CborCodec::::new(); 186 | let mut buff = BytesMut::new(); 187 | 188 | let item1 = TestStruct { 189 | name: "Test name".to_owned(), 190 | data: 16, 191 | }; 192 | codec.encode(item1.clone(), &mut buff).unwrap(); 193 | 194 | let item2 = codec.decode(&mut buff).unwrap().unwrap(); 195 | assert_eq!(item1, item2); 196 | 197 | assert_eq!(codec.decode(&mut buff).unwrap(), None); 198 | 199 | assert_eq!(buff.len(), 0); 200 | } 201 | 202 | #[test] 203 | fn cbor_codec_partial_decode() { 204 | let mut codec = CborCodec::::new(); 205 | let mut buff = BytesMut::new(); 206 | 207 | let item1 = TestStruct { 208 | name: "Test name".to_owned(), 209 | data: 34, 210 | }; 211 | codec.encode(item1, &mut buff).unwrap(); 212 | 213 | let mut start = buff.clone().split_to(4); 214 | assert_eq!(codec.decode(&mut start).unwrap(), None); 215 | 216 | codec.decode(&mut buff).unwrap().unwrap(); 217 | 218 | assert_eq!(buff.len(), 0); 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /src/framed_read.rs: -------------------------------------------------------------------------------- 1 | use super::fuse::Fuse; 2 | use super::Decoder; 3 | 4 | use bytes::BytesMut; 5 | use futures_sink::Sink; 6 | use futures_util::io::AsyncRead; 7 | use futures_util::ready; 8 | use futures_util::stream::{Stream, TryStreamExt}; 9 | use pin_project_lite::pin_project; 10 | use std::io; 11 | use std::marker::Unpin; 12 | use std::ops::{Deref, DerefMut}; 13 | use std::pin::Pin; 14 | use std::task::{Context, Poll}; 15 | 16 | /// A `Stream` of messages decoded from an `AsyncRead`. 17 | /// 18 | /// # Example 19 | /// ``` 20 | /// use futures_codec::{BytesCodec, FramedRead}; 21 | /// use futures::TryStreamExt; 22 | /// use bytes::{Bytes}; 23 | /// 24 | /// let buf = [3u8; 3]; 25 | /// let mut framed = FramedRead::new(&buf[..], BytesCodec); 26 | /// 27 | /// # futures::executor::block_on(async move { 28 | /// if let Some(bytes) = framed.try_next().await? { 29 | /// assert_eq!(bytes, Bytes::copy_from_slice(&buf[..])); 30 | /// } 31 | /// # Ok::<_, std::io::Error>(()) 32 | /// # }).unwrap(); 33 | /// ``` 34 | #[derive(Debug)] 35 | pub struct FramedRead { 36 | inner: FramedRead2>, 37 | } 38 | 39 | impl Deref for FramedRead { 40 | type Target = T; 41 | 42 | fn deref(&self) -> &T { 43 | &self.inner 44 | } 45 | } 46 | 47 | impl DerefMut for FramedRead { 48 | fn deref_mut(&mut self) -> &mut T { 49 | &mut self.inner 50 | } 51 | } 52 | 53 | impl FramedRead 54 | where 55 | T: AsyncRead, 56 | D: Decoder, 57 | { 58 | /// Creates a new `FramedRead` transport with the given `Decoder`. 59 | pub fn new(inner: T, decoder: D) -> Self { 60 | Self { 61 | inner: framed_read_2(Fuse::new(inner, decoder)), 62 | } 63 | } 64 | 65 | /// Release the I/O and Decoder 66 | pub fn release(self) -> (T, D) { 67 | let fuse = self.inner.release(); 68 | (fuse.t, fuse.u) 69 | } 70 | 71 | /// Consumes the `FramedRead`, returning its underlying I/O stream. 72 | /// 73 | /// Note that care should be taken to not tamper with the underlying stream 74 | /// of data coming in as it may corrupt the stream of frames otherwise 75 | /// being worked with. 76 | pub fn into_inner(self) -> T { 77 | self.release().0 78 | } 79 | 80 | /// Returns a reference to the underlying decoder. 81 | /// 82 | /// Note that care should be taken to not tamper with the underlying decoder 83 | /// as it may corrupt the stream of frames otherwise being worked with. 84 | pub fn decoder(&self) -> &D { 85 | &self.inner.u 86 | } 87 | 88 | /// Returns a mutable reference to the underlying decoder. 89 | /// 90 | /// Note that care should be taken to not tamper with the underlying decoder 91 | /// as it may corrupt the stream of frames otherwise being worked with. 92 | pub fn decoder_mut(&mut self) -> &mut D { 93 | &mut self.inner.u 94 | } 95 | 96 | /// Returns a reference to the read buffer. 97 | pub fn read_buffer(&self) -> &BytesMut { 98 | &self.inner.buffer 99 | } 100 | } 101 | 102 | impl Stream for FramedRead 103 | where 104 | T: AsyncRead + Unpin, 105 | D: Decoder, 106 | { 107 | type Item = Result; 108 | 109 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 110 | self.inner.try_poll_next_unpin(cx) 111 | } 112 | } 113 | 114 | pin_project! { 115 | #[derive(Debug)] 116 | pub struct FramedRead2 { 117 | #[pin] 118 | inner: T, 119 | buffer: BytesMut, 120 | } 121 | } 122 | 123 | impl Deref for FramedRead2 { 124 | type Target = T; 125 | 126 | fn deref(&self) -> &T { 127 | &self.inner 128 | } 129 | } 130 | 131 | impl DerefMut for FramedRead2 { 132 | fn deref_mut(&mut self) -> &mut T { 133 | &mut self.inner 134 | } 135 | } 136 | 137 | const INITIAL_CAPACITY: usize = 8 * 1024; 138 | 139 | pub fn framed_read_2(inner: T) -> FramedRead2 { 140 | FramedRead2 { 141 | inner, 142 | buffer: BytesMut::with_capacity(INITIAL_CAPACITY), 143 | } 144 | } 145 | 146 | impl Stream for FramedRead2 147 | where 148 | T: AsyncRead + Decoder + Unpin, 149 | { 150 | type Item = Result; 151 | 152 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 153 | let this = &mut *self; 154 | 155 | if let Some(item) = this.inner.decode(&mut this.buffer)? { 156 | return Poll::Ready(Some(Ok(item))); 157 | } 158 | 159 | let mut buf = [0u8; INITIAL_CAPACITY]; 160 | 161 | loop { 162 | let n = ready!(Pin::new(&mut this.inner).poll_read(cx, &mut buf))?; 163 | this.buffer.extend_from_slice(&buf[..n]); 164 | 165 | let ended = n == 0; 166 | 167 | match this.inner.decode(&mut this.buffer)? { 168 | Some(item) => return Poll::Ready(Some(Ok(item))), 169 | None if ended => { 170 | if this.buffer.is_empty() { 171 | return Poll::Ready(None); 172 | } else { 173 | match this.inner.decode_eof(&mut this.buffer)? { 174 | Some(item) => return Poll::Ready(Some(Ok(item))), 175 | None if this.buffer.is_empty() => return Poll::Ready(None), 176 | None => { 177 | return Poll::Ready(Some(Err(io::Error::new( 178 | io::ErrorKind::UnexpectedEof, 179 | "bytes remaining in stream", 180 | ) 181 | .into()))); 182 | } 183 | } 184 | } 185 | } 186 | _ => continue, 187 | } 188 | } 189 | } 190 | } 191 | 192 | impl Sink for FramedRead2 193 | where 194 | T: Sink + Unpin, 195 | { 196 | type Error = T::Error; 197 | 198 | fn poll_ready(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 199 | self.project().inner.poll_ready(cx) 200 | } 201 | fn start_send(self: Pin<&mut Self>, item: I) -> Result<(), Self::Error> { 202 | self.project().inner.start_send(item) 203 | } 204 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 205 | self.project().inner.poll_flush(cx) 206 | } 207 | fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 208 | self.project().inner.poll_close(cx) 209 | } 210 | } 211 | 212 | impl FramedRead2 { 213 | pub fn release(self) -> T { 214 | self.inner 215 | } 216 | 217 | pub fn buffer(&self) -> &BytesMut { 218 | &self.buffer 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /src/framed_write.rs: -------------------------------------------------------------------------------- 1 | use super::fuse::Fuse; 2 | use super::Encoder; 3 | use bytes::{Buf, BytesMut}; 4 | use futures_sink::Sink; 5 | use futures_util::io::{AsyncRead, AsyncWrite}; 6 | use futures_util::ready; 7 | use pin_project_lite::pin_project; 8 | use std::io::{Error, ErrorKind}; 9 | use std::marker::Unpin; 10 | use std::ops::{Deref, DerefMut}; 11 | use std::pin::Pin; 12 | use std::task::{Context, Poll}; 13 | 14 | pin_project! { 15 | /// A `Sink` of frames encoded to an `AsyncWrite`. 16 | /// 17 | /// # Example 18 | /// ``` 19 | /// use bytes::Bytes; 20 | /// use futures_codec::{FramedWrite, BytesCodec}; 21 | /// use futures::SinkExt; 22 | /// 23 | /// # futures::executor::block_on(async move { 24 | /// let mut buf = Vec::new(); 25 | /// let mut framed = FramedWrite::new(&mut buf, BytesCodec {}); 26 | /// 27 | /// let bytes = Bytes::from("Hello World!"); 28 | /// framed.send(bytes.clone()).await?; 29 | /// 30 | /// assert_eq!(&buf[..], &bytes[..]); 31 | /// # Ok::<_, std::io::Error>(()) 32 | /// # }).unwrap(); 33 | /// ``` 34 | #[derive(Debug)] 35 | pub struct FramedWrite { 36 | #[pin] 37 | inner: FramedWrite2>, 38 | } 39 | } 40 | 41 | impl FramedWrite 42 | where 43 | T: AsyncWrite, 44 | E: Encoder, 45 | { 46 | /// Creates a new `FramedWrite` transport with the given `Encoder`. 47 | pub fn new(inner: T, encoder: E) -> Self { 48 | Self { 49 | inner: framed_write_2(Fuse::new(inner, encoder)), 50 | } 51 | } 52 | 53 | /// High-water mark for writes, in bytes 54 | /// 55 | /// The send *high-water mark* prevents the `FramedWrite` 56 | /// from accepting additional messages to send when its 57 | /// buffer exceeds this length, in bytes. Attempts to enqueue 58 | /// additional messages will be deferred until progress is 59 | /// made on the underlying `AsyncWrite`. This applies 60 | /// back-pressure on fast senders and prevents unbounded 61 | /// buffer growth. 62 | /// 63 | /// See [`set_send_high_water_mark()`](#method.set_send_high_water_mark). 64 | pub fn send_high_water_mark(&self) -> usize { 65 | self.inner.high_water_mark 66 | } 67 | 68 | /// Sets high-water mark for writes, in bytes 69 | /// 70 | /// The send *high-water mark* prevents the `FramedWrite` 71 | /// from accepting additional messages to send when its 72 | /// buffer exceeds this length, in bytes. Attempts to enqueue 73 | /// additional messages will be deferred until progress is 74 | /// made on the underlying `AsyncWrite`. This applies 75 | /// back-pressure on fast senders and prevents unbounded 76 | /// buffer growth. 77 | /// 78 | /// The default high-water mark is 2^17 bytes. Applications 79 | /// which desire low latency may wish to reduce this value. 80 | /// There is little point to increasing this value beyond 81 | /// your socket's `SO_SNDBUF` size. On linux, this defaults 82 | /// to 212992 bytes but is user-adjustable. 83 | pub fn set_send_high_water_mark(&mut self, hwm: usize) { 84 | self.inner.high_water_mark = hwm; 85 | } 86 | 87 | /// Release the I/O and Encoder 88 | pub fn release(self) -> (T, E) { 89 | let fuse = self.inner.release(); 90 | (fuse.t, fuse.u) 91 | } 92 | 93 | /// Consumes the `FramedWrite`, returning its underlying I/O stream. 94 | /// 95 | /// Note that care should be taken to not tamper with the underlying stream 96 | /// of data coming in as it may corrupt the stream of frames otherwise 97 | /// being worked with. 98 | pub fn into_inner(self) -> T { 99 | self.release().0 100 | } 101 | 102 | /// Returns a reference to the underlying encoder. 103 | /// 104 | /// Note that care should be taken to not tamper with the underlying encoder 105 | /// as it may corrupt the stream of frames otherwise being worked with. 106 | pub fn encoder(&self) -> &E { 107 | &self.inner.u 108 | } 109 | 110 | /// Returns a mutable reference to the underlying encoder. 111 | /// 112 | /// Note that care should be taken to not tamper with the underlying encoder 113 | /// as it may corrupt the stream of frames otherwise being worked with. 114 | pub fn encoder_mut(&mut self) -> &mut E { 115 | &mut self.inner.u 116 | } 117 | } 118 | 119 | impl Sink for FramedWrite 120 | where 121 | T: AsyncWrite + Unpin, 122 | E: Encoder, 123 | { 124 | type Error = E::Error; 125 | 126 | fn poll_ready(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 127 | self.project().inner.poll_ready(cx) 128 | } 129 | fn start_send(self: Pin<&mut Self>, item: E::Item) -> Result<(), Self::Error> { 130 | self.project().inner.start_send(item) 131 | } 132 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 133 | self.project().inner.poll_flush(cx) 134 | } 135 | fn poll_close(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 136 | self.project().inner.poll_close(cx) 137 | } 138 | } 139 | 140 | impl Deref for FramedWrite { 141 | type Target = T; 142 | 143 | fn deref(&self) -> &T { 144 | &self.inner 145 | } 146 | } 147 | 148 | impl DerefMut for FramedWrite { 149 | fn deref_mut(&mut self) -> &mut T { 150 | &mut self.inner 151 | } 152 | } 153 | 154 | pin_project! { 155 | #[derive(Debug)] 156 | pub struct FramedWrite2 { 157 | #[pin] 158 | pub inner: T, 159 | pub high_water_mark: usize, 160 | buffer: BytesMut, 161 | } 162 | } 163 | 164 | impl Deref for FramedWrite2 { 165 | type Target = T; 166 | 167 | fn deref(&self) -> &T { 168 | &self.inner 169 | } 170 | } 171 | 172 | impl DerefMut for FramedWrite2 { 173 | fn deref_mut(&mut self) -> &mut T { 174 | &mut self.inner 175 | } 176 | } 177 | 178 | // 2^17 bytes, which is slightly over 60% of the default 179 | // TCP send buffer size (SO_SNDBUF) 180 | const DEFAULT_SEND_HIGH_WATER_MARK: usize = 131072; 181 | 182 | pub fn framed_write_2(inner: T) -> FramedWrite2 { 183 | FramedWrite2 { 184 | inner, 185 | high_water_mark: DEFAULT_SEND_HIGH_WATER_MARK, 186 | buffer: BytesMut::with_capacity(1028 * 8), 187 | } 188 | } 189 | 190 | impl AsyncRead for FramedWrite2 { 191 | fn poll_read( 192 | self: Pin<&mut Self>, 193 | cx: &mut Context<'_>, 194 | buf: &mut [u8], 195 | ) -> Poll> { 196 | self.project().inner.poll_read(cx, buf) 197 | } 198 | } 199 | 200 | impl Sink for FramedWrite2 201 | where 202 | T: AsyncWrite + Encoder + Unpin, 203 | { 204 | type Error = T::Error; 205 | 206 | fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 207 | let this = &mut *self; 208 | while this.buffer.len() >= this.high_water_mark { 209 | let num_write = ready!(Pin::new(&mut this.inner).poll_write(cx, &this.buffer))?; 210 | 211 | if num_write == 0 { 212 | return Poll::Ready(Err(err_eof().into())); 213 | } 214 | 215 | this.buffer.advance(num_write); 216 | } 217 | 218 | Poll::Ready(Ok(())) 219 | } 220 | fn start_send(mut self: Pin<&mut Self>, item: T::Item) -> Result<(), Self::Error> { 221 | let this = &mut *self; 222 | this.inner.encode(item, &mut this.buffer) 223 | } 224 | fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 225 | let mut this = self.project(); 226 | 227 | while !this.buffer.is_empty() { 228 | let num_write = ready!(Pin::new(&mut this.inner).poll_write(cx, &this.buffer))?; 229 | 230 | if num_write == 0 { 231 | return Poll::Ready(Err(err_eof().into())); 232 | } 233 | 234 | this.buffer.advance(num_write); 235 | } 236 | 237 | this.inner.poll_flush(cx).map_err(Into::into) 238 | } 239 | fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { 240 | ready!(self.as_mut().poll_flush(cx))?; 241 | self.project().inner.poll_close(cx).map_err(Into::into) 242 | } 243 | } 244 | 245 | impl FramedWrite2 { 246 | pub fn release(self) -> T { 247 | self.inner 248 | } 249 | } 250 | 251 | fn err_eof() -> Error { 252 | Error::new(ErrorKind::UnexpectedEof, "End of file") 253 | } 254 | --------------------------------------------------------------------------------