├── .gitignore ├── README.md ├── lines ├── Cargo.lock ├── Cargo.toml ├── README.md └── src │ ├── bin │ ├── client.rs │ └── server.rs │ └── lib.rs ├── protocol ├── Cargo.lock ├── Cargo.toml ├── README.md └── src │ ├── bin │ ├── client.rs │ └── server.rs │ └── lib.rs └── raw ├── Cargo.lock ├── Cargo.toml ├── README.md ├── res └── tcpstream_raw.png └── src ├── bin ├── client.rs └── server.rs └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ready & Writing data with Rust's TcpStream 2 | 3 | This repo contains examples of using Rust's [TcpStream](https://doc.rust-lang.org/stable/std/net/struct.TcpStream.html) to send & receive data between a client and server. 4 | This example shows low-level data (raw bytes) reading & writing with Rust's [TcpStream](https://doc.rust-lang.org/stable/std/net/struct.TcpStream.html). Subsequent examples add abstractions over this, but it's helpful to understand what's happening under the hood and why abstractions make things easier. 5 | 6 | ## [Raw TCP Bytes](./raw) 7 | See how the [Read](https://doc.rust-lang.org/stable/std/io/trait.Read.html) and [Write](https://doc.rust-lang.org/stable/std/io/trait.Write.html) traits work with low-level TcpStream Tx/Rx 8 | 9 | ## [Line-based Codec](./lines) 10 | Step up a level of abstraction using line-based messaging (newline delimited) and how the [BufRead](https://doc.rust-lang.org/stable/std/io/trait.BufRead.html) and [BufWrite](https://doc.rust-lang.org/stable/std/io/trait.BufWrite.html) traits can be more effecient 11 | 12 | ## [Message Protocol](./protocol) 13 | If we want to send more than just lines, we can abstract even further into a protocol of structs, handling serialization & deserialization with [byteorder](https://docs.rs/byteorder/1.3.4/byteorder/) 14 | -------------------------------------------------------------------------------- /lines/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "ansi_term" 5 | version = "0.11.0" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 8 | dependencies = [ 9 | "winapi", 10 | ] 11 | 12 | [[package]] 13 | name = "atty" 14 | version = "0.2.14" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 17 | dependencies = [ 18 | "hermit-abi", 19 | "libc", 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "bitflags" 25 | version = "1.2.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 28 | 29 | [[package]] 30 | name = "clap" 31 | version = "2.33.1" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" 34 | dependencies = [ 35 | "ansi_term", 36 | "atty", 37 | "bitflags", 38 | "strsim", 39 | "textwrap", 40 | "unicode-width", 41 | "vec_map", 42 | ] 43 | 44 | [[package]] 45 | name = "heck" 46 | version = "0.3.1" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" 49 | dependencies = [ 50 | "unicode-segmentation", 51 | ] 52 | 53 | [[package]] 54 | name = "hermit-abi" 55 | version = "0.1.14" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" 58 | dependencies = [ 59 | "libc", 60 | ] 61 | 62 | [[package]] 63 | name = "lazy_static" 64 | version = "1.4.0" 65 | source = "registry+https://github.com/rust-lang/crates.io-index" 66 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 67 | 68 | [[package]] 69 | name = "libc" 70 | version = "0.2.71" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" 73 | 74 | [[package]] 75 | name = "proc-macro-error" 76 | version = "1.0.2" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" 79 | dependencies = [ 80 | "proc-macro-error-attr", 81 | "proc-macro2", 82 | "quote", 83 | "syn", 84 | "version_check", 85 | ] 86 | 87 | [[package]] 88 | name = "proc-macro-error-attr" 89 | version = "1.0.2" 90 | source = "registry+https://github.com/rust-lang/crates.io-index" 91 | checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" 92 | dependencies = [ 93 | "proc-macro2", 94 | "quote", 95 | "syn", 96 | "syn-mid", 97 | "version_check", 98 | ] 99 | 100 | [[package]] 101 | name = "proc-macro2" 102 | version = "1.0.18" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" 105 | dependencies = [ 106 | "unicode-xid", 107 | ] 108 | 109 | [[package]] 110 | name = "quote" 111 | version = "1.0.7" 112 | source = "registry+https://github.com/rust-lang/crates.io-index" 113 | checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" 114 | dependencies = [ 115 | "proc-macro2", 116 | ] 117 | 118 | [[package]] 119 | name = "strsim" 120 | version = "0.8.0" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 123 | 124 | [[package]] 125 | name = "structopt" 126 | version = "0.3.15" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "de2f5e239ee807089b62adce73e48c625e0ed80df02c7ab3f068f5db5281065c" 129 | dependencies = [ 130 | "clap", 131 | "lazy_static", 132 | "structopt-derive", 133 | ] 134 | 135 | [[package]] 136 | name = "structopt-derive" 137 | version = "0.4.8" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "510413f9de616762a4fbeab62509bf15c729603b72d7cd71280fbca431b1c118" 140 | dependencies = [ 141 | "heck", 142 | "proc-macro-error", 143 | "proc-macro2", 144 | "quote", 145 | "syn", 146 | ] 147 | 148 | [[package]] 149 | name = "syn" 150 | version = "1.0.33" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" 153 | dependencies = [ 154 | "proc-macro2", 155 | "quote", 156 | "unicode-xid", 157 | ] 158 | 159 | [[package]] 160 | name = "syn-mid" 161 | version = "0.5.0" 162 | source = "registry+https://github.com/rust-lang/crates.io-index" 163 | checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" 164 | dependencies = [ 165 | "proc-macro2", 166 | "quote", 167 | "syn", 168 | ] 169 | 170 | [[package]] 171 | name = "tcp_demo_lines" 172 | version = "0.1.0" 173 | dependencies = [ 174 | "structopt", 175 | ] 176 | 177 | [[package]] 178 | name = "textwrap" 179 | version = "0.11.0" 180 | source = "registry+https://github.com/rust-lang/crates.io-index" 181 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 182 | dependencies = [ 183 | "unicode-width", 184 | ] 185 | 186 | [[package]] 187 | name = "unicode-segmentation" 188 | version = "1.6.0" 189 | source = "registry+https://github.com/rust-lang/crates.io-index" 190 | checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" 191 | 192 | [[package]] 193 | name = "unicode-width" 194 | version = "0.1.7" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" 197 | 198 | [[package]] 199 | name = "unicode-xid" 200 | version = "0.2.0" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 203 | 204 | [[package]] 205 | name = "vec_map" 206 | version = "0.8.2" 207 | source = "registry+https://github.com/rust-lang/crates.io-index" 208 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 209 | 210 | [[package]] 211 | name = "version_check" 212 | version = "0.9.2" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" 215 | 216 | [[package]] 217 | name = "winapi" 218 | version = "0.3.8" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 221 | dependencies = [ 222 | "winapi-i686-pc-windows-gnu", 223 | "winapi-x86_64-pc-windows-gnu", 224 | ] 225 | 226 | [[package]] 227 | name = "winapi-i686-pc-windows-gnu" 228 | version = "0.4.0" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 231 | 232 | [[package]] 233 | name = "winapi-x86_64-pc-windows-gnu" 234 | version = "0.4.0" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 237 | -------------------------------------------------------------------------------- /lines/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tcp_demo_lines" 3 | version = "0.1.0" 4 | authors = ["Mat Wood "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | structopt = "0.3.14" -------------------------------------------------------------------------------- /lines/README.md: -------------------------------------------------------------------------------- 1 | # Building a LinesCodec for TcpStream 2 | 3 | In the [previous demo](../raw) we learned how to read & write bytes with our TcpStream, but the calling code had to be aware of that and do the serialization/deserialization. In this demo, we'll continue using `BufRead` and `BufReader` to build: 4 | 5 | - A `LinesCodec` that abstracts away `String` serialization/deserialization & TcpStream I/O 6 | - A client that uses the `LinesCodec` to send and print returned `String`s 7 | - A server that also uses the `LinesCodec` and reverses `String`s before echoing them back 8 | 9 | Server 10 | ```sh 11 | $ cargo run --bin server 12 | Starting server on '127.0.0.1:4000' 13 | ... 14 | ``` 15 | 16 | Client 17 | ```sh 18 | $ cargo run --bin client -- Testing 19 | gnitseT 20 | ``` 21 | 22 | 23 | # LinesCodec 24 | Our goals for this LinesCodec implementation are to abstract away: 25 | - `TcpStream` reading & writing 26 | - We know enough about this now to know that the client & server shouldn't worry about the correct incantation to send & receive data 27 | - Even copy/pasting the code around is bound to cause an issue (like a forgotten `stream.flush()`) 28 | - `String` serialization & deserialization 29 | - The client/server code shouldn't care how the data is represented on the wire, our codec will take care of that! 30 | 31 | ## A Type to the rescue! 32 | Starting off with the I/O management abstraction, let's define a new type to own and interact with `TcpStream`. From the previous demo, we know we'll want to use the [BufReader](https://doc.rust-lang.org/std/io/struct.BufReader.html) and its `read_line()` method for reading data. 33 | 34 | For writing data, we have three options: 35 | - Use TcpStream directly, identical to what we did in the previous demo 36 | - Use [BufWriter](https://doc.rust-lang.org/std/io/struct.BufWriter.html) which is a good logical jump given our use of `BufReader` 37 | - Use [LineWriter](https://doc.rust-lang.org/stable/std/io/struct.LineWriter.html) which sounds like an even closer match to what we want 38 | 39 | 40 | I'm choosing to use `LineWriter` because it seems like a better approach to what we're wanting to do, based of of its documentation: 41 | 42 | > Wraps a writer and buffers output to it, flushing whenever a newline (0x0a, `\n`) is detected. 43 | > 44 | > The `BufWriter` struct wraps a writer and buffers its output. But it only does this batched write when it goes out of scope, or when the internal buffer is full. Sometimes, you'd prefer to write each line as it's completed, rather than the entire buffer at once. Enter LineWriter. It does exactly that. 45 | 46 | Great! Let's define our type and define `LinesCodec::new()` to build new instances: 47 | 48 | ```rust 49 | use std::io::{self, BufRead, Write}; 50 | use std::net::TcpStream; 51 | 52 | pub struct LinesCodec { 53 | // Our buffered reader & writers 54 | reader: io::BufReader, 55 | writer: io::LineWriter, 56 | } 57 | 58 | impl LinesCodec { 59 | /// Encapsulate a TcpStream with buffered reader/writer functionality 60 | pub fn new(stream: TcpStream) -> io::Result { 61 | // Both BufReader and LineWriter need to own a stream 62 | // We can clone the stream to simulate splitting Tx & Rx with `try_clone()` 63 | let writer = io::LineWriter::new(stream.try_clone()?); 64 | let reader = io::BufReader::new(stream); 65 | Ok(Self { reader, writer }) 66 | } 67 | } 68 | ``` 69 | 70 | ### Reading and Writing 71 | With the `LinesCodec` struct and its buffered reader/writers, we can continue our implementation to borrow the reading and writing code from the previous demo: 72 | 73 | ```rust 74 | ... 75 | impl LinesCodec { 76 | /// Write the given message (appending a newline) to the TcpStream 77 | pub fn send_message(&mut self, message: &str) -> io::Result<()> { 78 | self.writer.write(&message.as_bytes())?; 79 | // This will also signal a `writer.flush()` for us; thanks LineWriter! 80 | self.writer.write(&['\n' as u8])?; 81 | Ok(()) 82 | } 83 | 84 | /// Read a received message from the TcpStream 85 | pub fn read_message(&mut self) -> io::Result { 86 | let mut line = String::new(); 87 | // Use `BufRead::read_line()` to read a line from the TcpStream 88 | self.reader.read_line(&mut line)?; 89 | line.pop(); // Remove the trailing "\n" 90 | Ok(line) 91 | } 92 | } 93 | ``` 94 | 95 | # Using LinesCodec in the client 96 | With the `TcpStream` out of the way, let's refactor our client code from the previous demo and see how easy this can be: 97 | 98 | ```rust 99 | use std::io; 100 | use std::new::TcpStream; 101 | 102 | use crate::LinesCodec; 103 | 104 | 105 | fn main() -> io::Result<()> { 106 | // Establish a TCP connection with the farend 107 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 108 | 109 | // Codec is our interface for reading/writing messages. 110 | // No need to handle reading/writing directly 111 | let mut codec = LinesCodec::new(stream)?; 112 | 113 | // Serializing & Sending is now just one line 114 | codec.send_message("Hello")?; 115 | 116 | // And same with receiving the response! 117 | println!("{}", codec.read_message()?); 118 | Ok(()) 119 | 120 | } 121 | ``` 122 | 123 | # Using LinesCodec in the server 124 | And now we have some very similar work to update our server to use `LinesCodec`. We'll define a `handle_connection` function that: 125 | - Takes ownership of a `TcpStream` and wraps it in `LinesCodec` 126 | - Receives a message (`String`) from the client and reverses it 127 | - Sends the message back to the client, again using `LinesCodec` 128 | 129 | ```rust 130 | use std::io; 131 | use std::net::TcpStream; 132 | 133 | use crate:LinesCodec; 134 | 135 | /// Given a TcpStream: 136 | /// - Deserialize the message 137 | /// - Serialize and write the echo message to the stream 138 | fn handle_connection(stream: TcpStream) -> io::Result<()> { 139 | let mut codec = LinesCodec::new(stream)?; 140 | 141 | // Read & reverse the received message 142 | let message: String = codec 143 | .read_message() 144 | // Reverse message 145 | .map(|m| m.chars().rev().collect())?; 146 | 147 | // And use the codec to return it 148 | codec.send_message(&message)?; 149 | Ok(()) 150 | } 151 | ``` 152 | 153 | Using the codec makes the business logic code much clearer and there are fewer opportunities to mis-manage newlines or forget `flush()`. Check out the [client](src/bin/client.rs) and [server](src/bin/server.rs) code for a full runnable example. 154 | 155 | In the [next demo](../protocol) we dive even deeper to build a custom protocol with our own serialization/deserialization! 156 | 157 | # Running the demo 158 | From within this `./lines` directory we can start the server, and then in another terminal (tmux pane, ssh session, etc), run the client with a message of your choice 159 | 160 | Server 161 | ```sh 162 | $ cargo run --bin server 163 | Starting server on '127.0.0.1:4000' 164 | ... 165 | ``` 166 | 167 | Client 168 | ```sh 169 | $ cargo run --bin client -- Testing 170 | gnitseT 171 | ``` 172 | 173 | 174 | **(Inspired by the now removed [tokio example](https://github.com/tokio-rs/tokio/blob/9d4d076189822e32574f8123efe21c732103f4d4/examples/chat.rs))** -------------------------------------------------------------------------------- /lines/src/bin/client.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::net::{SocketAddr, TcpStream}; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_lines::{LinesCodec, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "client")] 10 | struct Args { 11 | message: String, 12 | /// Server destination address 13 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 14 | addr: SocketAddr, 15 | } 16 | 17 | fn main() -> io::Result<()> { 18 | let args = Args::from_args(); 19 | 20 | let stream = TcpStream::connect(args.addr)?; 21 | 22 | // Codec is our interface for reading/writing messages. 23 | // No need to handle reading/writing directly 24 | let mut codec = LinesCodec::new(stream)?; 25 | 26 | codec.send_message(&args.message)?; 27 | println!("{}", codec.read_message()?); 28 | Ok(()) 29 | } 30 | -------------------------------------------------------------------------------- /lines/src/bin/server.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::net::{SocketAddr, TcpListener, TcpStream}; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_lines::{LinesCodec, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "server")] 10 | struct Args { 11 | /// Service listening address 12 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 13 | addr: SocketAddr, 14 | } 15 | 16 | /// Given a TcpStream: 17 | /// - Deserialize the message 18 | /// - Serialize and write the echo message to the stream 19 | fn handle_connection(stream: TcpStream) -> io::Result<()> { 20 | let peer_addr = stream.peer_addr().expect("Stream has peer_addr"); 21 | eprintln!("Incoming from {}", peer_addr); 22 | let mut codec = LinesCodec::new(stream)?; 23 | 24 | let message: String = codec 25 | .read_message() 26 | // Reverse message 27 | .map(|m| m.chars().rev().collect())?; 28 | codec.send_message(&message)?; 29 | Ok(()) 30 | } 31 | 32 | fn main() -> io::Result<()> { 33 | let args = Args::from_args(); 34 | eprintln!("Starting server on '{}'", args.addr); 35 | 36 | let listener = TcpListener::bind(args.addr)?; 37 | for stream in listener.incoming() { 38 | if let Ok(stream) = stream { 39 | std::thread::spawn(move || { 40 | handle_connection(stream).map_err(|e| eprintln!("Error: {}", e)) 41 | }); 42 | } 43 | } 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /lines/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Shared code between client & server 2 | 3 | use std::io::{self, BufRead, Write}; 4 | use std::net::TcpStream; 5 | 6 | pub const DEFAULT_SERVER_ADDR: &str = "127.0.0.1:4000"; 7 | 8 | ///A smarter implementation of `extract_line` that supports writing messages also 9 | pub struct LinesCodec { 10 | reader: io::BufReader, 11 | writer: io::LineWriter, 12 | } 13 | 14 | impl LinesCodec { 15 | /// Encapsulate a TcpStream with reader/writer functionality 16 | pub fn new(stream: TcpStream) -> io::Result { 17 | let writer = io::LineWriter::new(stream.try_clone()?); 18 | let reader = io::BufReader::new(stream); 19 | Ok(Self { reader, writer }) 20 | } 21 | 22 | /// Write this line (with a '\n' suffix) to the TcpStream 23 | pub fn send_message(&mut self, message: &str) -> io::Result<()> { 24 | self.writer.write_all(&message.as_bytes())?; 25 | // This will also signal a `writer.flush()` for us! 26 | self.writer.write(&['\n' as u8])?; 27 | Ok(()) 28 | } 29 | 30 | /// Read a received message from the TcpStream 31 | pub fn read_message(&mut self) -> io::Result { 32 | let mut line = String::new(); 33 | // Use `BufRead::read_line()` to read a line from the TcpStream 34 | self.reader.read_line(&mut line)?; 35 | line.pop(); // Drop the trailing "\n" 36 | Ok(line) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /protocol/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "ansi_term" 5 | version = "0.11.0" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 8 | dependencies = [ 9 | "winapi", 10 | ] 11 | 12 | [[package]] 13 | name = "atty" 14 | version = "0.2.14" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 17 | dependencies = [ 18 | "hermit-abi", 19 | "libc", 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "bitflags" 25 | version = "1.2.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 28 | 29 | [[package]] 30 | name = "byteorder" 31 | version = "1.3.4" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" 34 | 35 | [[package]] 36 | name = "clap" 37 | version = "2.33.1" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" 40 | dependencies = [ 41 | "ansi_term", 42 | "atty", 43 | "bitflags", 44 | "strsim", 45 | "textwrap", 46 | "unicode-width", 47 | "vec_map", 48 | ] 49 | 50 | [[package]] 51 | name = "heck" 52 | version = "0.3.1" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" 55 | dependencies = [ 56 | "unicode-segmentation", 57 | ] 58 | 59 | [[package]] 60 | name = "hermit-abi" 61 | version = "0.1.14" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" 64 | dependencies = [ 65 | "libc", 66 | ] 67 | 68 | [[package]] 69 | name = "lazy_static" 70 | version = "1.4.0" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 73 | 74 | [[package]] 75 | name = "libc" 76 | version = "0.2.71" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" 79 | 80 | [[package]] 81 | name = "proc-macro-error" 82 | version = "1.0.2" 83 | source = "registry+https://github.com/rust-lang/crates.io-index" 84 | checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" 85 | dependencies = [ 86 | "proc-macro-error-attr", 87 | "proc-macro2", 88 | "quote", 89 | "syn", 90 | "version_check", 91 | ] 92 | 93 | [[package]] 94 | name = "proc-macro-error-attr" 95 | version = "1.0.2" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" 98 | dependencies = [ 99 | "proc-macro2", 100 | "quote", 101 | "syn", 102 | "syn-mid", 103 | "version_check", 104 | ] 105 | 106 | [[package]] 107 | name = "proc-macro2" 108 | version = "1.0.18" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" 111 | dependencies = [ 112 | "unicode-xid", 113 | ] 114 | 115 | [[package]] 116 | name = "quote" 117 | version = "1.0.7" 118 | source = "registry+https://github.com/rust-lang/crates.io-index" 119 | checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" 120 | dependencies = [ 121 | "proc-macro2", 122 | ] 123 | 124 | [[package]] 125 | name = "strsim" 126 | version = "0.8.0" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 129 | 130 | [[package]] 131 | name = "structopt" 132 | version = "0.3.15" 133 | source = "registry+https://github.com/rust-lang/crates.io-index" 134 | checksum = "de2f5e239ee807089b62adce73e48c625e0ed80df02c7ab3f068f5db5281065c" 135 | dependencies = [ 136 | "clap", 137 | "lazy_static", 138 | "structopt-derive", 139 | ] 140 | 141 | [[package]] 142 | name = "structopt-derive" 143 | version = "0.4.8" 144 | source = "registry+https://github.com/rust-lang/crates.io-index" 145 | checksum = "510413f9de616762a4fbeab62509bf15c729603b72d7cd71280fbca431b1c118" 146 | dependencies = [ 147 | "heck", 148 | "proc-macro-error", 149 | "proc-macro2", 150 | "quote", 151 | "syn", 152 | ] 153 | 154 | [[package]] 155 | name = "syn" 156 | version = "1.0.33" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" 159 | dependencies = [ 160 | "proc-macro2", 161 | "quote", 162 | "unicode-xid", 163 | ] 164 | 165 | [[package]] 166 | name = "syn-mid" 167 | version = "0.5.0" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" 170 | dependencies = [ 171 | "proc-macro2", 172 | "quote", 173 | "syn", 174 | ] 175 | 176 | [[package]] 177 | name = "tcp_demo_protocol" 178 | version = "0.1.0" 179 | dependencies = [ 180 | "byteorder", 181 | "structopt", 182 | ] 183 | 184 | [[package]] 185 | name = "textwrap" 186 | version = "0.11.0" 187 | source = "registry+https://github.com/rust-lang/crates.io-index" 188 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 189 | dependencies = [ 190 | "unicode-width", 191 | ] 192 | 193 | [[package]] 194 | name = "unicode-segmentation" 195 | version = "1.6.0" 196 | source = "registry+https://github.com/rust-lang/crates.io-index" 197 | checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" 198 | 199 | [[package]] 200 | name = "unicode-width" 201 | version = "0.1.7" 202 | source = "registry+https://github.com/rust-lang/crates.io-index" 203 | checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" 204 | 205 | [[package]] 206 | name = "unicode-xid" 207 | version = "0.2.0" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 210 | 211 | [[package]] 212 | name = "vec_map" 213 | version = "0.8.2" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 216 | 217 | [[package]] 218 | name = "version_check" 219 | version = "0.9.2" 220 | source = "registry+https://github.com/rust-lang/crates.io-index" 221 | checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" 222 | 223 | [[package]] 224 | name = "winapi" 225 | version = "0.3.8" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 228 | dependencies = [ 229 | "winapi-i686-pc-windows-gnu", 230 | "winapi-x86_64-pc-windows-gnu", 231 | ] 232 | 233 | [[package]] 234 | name = "winapi-i686-pc-windows-gnu" 235 | version = "0.4.0" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 238 | 239 | [[package]] 240 | name = "winapi-x86_64-pc-windows-gnu" 241 | version = "0.4.0" 242 | source = "registry+https://github.com/rust-lang/crates.io-index" 243 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 244 | -------------------------------------------------------------------------------- /protocol/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tcp_demo_protocol" 3 | version = "0.1.0" 4 | authors = ["Mat Wood "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | byteorder = "1.3.4" 9 | structopt = "0.3.14" -------------------------------------------------------------------------------- /protocol/README.md: -------------------------------------------------------------------------------- 1 | # Create a Messaging Protocol for TcpStream 2 | 3 | In this series so far we've learned how to [read & write bytes](../raw) with `TcpStream` and then how to [abstract over that with a `LinesCodec`](../lines) for sending and receiving `String` messages. In this demo we'll look into what it takes to build a custom protocol for message passing more than a single type of thing (like a `String`). 4 | 5 | # Defining our Message Structs 6 | To give our client and server more options for communicating we'll create: 7 | 8 | - A `Request` message that allow the client to request *either*: 9 | - Echo a string 10 | - Jumble a string with a specified amount of jumbling entropy 11 | - A `Response` message for the server to respond with the successfully echo/jumbled `String` 12 | 13 | ```rust 14 | /// Request object (client -> server) 15 | #[derive(Debug)] 16 | pub enum Request { 17 | /// Echo a message back 18 | Echo(String), 19 | /// Jumble up a message with given amount of entropy before returning 20 | Jumble { message: String, amount: u16 }, 21 | } 22 | 23 | /// Response object from server 24 | /// 25 | /// In the real-world, this would likely be an enum as well to signal Success vs. Error 26 | /// But since we're showing that capability with the `Request` struct, we'll keep this one simple 27 | #[derive(Debug)] 28 | pub struct Response(pub String); 29 | ``` 30 | 31 | # Serialization 32 | In the previous demos, we relied on `String::as_bytes()` to serialize the `String` characters to the byte slice (`&[u8]`) we pass to `TcpStream::write_all()`. The structs defined above don't have any default serialization capabilities so this example focuses on how to implement that ourselves. 33 | 34 | 35 | ## Serialization/Deserialization Libraries 36 | **Quick detour**: As you'll soon find out, serializing our custom structs is a lot of work. The good news is that there are some really incredible, performant, and battle-tested libraries to make doing this easier! It's fun to study how this can be implemented, but I highly recommend checking out these libraries for your actual serialization needs: 37 | 38 | - [Serde](https://docs.rs/serde/1.0.114/serde/index.html) 39 | - [tokio_util::codec](https://docs.rs/tokio-util/0.3.1/tokio_util/codec/index.html) 40 | - [bincode](https://github.com/servo/bincode) 41 | 42 | ## Serializing the Request struct 43 | Serialization & Deserialization needs to be **symmetric** (i.e., it round-trips) so that the struct serialized by the client is deserialized by the server back into an identical struct. 44 | 45 | Serializing Symmetry (round-trip) pseudo-code: 46 | ```rust 47 | let message = Request { ... }; 48 | 49 | // Equivalent to std::str::from_utf8("Hello".as_bytes()) 50 | let roundtripped_message = deserialize_message(serialize_message(&message)); 51 | 52 | // Symmetric serialization means these will match (given the struct has `Eq`) 53 | assert_eq!(message, roundtripped_message); 54 | ``` 55 | 56 | In order to serialize `Request` we need to decide what it looks like on the wire (as a `[u8]`). Since `Request` has two "types" (Echo and Jumble), we'll need to encode that type info to instruct the deserialization code correctly. Here is an approach for the `Request` byte layout: 57 | 58 | ``` 59 | | u8 | u16 | [u8] | ... u16 | ... [u8] | 60 | | type | length | bytes | ... length | ... bytes | 61 | ^--- struct field --^-- possibly more fields? --^-- ... 62 | ``` 63 | 64 | - **type**: A codified representation of the `Request` type 65 | - E.g. Echo == 1, Jumble == 2 66 | - **length**/**bytes**: The **L**ength and **V**alue from **T**ype-Length-Value [TLV] 67 | - Each message struct has specific field types so we can get away with only LV, but TLV is useful when fields are variadic and not derivable otherwise 68 | - **possibly more fields**: 69 | - Again, each message struct knows it's fields for deserialization, so these length/byte groups can repeat for each member field when needed (like in the case of Jumble) 70 | 71 | We know how to serialize a `String` with `as_bytes()`, but for number values we can use [byteorder](https://crates.io/crates/byteorder) to serialize with the correct [Endianness](https://en.wikipedia.org/wiki/Endianness) (spoiler: `BigEndian`, aliased as `NetworkEndian`). Let's walk through the serialization steps: 72 | 73 | ### Request Type 74 | As we can see above we need to codify our `Request` types (Echo and Jumble) into a number (`u8`, which allows us up to 255 types!). To make this definition clear we can implement `From` for `Request` -> `u8` to have a consistent way of serializing the type: 75 | 76 | ```rust 77 | use std::convert::From; 78 | 79 | /// Encode the Request type as a single byte (as long as we don't exceed 255 types) 80 | /// 81 | /// We use `&Request` since we don't actually need to own or mutate the request fields 82 | impl From<&Request> for u8 { 83 | fn from(req: &Request) -> Self { 84 | match req { 85 | Request::Echo(_) => 1, 86 | Request::Jumble { .. } => 2, 87 | } 88 | } 89 | } 90 | ``` 91 | 92 | ### Request Fields 93 | Let's finish the building blocks for serializing the `Request` with examples for writing the **length**/**value** for `String` and numbers: 94 | 95 | `String` is straightforward and we've used `as_bytes()` in previous demos, and `byteorder` adds extension methods to `Write` for numbers: 96 | ```rust 97 | use std::io::{self, Write}; 98 | use byteorder::{NetworkEndian, WriteBytesExt}; 99 | 100 | let mut bytes: Vec = vec![]; // Just an example buffer that supports `Write` 101 | 102 | let message = String::from("Hello"); 103 | // byteorder in action 104 | bytes.write_u16::(message.len()).unwrap(); 105 | bytes.write_all(message.as_bytes()).unwrap(); 106 | ``` 107 | 108 | ### Write Request 109 | We now have all the pieces to add a method for `Request` that receives a mutable reference to some buffer that implements `Write` (like our `TcpStream` :)), and serialize a `Request::Echo`: 110 | 111 | ```rust 112 | /// Starts with a type, and then is an arbitrary length of (length/bytes) tuples 113 | impl Request { 114 | /// Serialize Request to bytes (to send to server) 115 | pub fn serialize(&self, buf: &mut impl Write) -> io::Result<()> { 116 | // Derive the u8 Type field from `impl From<&Request> for u8` 117 | buf.write_u8(self.into())?; 118 | 119 | // Select the serialization based on our `Request` type 120 | match self { 121 | Request::Echo(message) => { 122 | // Write the variable length message string, preceded by it's length 123 | let message = message.as_bytes(); 124 | buf.write_u16::(message.len() as u16)?; 125 | buf.write_all(&message)?; 126 | } 127 | Request::Jumble { message, amount } => { 128 | todo!() 129 | } 130 | } 131 | Ok(()) 132 | } 133 | } 134 | ``` 135 | 136 | Now the `Request::Jumble` serialization is just a slight variation (adding the `amount` field after the `message`): 137 | 138 | ```rust 139 | // ... 140 | match self { 141 | // ... 142 | Request::Jumble { message, amount } => { 143 | // Write the variable length message string, preceded by it's length 144 | let message_bytes = message.as_bytes(); 145 | buf.write_u16::(message_bytes.len() as u16)?; 146 | buf.write_all(&message_bytes)?; 147 | 148 | // We know that `amount` is always 2 bytes long 149 | buf.write_u16::(2)?; 150 | // followed by the amount value 151 | buf.write_u16::(*amount)?; 152 | } 153 | } 154 | /// ... 155 | ``` 156 | 157 | Tada! A serialized `Request` in the bank! The `Response` struct is even simpler with its single `String` value, so you can review the serialization code in the [demo lib.rs](src/lib.rs#L123) 158 | 159 | 160 | ## Deserializing the Request struct 161 | We already have the byte layout figured out so deserializing should essentially be the reverse of our `Request::serialize()` method above. `byteorder` also gives us `ReadBytesExt` to add extensions to `Read` that `TcpStream` implements. The trickiest part is reading the variable length `String` and this will happen in a few places for `Request::Echo`, `Request::Jumble`, and `Response` so let's break out this logic into a function: 162 | 163 | ```rust 164 | /// From a given readable buffer (TcpStream), read the next length (u16) and extract the string bytes ([u8]) 165 | fn extract_string(buf: &mut impl Read) -> io::Result { 166 | // byteorder ReadBytesExt 167 | let length = buf.read_u16::()?; 168 | 169 | // Given the length of our string, only read in that quantity of bytes 170 | let mut bytes = vec![0u8; length as usize]; 171 | buf.read_exact(&mut bytes)?; 172 | 173 | // And attempt to decode it as UTF8 174 | String::from_utf8(bytes).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid utf8")) 175 | } 176 | ``` 177 | 178 | Our `deserialize()` method should be straight-forward to read now especially with `extract_string` at our disposal: 179 | 180 | ```rust 181 | use std::io::{self, Read}; 182 | use byteorder::{NetworkEndian, ReadBytesExt}; 183 | 184 | 185 | impl Request { 186 | /// Deserialize Request from bytes (to receive from TcpStream) 187 | /// returning a `Request` struct 188 | pub fn deserialize(mut buf: &mut impl Read) -> io::Result { 189 | /// We'll match the same `u8` that is used to recognize which request type this is 190 | match buf.read_u8()? { 191 | // Echo 192 | 1 => Ok(Request::Echo(extract_string(&mut buf)?)), 193 | // Jumble 194 | 2 => { 195 | let message = extract_string(&mut buf)?; 196 | // amount length is not used since we know it's 2 bytes 197 | let _amount_len = buf.read_u16::()?; 198 | let amount = buf.read_u16::()?; 199 | Ok(Request::Jumble { message, amount }) 200 | } 201 | _ => Err(io::Error::new( 202 | io::ErrorKind::InvalidData, 203 | "Invalid Request Type", 204 | )), 205 | } 206 | } 207 | 208 | } 209 | ``` 210 | 211 | Wow! We now have the ability to test round-tripping of our structs! 212 | 213 | ```rust 214 | use crate::*; 215 | use std::io::Cursor; 216 | 217 | #[test] 218 | fn test_request_roundtrip() { 219 | let req = Request::Echo(String::from("Hello")); 220 | 221 | let mut bytes: Vec = vec![]; 222 | req.serialize(&mut bytes).unwrap(); 223 | 224 | let mut reader = Cursor::new(bytes); // Simulating our TcpStream 225 | let roundtrip_req = Request::deserialize(&mut reader).unwrap(); 226 | 227 | assert!(matches!(roundtrip_req, Request::Echo(_))); 228 | assert_eq!(roundtrip_req.message(), "Hello"); 229 | } 230 | ``` 231 | 232 | # Using our new Protocol 233 | If you're still with me here, congrats! That was a lot of work and you're about to see how it all pays off when we use the message structs in our client and server. 234 | 235 | I'll leave it up as an exercise to check out the [full protocol implementation](src/lib.rs) where we add `Serialize` and `Deserialize` traits for our methods above and make using our protocol as easy as: 236 | 237 | ```rust 238 | use std::io; 239 | 240 | fn main() -> io::Request<()> { 241 | let req = Request::Jumble { 242 | message: "Hello", 243 | amount: 80, 244 | }; 245 | 246 | Protocol::connect("127.0.0.1:4000") 247 | .and_then(|mut client| { 248 | client.send_message(&req)?; 249 | Ok(client) 250 | }) 251 | .and_then(|mut client| client.read_message::()) 252 | .map(|resp| println!("{}", resp.message())) 253 | } 254 | ``` 255 | 256 | 257 | # Running the demo 258 | From within this `./protocol` directory we can start the server, and then in another terminal (tmux pane, ssh session, etc), run the client with a message of your choice 259 | 260 | Server 261 | ```sh 262 | $ cargo run --bin server 263 | Starting server on '127.0.0.1:4000' 264 | ... 265 | ``` 266 | 267 | Client 268 | ```sh 269 | $ cargo run --bin client -- Hello 270 | Connecting to 127.0.0.1:4000 271 | 'Hello' from the other side! 272 | $ cargo run --bin client -- "This is my message" -j 100 273 | Connecting to 127.0.0.1:4000 274 | issageThis s my me 275 | ``` -------------------------------------------------------------------------------- /protocol/src/bin/client.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::net::SocketAddr; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_protocol::{Protocol, Request, Response, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "client")] 10 | struct Args { 11 | message: String, 12 | // Jumble the message by how much (default = will not jumble) 13 | #[structopt(short, long, default_value = "0")] 14 | jumble: u16, 15 | /// Server destination address 16 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 17 | addr: SocketAddr, 18 | } 19 | 20 | fn main() -> io::Result<()> { 21 | let args = Args::from_args(); 22 | 23 | let req = if args.jumble > 0 { 24 | Request::Jumble { 25 | message: args.message, 26 | amount: args.jumble, 27 | } 28 | } else { 29 | Request::Echo(args.message) 30 | }; 31 | 32 | Protocol::connect(args.addr) 33 | .and_then(|mut client| { 34 | client.send_message(&req)?; 35 | Ok(client) 36 | }) 37 | .and_then(|mut client| client.read_message::()) 38 | .map(|resp| println!("{}", resp.message())) 39 | } 40 | -------------------------------------------------------------------------------- /protocol/src/bin/server.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::net::{SocketAddr, TcpListener, TcpStream}; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_protocol::{Protocol, Request, Response, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "server")] 10 | struct Args { 11 | /// Service listening address 12 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 13 | addr: SocketAddr, 14 | } 15 | 16 | /// Given a TcpStream: 17 | /// - Deserialize the request 18 | /// - Handle the request 19 | /// - Serialize and write the Response to the stream 20 | fn handle_connection(stream: TcpStream) -> io::Result<()> { 21 | let peer_addr = stream.peer_addr().expect("Stream has peer_addr"); 22 | let mut protocol = Protocol::with_stream(stream)?; 23 | 24 | let request = protocol.read_message::()?; 25 | eprintln!("Incoming {:?} [{}]", request, peer_addr); 26 | let resp = match request { 27 | Request::Echo(message) => Response(format!("'{}' from the other side!", message)), 28 | Request::Jumble { message, amount } => Response(jumble_message(&message, amount)), 29 | }; 30 | 31 | protocol.send_message(&resp) 32 | } 33 | 34 | /// Shake the characters around a little bit 35 | fn jumble_message(message: &str, amount: u16) -> String { 36 | let mut chars: Vec = message.chars().collect(); 37 | // Do some jumbling 38 | for i in 1..=amount as usize { 39 | let shuffle = i % chars.len(); 40 | chars.swap(0, shuffle); 41 | } 42 | chars.into_iter().collect() 43 | } 44 | 45 | fn main() -> io::Result<()> { 46 | let args = Args::from_args(); 47 | eprintln!("Starting server on '{}'", args.addr); 48 | 49 | let listener = TcpListener::bind(args.addr)?; 50 | for stream in listener.incoming() { 51 | if let Ok(stream) = stream { 52 | std::thread::spawn(move || { 53 | handle_connection(stream).map_err(|e| eprintln!("Error: {}", e)) 54 | }); 55 | } 56 | } 57 | Ok(()) 58 | } 59 | -------------------------------------------------------------------------------- /protocol/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Message Serialization/Deserialization (Protocol) for client <-> server communication 2 | //! 3 | //! Ideally you would use some existing Serialization/Deserialization, 4 | //! but this is here to see what's going on under the hood. 5 | //! 6 | //! ## Libraries for serialization/deserialization: 7 | //! [Serde](https://docs.rs/serde/1.0.114/serde/index.html) 8 | //! [tokio_util::codec](https://docs.rs/tokio-util/0.3.1/tokio_util/codec/index.html) 9 | //! [bincode](https://github.com/servo/bincode) 10 | 11 | use std::convert::From; 12 | use std::io::{self, Read, Write}; 13 | use std::net::{SocketAddr, TcpStream}; 14 | 15 | use byteorder::{NetworkEndian, ReadBytesExt, WriteBytesExt}; 16 | 17 | pub const DEFAULT_SERVER_ADDR: &str = "127.0.0.1:4000"; 18 | 19 | /// Trait for something that can be converted to bytes (&[u8]) 20 | pub trait Serialize { 21 | /// Serialize to a `Write`able buffer 22 | fn serialize(&self, buf: &mut impl Write) -> io::Result; 23 | } 24 | /// Trait for something that can be converted from bytes (&[u8]) 25 | pub trait Deserialize { 26 | /// The type that this deserializes to 27 | type Output; 28 | 29 | /// Deserialize from a `Read`able buffer 30 | fn deserialize(buf: &mut impl Read) -> io::Result; 31 | } 32 | 33 | /// Request object (client -> server) 34 | #[derive(Debug)] 35 | pub enum Request { 36 | /// Echo a message back 37 | Echo(String), 38 | /// Jumble up a message with given amount of entropy before echoing 39 | Jumble { message: String, amount: u16 }, 40 | } 41 | 42 | /// Encode the Request type as a single byte (as long as we don't exceed 255 types) 43 | /// 44 | /// We use `&Request` since we don't actually need to own or mutate the request fields 45 | impl From<&Request> for u8 { 46 | fn from(req: &Request) -> Self { 47 | match req { 48 | Request::Echo(_) => 1, 49 | Request::Jumble { .. } => 2, 50 | } 51 | } 52 | } 53 | 54 | /// Message format for Request is: 55 | /// ```ignore 56 | /// | u8 | u16 | [u8] | ... u16 | ... [u8] | 57 | /// | type | length | value bytes | ... length | ... value bytes | 58 | /// ``` 59 | /// 60 | /// Starts with a type, and then is an arbitrary length of (length/bytes) tuples 61 | impl Request { 62 | /// View the message portion of this request 63 | pub fn message(&self) -> &str { 64 | match self { 65 | Request::Echo(message) => &message, 66 | Request::Jumble { message, .. } => &message, 67 | } 68 | } 69 | } 70 | 71 | impl Serialize for Request { 72 | /// Serialize Request to bytes (to send to server) 73 | fn serialize(&self, buf: &mut impl Write) -> io::Result { 74 | buf.write_u8(self.into())?; // Message Type byte 75 | let mut bytes_written: usize = 1; 76 | match self { 77 | Request::Echo(message) => { 78 | // Write the variable length message string, preceded by it's length 79 | let message = message.as_bytes(); 80 | buf.write_u16::(message.len() as u16)?; 81 | buf.write_all(&message)?; 82 | bytes_written += 2 + message.len(); 83 | } 84 | Request::Jumble { message, amount } => { 85 | // Write the variable length message string, preceded by it's length 86 | let message_bytes = message.as_bytes(); 87 | buf.write_u16::(message_bytes.len() as u16)?; 88 | buf.write_all(&message_bytes)?; 89 | bytes_written += 2 + message.len(); 90 | 91 | // We know that `amount` is always 2 bytes long, but are adding 92 | // the length here to stay consistent 93 | buf.write_u16::(2)?; 94 | buf.write_u16::(*amount)?; 95 | bytes_written += 4; 96 | } 97 | } 98 | Ok(bytes_written) 99 | } 100 | } 101 | 102 | impl Deserialize for Request { 103 | type Output = Request; 104 | 105 | /// Deserialize Request from bytes (to receive from TcpStream) 106 | fn deserialize(mut buf: &mut impl Read) -> io::Result { 107 | match buf.read_u8()? { 108 | // Echo 109 | 1 => Ok(Request::Echo(extract_string(&mut buf)?)), 110 | // Jumble 111 | 2 => { 112 | let message = extract_string(&mut buf)?; 113 | let _amount_len = buf.read_u16::()?; 114 | let amount = buf.read_u16::()?; 115 | Ok(Request::Jumble { message, amount }) 116 | } 117 | _ => Err(io::Error::new( 118 | io::ErrorKind::InvalidData, 119 | "Invalid Request Type", 120 | )), 121 | } 122 | } 123 | } 124 | 125 | /// Response object from server 126 | /// 127 | /// In the real-world, this would likely be an enum as well to signal Success vs. Error 128 | /// But since we're showing that capability with the `Request` struct, we'll keep this one simple 129 | #[derive(Debug)] 130 | pub struct Response(pub String); 131 | 132 | /// Message format for Response is: 133 | /// ```ignore 134 | /// | u16 | [u8] | 135 | /// | length | value bytes | 136 | /// ``` 137 | /// 138 | impl Response { 139 | /// Create a new response with a given message 140 | pub fn new(message: String) -> Self { 141 | Self(message) 142 | } 143 | 144 | /// Get the response message value 145 | pub fn message(&self) -> &str { 146 | &self.0 147 | } 148 | } 149 | 150 | impl Serialize for Response { 151 | /// Serialize Response to bytes (to send to client) 152 | /// 153 | /// Returns the number of bytes written 154 | fn serialize(&self, buf: &mut impl Write) -> io::Result { 155 | let resp_bytes = self.0.as_bytes(); 156 | buf.write_u16::(resp_bytes.len() as u16)?; 157 | buf.write_all(&resp_bytes)?; 158 | Ok(3 + resp_bytes.len()) // Type + len + bytes 159 | } 160 | } 161 | 162 | impl Deserialize for Response { 163 | type Output = Response; 164 | /// Deserialize Response to bytes (to receive from server) 165 | fn deserialize(mut buf: &mut impl Read) -> io::Result { 166 | let value = extract_string(&mut buf)?; 167 | Ok(Response(value)) 168 | } 169 | } 170 | 171 | /// From a given readable buffer, read the next length (u16) and extract the string bytes 172 | fn extract_string(buf: &mut impl Read) -> io::Result { 173 | // byteorder ReadBytesExt 174 | let length = buf.read_u16::()?; 175 | // Given the length of our string, only read in that quantity of bytes 176 | let mut bytes = vec![0u8; length as usize]; 177 | buf.read_exact(&mut bytes)?; 178 | // And attempt to decode it as UTF8 179 | String::from_utf8(bytes).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid utf8")) 180 | } 181 | 182 | /// Abstracted Protocol that wraps a TcpStream and manages 183 | /// sending & receiving of messages 184 | pub struct Protocol { 185 | reader: io::BufReader, 186 | stream: TcpStream, 187 | } 188 | 189 | impl Protocol { 190 | /// Wrap a TcpStream with Protocol 191 | pub fn with_stream(stream: TcpStream) -> io::Result { 192 | Ok(Self { 193 | reader: io::BufReader::new(stream.try_clone()?), 194 | stream, 195 | }) 196 | } 197 | 198 | /// Establish a connection, wrap stream in BufReader/Writer 199 | pub fn connect(dest: SocketAddr) -> io::Result { 200 | let stream = TcpStream::connect(dest)?; 201 | eprintln!("Connecting to {}", dest); 202 | Self::with_stream(stream) 203 | } 204 | 205 | /// Serialize a message to the server and write it to the TcpStream 206 | pub fn send_message(&mut self, message: &impl Serialize) -> io::Result<()> { 207 | message.serialize(&mut self.stream)?; 208 | self.stream.flush() 209 | } 210 | 211 | /// Read a message from the inner TcpStream 212 | /// 213 | /// NOTE: Will block until there's data to read (or deserialize fails with io::ErrorKind::Interrupted) 214 | /// so only use when a message is expected to arrive 215 | pub fn read_message(&mut self) -> io::Result { 216 | T::deserialize(&mut self.reader) 217 | } 218 | } 219 | 220 | #[cfg(test)] 221 | mod test { 222 | use super::*; 223 | use std::io::Cursor; 224 | 225 | #[test] 226 | fn test_request_echo_roundtrip() { 227 | let req = Request::Echo(String::from("Hello")); 228 | 229 | let mut bytes: Vec = vec![]; 230 | req.serialize(&mut bytes).unwrap(); 231 | 232 | let mut reader = Cursor::new(bytes); 233 | let roundtrip_req = Request::deserialize(&mut reader).unwrap(); 234 | 235 | assert!(matches!(roundtrip_req, Request::Echo(_))); 236 | assert_eq!(roundtrip_req.message(), "Hello"); 237 | } 238 | 239 | #[test] 240 | fn test_request_jumble_roundtrip() { 241 | let req = Request::Jumble { 242 | message: String::from("Hello"), 243 | amount: 42, 244 | }; 245 | 246 | let mut bytes: Vec = vec![]; 247 | req.serialize(&mut bytes).unwrap(); 248 | 249 | let mut reader = Cursor::new(bytes); 250 | let roundtrip_req = Request::deserialize(&mut reader).unwrap(); 251 | 252 | assert!(matches!(roundtrip_req, Request::Jumble { .. })); 253 | assert_eq!(roundtrip_req.message(), "Hello"); 254 | } 255 | 256 | #[test] 257 | fn test_response_roundtrip() { 258 | let resp = Response(String::from("Hello")); 259 | 260 | let mut bytes: Vec = vec![]; 261 | resp.serialize(&mut bytes).unwrap(); 262 | 263 | let mut reader = Cursor::new(bytes); 264 | let roundtrip_resp = Response::deserialize(&mut reader).unwrap(); 265 | 266 | assert!(matches!(roundtrip_resp, Response(_))); 267 | assert_eq!(roundtrip_resp.0, "Hello"); 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /raw/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "ansi_term" 5 | version = "0.11.0" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 8 | dependencies = [ 9 | "winapi", 10 | ] 11 | 12 | [[package]] 13 | name = "atty" 14 | version = "0.2.14" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 17 | dependencies = [ 18 | "hermit-abi", 19 | "libc", 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "bitflags" 25 | version = "1.2.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 28 | 29 | [[package]] 30 | name = "clap" 31 | version = "2.33.1" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" 34 | dependencies = [ 35 | "ansi_term", 36 | "atty", 37 | "bitflags", 38 | "strsim", 39 | "textwrap", 40 | "unicode-width", 41 | "vec_map", 42 | ] 43 | 44 | [[package]] 45 | name = "heck" 46 | version = "0.3.1" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" 49 | dependencies = [ 50 | "unicode-segmentation", 51 | ] 52 | 53 | [[package]] 54 | name = "hermit-abi" 55 | version = "0.1.14" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" 58 | dependencies = [ 59 | "libc", 60 | ] 61 | 62 | [[package]] 63 | name = "lazy_static" 64 | version = "1.4.0" 65 | source = "registry+https://github.com/rust-lang/crates.io-index" 66 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 67 | 68 | [[package]] 69 | name = "libc" 70 | version = "0.2.71" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" 73 | 74 | [[package]] 75 | name = "proc-macro-error" 76 | version = "1.0.2" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" 79 | dependencies = [ 80 | "proc-macro-error-attr", 81 | "proc-macro2", 82 | "quote", 83 | "syn", 84 | "version_check", 85 | ] 86 | 87 | [[package]] 88 | name = "proc-macro-error-attr" 89 | version = "1.0.2" 90 | source = "registry+https://github.com/rust-lang/crates.io-index" 91 | checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" 92 | dependencies = [ 93 | "proc-macro2", 94 | "quote", 95 | "syn", 96 | "syn-mid", 97 | "version_check", 98 | ] 99 | 100 | [[package]] 101 | name = "proc-macro2" 102 | version = "1.0.18" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" 105 | dependencies = [ 106 | "unicode-xid", 107 | ] 108 | 109 | [[package]] 110 | name = "quote" 111 | version = "1.0.7" 112 | source = "registry+https://github.com/rust-lang/crates.io-index" 113 | checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" 114 | dependencies = [ 115 | "proc-macro2", 116 | ] 117 | 118 | [[package]] 119 | name = "strsim" 120 | version = "0.8.0" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 123 | 124 | [[package]] 125 | name = "structopt" 126 | version = "0.3.15" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "de2f5e239ee807089b62adce73e48c625e0ed80df02c7ab3f068f5db5281065c" 129 | dependencies = [ 130 | "clap", 131 | "lazy_static", 132 | "structopt-derive", 133 | ] 134 | 135 | [[package]] 136 | name = "structopt-derive" 137 | version = "0.4.8" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "510413f9de616762a4fbeab62509bf15c729603b72d7cd71280fbca431b1c118" 140 | dependencies = [ 141 | "heck", 142 | "proc-macro-error", 143 | "proc-macro2", 144 | "quote", 145 | "syn", 146 | ] 147 | 148 | [[package]] 149 | name = "syn" 150 | version = "1.0.33" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" 153 | dependencies = [ 154 | "proc-macro2", 155 | "quote", 156 | "unicode-xid", 157 | ] 158 | 159 | [[package]] 160 | name = "syn-mid" 161 | version = "0.5.0" 162 | source = "registry+https://github.com/rust-lang/crates.io-index" 163 | checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" 164 | dependencies = [ 165 | "proc-macro2", 166 | "quote", 167 | "syn", 168 | ] 169 | 170 | [[package]] 171 | name = "tcp_demo_raw" 172 | version = "0.1.0" 173 | dependencies = [ 174 | "structopt", 175 | ] 176 | 177 | [[package]] 178 | name = "textwrap" 179 | version = "0.11.0" 180 | source = "registry+https://github.com/rust-lang/crates.io-index" 181 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 182 | dependencies = [ 183 | "unicode-width", 184 | ] 185 | 186 | [[package]] 187 | name = "unicode-segmentation" 188 | version = "1.6.0" 189 | source = "registry+https://github.com/rust-lang/crates.io-index" 190 | checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" 191 | 192 | [[package]] 193 | name = "unicode-width" 194 | version = "0.1.7" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" 197 | 198 | [[package]] 199 | name = "unicode-xid" 200 | version = "0.2.0" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 203 | 204 | [[package]] 205 | name = "vec_map" 206 | version = "0.8.2" 207 | source = "registry+https://github.com/rust-lang/crates.io-index" 208 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 209 | 210 | [[package]] 211 | name = "version_check" 212 | version = "0.9.2" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" 215 | 216 | [[package]] 217 | name = "winapi" 218 | version = "0.3.8" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 221 | dependencies = [ 222 | "winapi-i686-pc-windows-gnu", 223 | "winapi-x86_64-pc-windows-gnu", 224 | ] 225 | 226 | [[package]] 227 | name = "winapi-i686-pc-windows-gnu" 228 | version = "0.4.0" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 231 | 232 | [[package]] 233 | name = "winapi-x86_64-pc-windows-gnu" 234 | version = "0.4.0" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 237 | -------------------------------------------------------------------------------- /raw/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tcp_demo_raw" 3 | version = "0.1.0" 4 | authors = ["Mat Wood "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | structopt = "0.3.14" -------------------------------------------------------------------------------- /raw/README.md: -------------------------------------------------------------------------------- 1 | # Reading and writing data with Rust's TcpStream 2 | 3 | For the start of our journey with [TcpStream](https://doc.rust-lang.org/stable/std/net/struct.TcpStream.html) we'll take a look at what it takes to send and receive raw bytes. 4 | 5 | This demo is going to build up some concepts that result in a [demo found here](https://github.com/thepacketgeek/rust-tcpstream-demo/tree/master/raw). The working concept is that we want to have a client send some data (a `String`) to a server that will echo the message back to the client. In further demos, we'll progress this concept to do slightly more interesting things with the data on the server before sending back. 6 | 7 | # Writing Data 8 | Let's start with our client and figuring out how to send the message bytes (`String` characters) to the server. `TcpStream`'s implementation of the [Write](https://doc.rust-lang.org/stable/std/io/trait.Write.html) trait gives us functionality to write data to the stream (allowing TCP to send bytes across the wire). 9 | 10 | ### write() 11 | The [`write()`](https://doc.rust-lang.org/stable/std/io/trait.Write.html#tymethod.write) method takes a slice of bytes (`&[u8]`) and attempts to queue them in the `TcpStream`'s buffer (without sending on the wire yet) and returns the number of bytes successfully written as a `usize`. If more bytes are written than can be buffered, `write()` signals this with a `usize` less than the length of bytes sent (which you can see with the `bytes_written` variable in examples coming soon). 12 | 13 | After queuing the bytes in the TCP buffer, `flush()` signals to TCP that it should send the bytes. As the last bytes are sent, the **PSH** bit will be set to signal to the receiving TCP stack that this data is ready to send to the application. 14 | 15 | ```rust 16 | use std::io::{self, Write}; 17 | use std::net::TcpStream; 18 | 19 | fn main() -> io::Result<()> { 20 | // Establish a TCP connection with the farend 21 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 22 | 23 | // Buffer the bytes 24 | let _bytes_written = stream.write(b"Hello")?; 25 | // Tell TCP to send the buffered data on the wire 26 | stream.flush()?; 27 | 28 | Ok(()) 29 | } 30 | ``` 31 | 32 | To prevent missing data, the `bytes_written` should be validated and signaled if not all the data could be written, which could look like this: 33 | 34 | ```rust 35 | ... 36 | let data = b"Hello"; 37 | let bytes_written = stream.write(data)?; 38 | 39 | if bytes_written < data.len() { 40 | return Err(io::Error::new( 41 | io::ErrorKind::Interrupted, 42 | format!("Sent {}/{} bytes", bytes_written, data.len()), 43 | )); 44 | } 45 | 46 | stream.flush()?; 47 | ... 48 | ``` 49 | 50 | ### write_all() 51 | Validating the bytes queued is a common enough pattern to warrant a dedicated method to save us some work: [`write_all()`](https://doc.rust-lang.org/stable/std/io/trait.Write.html#method.write_all). A simplified version of the example above becomes: 52 | 53 | ```rust 54 | use std::io::{self, Write}; 55 | use std::net::TcpStream; 56 | 57 | fn main() -> io::Result<()> { 58 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 59 | 60 | // write_all() will return Err(io::Error(io::ErrorKind::Interrupted)) 61 | // if it is unable to queue all bytes 62 | stream.write_all(b"Hello")?; 63 | stream.flush() 64 | } 65 | ``` 66 | 67 | This is what the the TcpStream looks like in Wireshark. Notice the **PSH** bit set on the packet 5 (carrying the data segment "Hello"): 68 | 69 | ![tcp in wireshark](res/tcpstream_raw.png) 70 | 71 | # Reading Data 72 | Counterpart to the `Write` trait's `write()`/`write_all()`, the [Read](https://doc.rust-lang.org/stable/std/io/trait.Read.html) trait gives us `read()` for the ability to read received bytes from a `TcpStream`. It can be a bit more nuanced than `Write` and I'll show you why you'll tend to want to use the [BufRead](https://doc.rust-lang.org/stable/std/io/trait.BufRead.html) trait with [BufReader](https://doc.rust-lang.org/std/io/struct.BufReader.html). 73 | 74 | ### read() 75 | The `read()` method takes a mutable reference (`&mut`) to a buffer (like a `Vec` or `[u8]`) and will attempt to fill this buffer with received bytes. The nuanced side of `read()` is knowing **how many** bytes to read, as reading to a growable array will cause an error since it can't be filled: 76 | 77 | ``` 78 | Error("failed to fill whole buffer") 79 | ``` 80 | 81 | So, knowing how many bytes to read would be easy if we knew that the messages received are always a fixed size: 82 | 83 | ```rust 84 | use std::io::{self, Read}; 85 | use std::net::TcpStream; 86 | 87 | const MESSAGE_SIZE: usize = 5; 88 | 89 | fn main() -> io::Result<()> { 90 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 91 | 92 | // Array with a fixed size 93 | let mut rx_bytes = [0u8; MESSAGE_SIZE]; 94 | // Read from the current data in the TcpStream 95 | stream.read(&mut rx_bytes)?; 96 | 97 | let received = std::str::from_utf8(&rx_bytes).expect("valid utf8"); 98 | eprintln!("{}", received); 99 | Ok(()) 100 | } 101 | ``` 102 | 103 | Yet we live in a world with variable length strings and the approach above doesn't quite work out. However `read()` returns a `usize` of bytes_read, so we can get fancy reading to our statically sized array repeatedly until we have no more bytes to read: 104 | 105 | ```rust 106 | ... 107 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 108 | 109 | // Store all the bytes for our received String 110 | let mut received: Vec = vec![]; 111 | 112 | // Array with a fixed size 113 | let mut rx_bytes = [0u8; MESSAGE_SIZE]; 114 | loop { 115 | // Read from the current data in the TcpStream 116 | let bytes_read = stream.read(&mut rx_bytes)?; 117 | 118 | // However many bytes we read, extend the `received` string bytes 119 | received.extend_from_slice(&rx_bytes[..bytes_read]); 120 | 121 | // If we didn't fill the array 122 | // stop reading because there's no more data (we hope!) 123 | if bytes_read < MESSAGE_SIZE { 124 | break; 125 | } 126 | } 127 | 128 | String::from_utf8(received) 129 | .map(|msg| println!("{}", msg)) 130 | .map_err(|_| { 131 | io::Error::new( 132 | io::ErrorKind::InvalidData, 133 | "Couldn't parse received string as utf8", 134 | ) 135 | }) 136 | ... 137 | ``` 138 | 139 | Alas, there are some issues with this code. Each time `read()` is called can be a syscall to fetch bytes from the TCP stack and we're also stuck with the decision of how to size the fixed length array: 140 | - A large array (E.g. > 512 bytes) can waste stack space for bytes we may never read 141 | - A small array results in more calls to `read()`, which may be expensive 142 | 143 | Fortunately though there's a solution for us... 144 | 145 | ## BufRead and BufReader 146 | The [BufRead](https://doc.rust-lang.org/stable/std/io/trait.BufRead.html) trait and [BufReader](https://doc.rust-lang.org/std/io/struct.BufReader.html) give us some really convenient read capabilities: 147 | 148 | - `fill_buf()` and `consume()` - a tag team of methods to read & verify data 149 | - `read_line()` (or the `Iterator` version: `lines()`) allow us to read by line (String ending with a newline) 150 | - *You can see this one in action to create a `LinesCodec` in the [next demo](../lines)* 151 | 152 | ### fill_buff() and consume() 153 | Instead of repeatedly trying to fill an array with bytes, `BufRead` offers a `fill_buf()` method which returns a slice to all the bytes in the buffer, super convenient! 154 | 155 | `fill_buf()` does not free/drop the bytes read; a subsequent call to `consume()` is necessary to tell the `BufReader` that the bytes were successfully read and it can move its internal cursor. This combo is helpful in the case where TCP may not have received all the data yet, and your code needs to check for a delimiter (*cough* like a newline *cough*) or certain number of bytes: 156 | 157 | ```rust 158 | use std::io::{self, BufRead}; 159 | use std::net::TcpStream; 160 | 161 | fn main() -> io::Result<()> { 162 | let mut stream = TcpStream::connect("127.0.0.1:4000")?; 163 | 164 | // Wrap the stream in a BufReader, so we can use the BufRead methods 165 | let mut reader = io::BufReader::new(&mut stream); 166 | 167 | // Read current current data in the TcpStream 168 | let received: Vec = reader.fill_buf()?.to_vec(); 169 | 170 | // Do some processing or validation to make sure the whole line is present? 171 | // ... 172 | 173 | // Mark the bytes read as consumed so the buffer will not return them in a subsequent read 174 | reader.consume(received.len()); 175 | 176 | String::from_utf8(received) 177 | .map(|msg| println!("{}", msg)) 178 | .map_err(|_| { 179 | io::Error::new( 180 | io::ErrorKind::InvalidData, 181 | "Couldn't parse received string as utf8", 182 | ) 183 | }) 184 | } 185 | ``` 186 | 187 | # Conclusion 188 | 189 | We've covered how to read and write bytes with `TcpStream` but it could be easier for us. The [next demo](../lines) will expand on the usage of `BufRead` to build a Lines Codec to abstract away the read/write detail in the client and server code. 190 | 191 | # Running the demo 192 | From within this `./raw` directory we can start the server, and then in another terminal (tmux pane, ssh session, etc), run the client with a message of your choice 193 | 194 | Server 195 | ``` 196 | $ cargo run --bin server 197 | Starting server on '127.0.0.1:4000' 198 | ... 199 | ``` 200 | 201 | Client 202 | ``` 203 | $ cargo run --bin client -- Hello 204 | Hello 205 | ``` 206 | -------------------------------------------------------------------------------- /raw/res/tcpstream_raw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepacketgeek/rust-tcpstream-demo/ebb44428923e004e5fc700768d2d47713b289698/raw/res/tcpstream_raw.png -------------------------------------------------------------------------------- /raw/src/bin/client.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::net::{SocketAddr, TcpStream}; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_raw::{extract_string_unbuffered, write_data, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "client")] 10 | struct Args { 11 | message: String, 12 | /// Server destination address 13 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 14 | addr: SocketAddr, 15 | } 16 | 17 | fn main() -> io::Result<()> { 18 | let args = Args::from_args(); 19 | 20 | let mut stream = TcpStream::connect(args.addr)?; 21 | write_data(&mut stream, &args.message.as_bytes())?; 22 | 23 | // Now read & print the response 24 | // (this will block until all data has been received) 25 | extract_string_unbuffered(&mut stream).map(|resp| println!("{}", resp)) 26 | } 27 | -------------------------------------------------------------------------------- /raw/src/bin/server.rs: -------------------------------------------------------------------------------- 1 | use std::io::{self, BufReader, BufWriter}; 2 | use std::net::{SocketAddr, TcpListener, TcpStream}; 3 | 4 | use structopt::StructOpt; 5 | 6 | use tcp_demo_raw::{extract_string_buffered, write_data, DEFAULT_SERVER_ADDR}; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(name = "server")] 10 | struct Args { 11 | /// Service listening address 12 | #[structopt(long, default_value = DEFAULT_SERVER_ADDR, global = true)] 13 | addr: SocketAddr, 14 | } 15 | 16 | /// Given a TcpStream: 17 | /// - Deserialize the message 18 | /// - Serialize and write the echo message to the stream 19 | fn handle_connection(stream: TcpStream) -> io::Result<()> { 20 | let peer_addr = stream.peer_addr().expect("Stream has peer_addr"); 21 | eprintln!("Incoming from {}", peer_addr); 22 | let mut reader = BufReader::new(stream.try_clone()?); 23 | let mut writer = BufWriter::new(stream); 24 | 25 | let message = extract_string_buffered(&mut reader)?; 26 | write_data(&mut writer, &message.as_bytes()) 27 | } 28 | 29 | fn main() -> io::Result<()> { 30 | let args = Args::from_args(); 31 | eprintln!("Starting server on '{}'", args.addr); 32 | 33 | let listener = TcpListener::bind(args.addr)?; 34 | for stream in listener.incoming() { 35 | if let Ok(stream) = stream { 36 | std::thread::spawn(move || { 37 | handle_connection(stream).map_err(|e| eprintln!("Error: {}", e)) 38 | }); 39 | } 40 | } 41 | Ok(()) 42 | } 43 | -------------------------------------------------------------------------------- /raw/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Shared code between client & server 2 | 3 | use std::io::{self, BufRead}; 4 | 5 | pub const DEFAULT_SERVER_ADDR: &str = "127.0.0.1:4000"; 6 | const MESSAGE_BUFFER_SIZE: usize = 32; 7 | 8 | /// Given a buffer (in this case, TcpStream), write the bytes 9 | /// to be transmitted via TCP 10 | pub fn write_data(stream: &mut impl io::Write, data: &[u8]) -> io::Result<()> { 11 | // Here, `write_all()` attempts to write the entire slice, raising an error if it cannot do so 12 | stream.write_all(data)?; 13 | 14 | // An alternative is `write()` which will return the number of bytes that *could* 15 | // be sent. This can be used if your app has a mechanism to handle this scenario. 16 | // E.g. TCP backpressure for high-bandwidth data 17 | // 18 | // This is an example of what `write_all()` does: 19 | // let bytes_to_write = data.len(); 20 | // let bytes_written = stream.write(data)?; 21 | // if bytes_written < bytes_to_write { 22 | // return Err(Error::new(ErrorKind::Interrupted, "Could not write all data")); 23 | // } 24 | 25 | // Signal that we're done writing and the data should be sent (with TCP PSH bit) 26 | stream.flush() 27 | } 28 | 29 | /// Given a buffer (in this case, TcpStream), attempt to read 30 | /// an unknown stream of bytes and decode to a String 31 | pub fn extract_string_unbuffered(buf: &mut impl io::Read) -> io::Result { 32 | let mut received: Vec = vec![]; 33 | 34 | // Use a statically sized array buffer 35 | // Picking a size is tricky: 36 | // - A large array can waste stack space for bytes we may never need 37 | // - A small array results in more syscalls (for this unbuffered approach) 38 | let mut rx_bytes = [0u8; MESSAGE_BUFFER_SIZE]; 39 | loop { 40 | // Read from the current data in the TcpStream 41 | // !NOTE: Each time this is called it can be a syscall 42 | let bytes_read = buf.read(&mut rx_bytes)?; 43 | 44 | // However many bytes we read, extend the `received` string bytes 45 | received.extend_from_slice(&rx_bytes[..bytes_read]); 46 | 47 | // And if we didn't fill the array,\ 48 | // stop reading because there's no more data (we hope!) 49 | if bytes_read < MESSAGE_BUFFER_SIZE { 50 | break; 51 | } 52 | } 53 | 54 | String::from_utf8(received).map_err(|_| { 55 | io::Error::new( 56 | io::ErrorKind::InvalidData, 57 | "Couldn't parse received string as utf8", 58 | ) 59 | }) 60 | } 61 | 62 | /// Given a buffer (in this case, TcpStream), use `BufReader` and `BufRead` trait 63 | /// to read the pending bytes in the stream 64 | pub fn extract_string_buffered(mut buf: &mut impl io::Read) -> io::Result { 65 | let mut reader = io::BufReader::new(&mut buf); 66 | 67 | // `fill_buf` will return a ref to the bytes pending (received by TCP) 68 | // This is still a lower-level call, so we have to follow it up with a call to consume 69 | let received: Vec = reader.fill_buf()?.to_vec(); 70 | 71 | // Mark the bytes read as consumed so the buffer will not return them in a subsequent read 72 | reader.consume(received.len()); 73 | 74 | String::from_utf8(received).map_err(|_| { 75 | io::Error::new( 76 | io::ErrorKind::InvalidData, 77 | "Couldn't parse received string as utf8", 78 | ) 79 | }) 80 | } 81 | 82 | #[cfg(test)] 83 | mod test { 84 | use super::*; 85 | use std::io::Cursor; 86 | 87 | #[test] 88 | fn test_extract_string_buffered() { 89 | let message = String::from("Hello"); 90 | let mut reader = Cursor::new(message.as_bytes()); 91 | let result = extract_string_buffered(&mut reader).unwrap(); 92 | 93 | assert_eq!(message, result); 94 | } 95 | 96 | #[test] 97 | fn test_extract_string_unbuffered() { 98 | let message = String::from("Hello"); 99 | let mut reader = Cursor::new(message.as_bytes()); 100 | let result = extract_string_unbuffered(&mut reader).unwrap(); 101 | 102 | assert_eq!(message, result); 103 | } 104 | } 105 | --------------------------------------------------------------------------------