├── .github └── workflows │ └── test.yml ├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── benches └── megabyte_parsing.rs ├── examples ├── hyper.rs └── warp.rs └── src ├── client.rs ├── filestream.rs ├── lib.rs └── server.rs /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md 2 | 3 | on: [push, pull_request] 4 | 5 | name: Test 6 | 7 | jobs: 8 | test: 9 | name: Test 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout sources 13 | uses: actions/checkout@v2 14 | 15 | - name: Install stable toolchain 16 | uses: actions-rs/toolchain@v1 17 | with: 18 | profile: minimal 19 | toolchain: stable 20 | override: true 21 | 22 | - name: Rustfmt 23 | run: cargo fmt --all -- --check 24 | 25 | - name: Clippy 26 | uses: actions-rs/cargo@v1 27 | with: 28 | command: clippy 29 | 30 | - name: Test 31 | uses: actions-rs/cargo@v1 32 | with: 33 | command: test 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | **/*.rs.bk 3 | /rls 4 | Cargo.lock -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mpart-async" 3 | version = "0.7.0" 4 | authors = ["cetra3 "] 5 | license = "MIT/Apache-2.0" 6 | description = "Asynchronous (Futures-Base) Multipart Requests for Rust" 7 | repository = "https://github.com/cetra3/mpart-async" 8 | readme = "README.md" 9 | edition = "2021" 10 | 11 | [dependencies] 12 | bytes = "1.0" 13 | futures-core = "0.3" 14 | futures-util = { version = "0.3", default-features = false } 15 | rand = "0.8" 16 | log = "0.4.1" 17 | httparse = "1.6.0" 18 | http = "1.0.0" 19 | memchr = "2.4.1" 20 | tokio = { version = "1.0", optional = true, features= ["fs"] } 21 | tokio-util = { version = "0.7.8", optional = true, features= ["codec"] } 22 | mime_guess = {version = "2.0.1", optional = true} 23 | thiserror = "1.0" 24 | pin-project-lite = "0.2" 25 | percent-encoding = "2.1.0" 26 | 27 | [dev-dependencies] 28 | hyper = "0.14" 29 | warp = "0.3" 30 | mime = "0.3.16" 31 | headers = "0.3.2" 32 | criterion = "0.5.1" 33 | rand = "0.8" 34 | tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] } 35 | tokio-stream = { version = "0.1" } 36 | 37 | [features] 38 | default = [ 39 | "filestream" 40 | ] 41 | filestream = [ 42 | "tokio", 43 | "tokio-util", 44 | "mime_guess" 45 | ] 46 | 47 | [[bench]] 48 | name = "megabyte_parsing" 49 | harness = false 50 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rust Multipart Async 2 | 3 | This crate allows the creation of client/server multipart streams for use with std futures. 4 | 5 | ## Quick Usage 6 | 7 | With clients, you want to create a `MultipartRequest` & add in your fields & files. 8 | 9 | ### Hyper Client Example 10 | 11 | Here is an [example](examples/hyper.rs) of how to use the client with hyper (`cargo run --example hyper`): 12 | 13 | ```rust 14 | use hyper::{header::CONTENT_TYPE, Body, Client, Request}; 15 | use hyper::{service::make_service_fn, service::service_fn, Response, Server}; 16 | use mpart_async::client::MultipartRequest; 17 | 18 | type Error = Box; 19 | 20 | #[tokio::main] 21 | async fn main() -> Result<(), Error> { 22 | //Setup a mock server to accept connections. 23 | setup_server(); 24 | 25 | let client = Client::new(); 26 | 27 | let mut mpart = MultipartRequest::default(); 28 | 29 | mpart.add_field("foo", "bar"); 30 | mpart.add_file("test", "Cargo.toml"); 31 | 32 | let request = Request::post("http://localhost:3000") 33 | .header( 34 | CONTENT_TYPE, 35 | format!("multipart/form-data; boundary={}", mpart.get_boundary()), 36 | ) 37 | .body(Body::wrap_stream(mpart))?; 38 | 39 | client.request(request).await?; 40 | 41 | Ok(()) 42 | } 43 | 44 | fn setup_server() { 45 | let addr = ([127, 0, 0, 1], 3000).into(); 46 | let make_svc = make_service_fn(|_conn| async { Ok::<_, Error>(service_fn(mock)) }); 47 | let server = Server::bind(&addr).serve(make_svc); 48 | 49 | tokio::spawn(server); 50 | } 51 | 52 | async fn mock(_: Request) -> Result, Error> { 53 | Ok(Response::new(Body::from(""))) 54 | } 55 | ``` 56 | 57 | ### Warp Server Example 58 | 59 | Here is an [example](examples/warp.rs) of using it with the warp server (`cargo run --example warp`): 60 | 61 | ```rust 62 | use warp::Filter; 63 | 64 | use bytes::Buf; 65 | use futures::stream::TryStreamExt; 66 | use futures::Stream; 67 | use mime::Mime; 68 | use mpart_async::server::MultipartStream; 69 | use std::convert::Infallible; 70 | 71 | #[tokio::main] 72 | async fn main() { 73 | // Match any request and return hello world! 74 | let routes = warp::any() 75 | .and(warp::header::("content-type")) 76 | .and(warp::body::stream()) 77 | .and_then(mpart); 78 | 79 | warp::serve(routes).run(([127, 0, 0, 1], 3030)).await; 80 | } 81 | 82 | async fn mpart( 83 | mime: Mime, 84 | body: impl Stream> + Unpin, 85 | ) -> Result { 86 | let boundary = mime.get_param("boundary").map(|v| v.to_string()).unwrap(); 87 | 88 | let mut stream = MultipartStream::new( 89 | boundary, 90 | body.map_ok(|mut buf| buf.copy_to_bytes(buf.remaining())), 91 | ); 92 | 93 | while let Ok(Some(mut field)) = stream.try_next().await { 94 | println!("Field received:{}", field.name().unwrap()); 95 | if let Ok(filename) = field.filename() { 96 | println!("Field filename:{}", filename); 97 | } 98 | 99 | while let Ok(Some(bytes)) = field.try_next().await { 100 | println!("Bytes received:{}", bytes.len()); 101 | } 102 | } 103 | 104 | Ok(format!("Thanks!\n")) 105 | } 106 | ``` 107 | 108 | To interact with the server: 109 | ```bash 110 | curl -F test=field -F file=@file.txt http://localhost:3030/ 111 | ``` 112 | -------------------------------------------------------------------------------- /benches/megabyte_parsing.rs: -------------------------------------------------------------------------------- 1 | use bytes::{Bytes, BytesMut}; 2 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 3 | use futures_core::Stream; 4 | use futures_util::StreamExt; 5 | use mpart_async::client::{ByteStream, MultipartRequest}; 6 | use mpart_async::server::MultipartStream; 7 | use rand::{thread_rng, RngCore}; 8 | use std::pin::Pin; 9 | use std::task::{Context, Poll}; 10 | use std::time::Duration; 11 | 12 | fn ten_megabytes_zero_byte(c: &mut Criterion) { 13 | let total_size = 1024 * 1024 * 10; 14 | 15 | let (bytes, boundary) = bytes_and_boundary(&[b'\0'; 1024 * 1024 * 10]); 16 | 17 | let mut group = c.benchmark_group("ten megabytes zero byte"); 18 | 19 | // with bigger chunk sizes this does get a lot of faster, but it would perhaps be better to 20 | // have multiple smaller files 21 | for chunk_size in &[512, 1024, 2048] { 22 | group.throughput(criterion::Throughput::Bytes(total_size as u64)); 23 | 24 | group.bench_with_input( 25 | criterion::BenchmarkId::from_parameter(chunk_size), 26 | &total_size, 27 | |b, &size| { 28 | b.iter(|| { 29 | let bytes = black_box(count_single_field_bytes( 30 | boundary.clone(), 31 | bytes.clone(), 32 | size, 33 | )); 34 | 35 | assert_eq!(bytes, total_size); 36 | }); 37 | }, 38 | ); 39 | } 40 | } 41 | 42 | fn ten_megabytes_r_byte(c: &mut Criterion) { 43 | let total_size = 1024 * 1024 * 10; 44 | 45 | let (bytes, boundary) = bytes_and_boundary(&[b'\r'; 1024 * 1024 * 10]); 46 | 47 | let mut group = c.benchmark_group("ten megabytes \\r byte"); 48 | 49 | // with bigger chunk sizes this does get a lot of faster, but it would perhaps be better to 50 | // have multiple smaller files 51 | for chunk_size in &[512, 1024, 2048] { 52 | group.throughput(criterion::Throughput::Bytes(total_size as u64)); 53 | 54 | group.bench_with_input( 55 | criterion::BenchmarkId::from_parameter(chunk_size), 56 | &total_size, 57 | |b, &size| { 58 | b.iter(|| { 59 | let bytes = black_box(count_single_field_bytes( 60 | boundary.clone(), 61 | bytes.clone(), 62 | size, 63 | )); 64 | 65 | assert_eq!(bytes, total_size); 66 | }); 67 | }, 68 | ); 69 | } 70 | } 71 | 72 | fn ten_megabytes_random(c: &mut Criterion) { 73 | let total_size = 1024 * 1024 * 10; 74 | 75 | let mut bytes_mut = BytesMut::with_capacity(total_size); 76 | 77 | bytes_mut.extend_from_slice(&[b'\0'; 1024 * 1024 * 10]); 78 | 79 | thread_rng().fill_bytes(&mut bytes_mut); 80 | 81 | let (bytes, boundary) = bytes_and_boundary(&bytes_mut); 82 | 83 | let mut group = c.benchmark_group("ten megabytes random"); 84 | 85 | // with bigger chunk sizes this does get a lot of faster, but it would perhaps be better to 86 | // have multiple smaller files 87 | for chunk_size in &[512, 1024, 2048] { 88 | group.throughput(criterion::Throughput::Bytes(total_size as u64)); 89 | 90 | group.bench_with_input( 91 | criterion::BenchmarkId::from_parameter(chunk_size), 92 | &total_size, 93 | |b, &size| { 94 | b.iter(|| { 95 | let bytes = black_box(count_single_field_bytes( 96 | boundary.clone(), 97 | bytes.clone(), 98 | size, 99 | )); 100 | 101 | assert_eq!(bytes, total_size); 102 | }); 103 | }, 104 | ); 105 | } 106 | } 107 | 108 | fn bytes_and_boundary(input: &[u8]) -> (Bytes, Bytes) { 109 | let rt = tokio::runtime::Runtime::new().unwrap(); 110 | let mut request = MultipartRequest::default(); 111 | 112 | let byte_stream = ByteStream::new(input); 113 | 114 | request.add_stream("name", "filename", "content_stream", byte_stream); 115 | 116 | let boundary = Bytes::from(request.get_boundary().to_string()); 117 | 118 | let bytes = rt 119 | .block_on(request.fold(BytesMut::new(), |mut buf, result| async { 120 | if let Ok(bytes) = result { 121 | buf.extend_from_slice(&bytes); 122 | }; 123 | 124 | buf 125 | })) 126 | .freeze(); 127 | 128 | (bytes, boundary) 129 | } 130 | 131 | fn count_single_field_bytes(boundary: Bytes, bytes: Bytes, chunk_size: usize) -> usize { 132 | let rt = tokio::runtime::Runtime::new().unwrap(); 133 | let stream = ChunkedStream(bytes, chunk_size); 134 | let mut stream = MultipartStream::new(boundary, stream); 135 | 136 | let mut field = rt.block_on(stream.next()).unwrap().unwrap(); 137 | let mut bytes = 0; 138 | loop { 139 | match rt.block_on(field.next()) { 140 | Some(Ok(read)) => bytes += read.len(), 141 | Some(Err(e)) => panic!("failed: {}", e), 142 | None => { 143 | break; 144 | } 145 | } 146 | } 147 | 148 | assert!(matches!(rt.block_on(stream.next()), None)); 149 | bytes 150 | } 151 | 152 | struct ChunkedStream(Bytes, usize); 153 | 154 | impl Stream for ChunkedStream { 155 | type Item = Result; 156 | 157 | fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 158 | let amt = self.1.min(self.0.len()); 159 | 160 | if amt > 0 { 161 | let bytes = self.as_mut().0.split_to(amt); 162 | Poll::Ready(Some(Ok(bytes))) 163 | } else { 164 | Poll::Ready(None) 165 | } 166 | } 167 | } 168 | 169 | criterion_group!( 170 | name = benches; 171 | config = Criterion::default().measurement_time(Duration::from_secs(10)); 172 | targets = ten_megabytes_zero_byte, ten_megabytes_r_byte, ten_megabytes_random 173 | ); 174 | criterion_main!(benches); 175 | -------------------------------------------------------------------------------- /examples/hyper.rs: -------------------------------------------------------------------------------- 1 | use hyper::{header::CONTENT_TYPE, Body, Client, Request}; 2 | use hyper::{service::make_service_fn, service::service_fn, Response, Server}; 3 | use mpart_async::client::MultipartRequest; 4 | 5 | type Error = Box; 6 | 7 | #[tokio::main] 8 | async fn main() -> Result<(), Error> { 9 | //Setup a mock server to accept connections. 10 | setup_server(); 11 | 12 | let client = Client::new(); 13 | 14 | let mut mpart = MultipartRequest::default(); 15 | 16 | mpart.add_field("foo", "bar"); 17 | mpart.add_file("test", "Cargo.toml"); 18 | 19 | let request = Request::post("http://localhost:3000") 20 | .header( 21 | CONTENT_TYPE, 22 | format!("multipart/form-data; boundary={}", mpart.get_boundary()), 23 | ) 24 | .body(Body::wrap_stream(mpart))?; 25 | 26 | client.request(request).await?; 27 | 28 | Ok(()) 29 | } 30 | 31 | fn setup_server() { 32 | let addr = ([127, 0, 0, 1], 3000).into(); 33 | let make_svc = make_service_fn(|_conn| async { Ok::<_, Error>(service_fn(mock)) }); 34 | let server = Server::bind(&addr).serve(make_svc); 35 | 36 | tokio::spawn(server); 37 | } 38 | 39 | async fn mock(_: Request) -> Result, Error> { 40 | Ok(Response::new(Body::from(""))) 41 | } 42 | -------------------------------------------------------------------------------- /examples/warp.rs: -------------------------------------------------------------------------------- 1 | use warp::Filter; 2 | 3 | use bytes::Buf; 4 | use futures_core::Stream; 5 | use futures_util::TryStreamExt; 6 | use mime::Mime; 7 | use mpart_async::server::MultipartStream; 8 | use std::convert::Infallible; 9 | 10 | #[tokio::main] 11 | async fn main() { 12 | // Match any request and return hello world! 13 | let routes = warp::any() 14 | .and(warp::header::("content-type")) 15 | .and(warp::body::stream()) 16 | .and_then(mpart); 17 | 18 | warp::serve(routes).run(([127, 0, 0, 1], 3030)).await; 19 | } 20 | 21 | async fn mpart( 22 | mime: Mime, 23 | body: impl Stream> + Unpin, 24 | ) -> Result { 25 | let boundary = mime.get_param("boundary").map(|v| v.to_string()).unwrap(); 26 | 27 | let mut stream = MultipartStream::new( 28 | boundary, 29 | body.map_ok(|mut buf| buf.copy_to_bytes(buf.remaining())), 30 | ); 31 | 32 | while let Ok(Some(mut field)) = stream.try_next().await { 33 | println!("Field received:{}", field.name().unwrap()); 34 | if let Ok(filename) = field.filename() { 35 | println!("Field filename:{}", filename); 36 | } 37 | 38 | while let Ok(Some(bytes)) = field.try_next().await { 39 | println!("Bytes received:{}", bytes.len()); 40 | } 41 | } 42 | 43 | Ok("Thanks!\n".to_string()) 44 | } 45 | -------------------------------------------------------------------------------- /src/client.rs: -------------------------------------------------------------------------------- 1 | use bytes::{Bytes, BytesMut}; 2 | use futures_core::Stream; 3 | use log::debug; 4 | use rand::{distributions::Alphanumeric, thread_rng, Rng}; 5 | use std::pin::Pin; 6 | use std::task::{Context, Poll}; 7 | use std::{collections::VecDeque, convert::Infallible}; 8 | 9 | /// The main `MultipartRequest` struct for sending Multipart submissions to servers 10 | pub struct MultipartRequest { 11 | boundary: String, 12 | items: VecDeque>, 13 | state: Option>, 14 | written: usize, 15 | } 16 | 17 | enum State { 18 | WritingField(MultipartField), 19 | WritingStream(MultipartStream), 20 | WritingStreamHeader(MultipartStream), 21 | Finished, 22 | } 23 | 24 | /// The enum for multipart items which is either a field or a stream 25 | pub enum MultipartItems { 26 | /// MultipartField variant 27 | Field(MultipartField), 28 | /// MultipartStream variant 29 | Stream(MultipartStream), 30 | } 31 | 32 | /// A stream which is part of a `MultipartRequest` and used to stream out bytes 33 | pub struct MultipartStream { 34 | name: String, 35 | filename: String, 36 | content_type: String, 37 | stream: S, 38 | } 39 | 40 | /// A MultipartField which is part of a `MultipartRequest` and used to add a standard text field 41 | pub struct MultipartField { 42 | name: String, 43 | value: String, 44 | } 45 | 46 | impl MultipartStream { 47 | /// Construct a new MultipartStream providing name, filename & content_type 48 | pub fn new>(name: I, filename: I, content_type: I, stream: S) -> Self { 49 | MultipartStream { 50 | name: name.into(), 51 | filename: filename.into(), 52 | content_type: content_type.into(), 53 | stream, 54 | } 55 | } 56 | 57 | fn write_header(&self, boundary: &str) -> Bytes { 58 | let mut buf = BytesMut::new(); 59 | 60 | buf.extend_from_slice(b"--"); 61 | buf.extend_from_slice(boundary.as_bytes()); 62 | buf.extend_from_slice(b"\r\n"); 63 | 64 | buf.extend_from_slice(b"Content-Disposition: form-data; name=\""); 65 | buf.extend_from_slice(self.name.as_bytes()); 66 | buf.extend_from_slice(b"\"; filename=\""); 67 | buf.extend_from_slice(self.filename.as_bytes()); 68 | buf.extend_from_slice(b"\"\r\n"); 69 | buf.extend_from_slice(b"Content-Type: "); 70 | buf.extend_from_slice(self.content_type.as_bytes()); 71 | buf.extend_from_slice(b"\r\n"); 72 | 73 | buf.extend_from_slice(b"\r\n"); 74 | 75 | buf.freeze() 76 | } 77 | } 78 | 79 | impl MultipartField { 80 | /// Construct a new MultipartField given a name and value 81 | pub fn new>(name: I, value: I) -> Self { 82 | MultipartField { 83 | name: name.into(), 84 | value: value.into(), 85 | } 86 | } 87 | 88 | fn get_bytes(&self, boundary: &str) -> Bytes { 89 | let mut buf = BytesMut::new(); 90 | 91 | buf.extend_from_slice(b"--"); 92 | buf.extend_from_slice(boundary.as_bytes()); 93 | buf.extend_from_slice(b"\r\n"); 94 | 95 | buf.extend_from_slice(b"Content-Disposition: form-data; name=\""); 96 | buf.extend_from_slice(self.name.as_bytes()); 97 | buf.extend_from_slice(b"\"\r\n"); 98 | 99 | buf.extend_from_slice(b"\r\n"); 100 | 101 | buf.extend_from_slice(self.value.as_bytes()); 102 | 103 | buf.extend_from_slice(b"\r\n"); 104 | 105 | buf.freeze() 106 | } 107 | } 108 | 109 | impl MultipartRequest 110 | where 111 | S: Stream> + Unpin, 112 | { 113 | /// Construct a new MultipartRequest with a given Boundary 114 | /// 115 | /// If you want a boundary generated automatically, then you can use `MultipartRequest::default()` 116 | pub fn new>(boundary: I) -> Self { 117 | let items = VecDeque::new(); 118 | 119 | let state = None; 120 | 121 | MultipartRequest { 122 | boundary: boundary.into(), 123 | items, 124 | state, 125 | written: 0, 126 | } 127 | } 128 | 129 | fn next_item(&mut self) -> State { 130 | match self.items.pop_front() { 131 | Some(MultipartItems::Field(new_field)) => State::WritingField(new_field), 132 | Some(MultipartItems::Stream(new_stream)) => State::WritingStreamHeader(new_stream), 133 | None => State::Finished, 134 | } 135 | } 136 | 137 | /// Add a raw Stream to the Multipart request 138 | /// 139 | /// The Stream should return items of `Result` 140 | pub fn add_stream>( 141 | &mut self, 142 | name: I, 143 | filename: I, 144 | content_type: I, 145 | stream: S, 146 | ) { 147 | let stream = MultipartStream::new(name, filename, content_type, stream); 148 | 149 | if self.state.is_some() { 150 | self.items.push_back(MultipartItems::Stream(stream)); 151 | } else { 152 | self.state = Some(State::WritingStreamHeader(stream)); 153 | } 154 | } 155 | 156 | /// Add a Field to the Multipart request 157 | pub fn add_field>(&mut self, name: I, value: I) { 158 | let field = MultipartField::new(name, value); 159 | 160 | if self.state.is_some() { 161 | self.items.push_back(MultipartItems::Field(field)); 162 | } else { 163 | self.state = Some(State::WritingField(field)); 164 | } 165 | } 166 | 167 | /// Gets the boundary for the MultipartRequest 168 | /// 169 | /// This is useful for supplying the `Content-Type` header 170 | pub fn get_boundary(&self) -> &str { 171 | &self.boundary 172 | } 173 | 174 | fn write_ending(&self) -> Bytes { 175 | let mut buf = BytesMut::new(); 176 | 177 | buf.extend_from_slice(b"--"); 178 | buf.extend_from_slice(self.boundary.as_bytes()); 179 | 180 | buf.extend_from_slice(b"--\r\n"); 181 | 182 | buf.freeze() 183 | } 184 | } 185 | 186 | #[cfg(feature = "filestream")] 187 | use crate::filestream::FileStream; 188 | #[cfg(feature = "filestream")] 189 | use std::path::PathBuf; 190 | 191 | #[cfg(feature = "filestream")] 192 | impl MultipartRequest { 193 | /// Add a FileStream to a MultipartRequest given a path to a file 194 | /// 195 | /// This will guess the Content Type based upon the path (i.e, .jpg will be `image/jpeg`) 196 | pub fn add_file, P: Into>(&mut self, name: I, path: P) { 197 | let buf = path.into(); 198 | 199 | let name = name.into(); 200 | 201 | let filename = buf 202 | .file_name() 203 | .expect("Should be a valid file") 204 | .to_string_lossy() 205 | .to_string(); 206 | let content_type = mime_guess::MimeGuess::from_path(&buf) 207 | .first_or_octet_stream() 208 | .to_string(); 209 | let stream = FileStream::new(buf); 210 | 211 | self.add_stream(name, filename, content_type, stream); 212 | } 213 | } 214 | 215 | impl Default for MultipartRequest 216 | where 217 | S: Stream> + Unpin, 218 | { 219 | fn default() -> Self { 220 | let mut rng = thread_rng(); 221 | 222 | let boundary: String = (&mut rng) 223 | .sample_iter(Alphanumeric) 224 | .take(60) 225 | .map(char::from) 226 | .collect(); 227 | 228 | let items = VecDeque::new(); 229 | 230 | let state = None; 231 | 232 | MultipartRequest { 233 | boundary, 234 | items, 235 | state, 236 | written: 0, 237 | } 238 | } 239 | } 240 | 241 | impl Stream for MultipartRequest 242 | where 243 | S: Stream> + Unpin, 244 | { 245 | type Item = Result; 246 | 247 | fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 248 | debug!("Poll hit"); 249 | 250 | let self_ref = self.get_mut(); 251 | 252 | let mut bytes = None; 253 | 254 | let mut new_state = None; 255 | 256 | let mut waiting = false; 257 | 258 | if let Some(state) = self_ref.state.take() { 259 | match state { 260 | State::WritingStreamHeader(stream) => { 261 | debug!("Writing Stream Header for:{}", &stream.filename); 262 | bytes = Some(stream.write_header(&self_ref.boundary)); 263 | 264 | new_state = Some(State::WritingStream(stream)); 265 | } 266 | State::WritingStream(mut stream) => { 267 | debug!("Writing Stream Body for:{}", &stream.filename); 268 | 269 | match Pin::new(&mut stream.stream).poll_next(cx) { 270 | Poll::Pending => { 271 | waiting = true; 272 | new_state = Some(State::WritingStream(stream)); 273 | } 274 | Poll::Ready(Some(Ok(some_bytes))) => { 275 | bytes = Some(some_bytes); 276 | new_state = Some(State::WritingStream(stream)); 277 | } 278 | Poll::Ready(None) => { 279 | let mut buf = BytesMut::new(); 280 | /* 281 | This is a special case that we want to include \r\n and then the next item 282 | */ 283 | buf.extend_from_slice(b"\r\n"); 284 | 285 | match self_ref.next_item() { 286 | State::WritingStreamHeader(stream) => { 287 | debug!("Writing new Stream Header"); 288 | buf.extend_from_slice(&stream.write_header(&self_ref.boundary)); 289 | new_state = Some(State::WritingStream(stream)); 290 | } 291 | State::Finished => { 292 | debug!("Writing new Stream Finished"); 293 | buf.extend_from_slice(&self_ref.write_ending()); 294 | } 295 | State::WritingField(field) => { 296 | debug!("Writing new Stream Field"); 297 | buf.extend_from_slice(&field.get_bytes(&self_ref.boundary)); 298 | new_state = Some(self_ref.next_item()); 299 | } 300 | _ => (), 301 | } 302 | 303 | bytes = Some(buf.freeze()) 304 | } 305 | an_error @ Poll::Ready(Some(Err(_))) => return an_error, 306 | } 307 | } 308 | State::Finished => { 309 | debug!("Writing Stream Finished"); 310 | bytes = Some(self_ref.write_ending()); 311 | } 312 | State::WritingField(field) => { 313 | debug!("Writing Field: {}", &field.name); 314 | bytes = Some(field.get_bytes(&self_ref.boundary)); 315 | new_state = Some(self_ref.next_item()); 316 | } 317 | } 318 | } 319 | 320 | if let Some(state) = new_state { 321 | self_ref.state = Some(state); 322 | } 323 | 324 | if waiting { 325 | return Poll::Pending; 326 | } 327 | 328 | if let Some(ref bytes) = bytes { 329 | debug!("Bytes: {}", bytes.len()); 330 | self_ref.written += bytes.len(); 331 | } else { 332 | debug!( 333 | "No bytes to write, finished stream, total bytes:{}", 334 | self_ref.written 335 | ); 336 | } 337 | 338 | Poll::Ready(bytes.map(|bytes| Ok(bytes))) 339 | } 340 | } 341 | 342 | /// A Simple In-Memory Stream that can be used to store bytes 343 | #[derive(Clone)] 344 | pub struct ByteStream { 345 | bytes: Option, 346 | } 347 | 348 | impl ByteStream { 349 | /// Create a new ByteStream based upon the byte slice (note: this will copy from the slice) 350 | pub fn new(bytes: &[u8]) -> Self { 351 | let mut buf = BytesMut::new(); 352 | 353 | buf.extend_from_slice(bytes); 354 | 355 | ByteStream { 356 | bytes: Some(buf.freeze()), 357 | } 358 | } 359 | } 360 | 361 | impl Stream for ByteStream { 362 | type Item = Result; 363 | 364 | fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 365 | Poll::Ready(self.as_mut().bytes.take().map(Ok)) 366 | } 367 | } 368 | 369 | #[cfg(test)] 370 | mod tests { 371 | use super::*; 372 | use futures_util::StreamExt; 373 | 374 | #[test] 375 | fn sets_boundary() { 376 | let req: MultipartRequest = MultipartRequest::new("AaB03x"); 377 | assert_eq!(req.get_boundary(), "AaB03x"); 378 | } 379 | 380 | #[test] 381 | fn writes_field_header() { 382 | let field = MultipartField::new("field_name", "field_value"); 383 | 384 | let input: &[u8] = b"--AaB03x\r\n\ 385 | Content-Disposition: form-data; name=\"field_name\"\r\n\ 386 | \r\n\ 387 | field_value\r\n"; 388 | 389 | let bytes = field.get_bytes("AaB03x"); 390 | 391 | assert_eq!(&bytes[..], input); 392 | } 393 | 394 | #[test] 395 | fn writes_stream_header() { 396 | let stream = MultipartStream::new("file", "test.txt", "text/plain", ByteStream::new(b"")); 397 | 398 | let input: &[u8] = b"--AaB03x\r\n\ 399 | Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\r\n\ 400 | Content-Type: text/plain\r\n\ 401 | \r\n"; 402 | 403 | let bytes = stream.write_header("AaB03x"); 404 | 405 | assert_eq!(&bytes[..], input); 406 | } 407 | 408 | #[tokio::test] 409 | async fn writes_fields() { 410 | let mut req: MultipartRequest = MultipartRequest::new("AaB03x"); 411 | 412 | req.add_field("name1", "value1"); 413 | req.add_field("name2", "value2"); 414 | 415 | let input: &[u8] = b"--AaB03x\r\n\ 416 | Content-Disposition: form-data; name=\"name1\"\r\n\ 417 | \r\n\ 418 | value1\r\n\ 419 | --AaB03x\r\n\ 420 | Content-Disposition: form-data; name=\"name2\"\r\n\ 421 | \r\n\ 422 | value2\r\n\ 423 | --AaB03x--\r\n"; 424 | 425 | let output = req 426 | .fold(BytesMut::new(), |mut buf, result| async { 427 | if let Ok(bytes) = result { 428 | buf.extend_from_slice(&bytes); 429 | }; 430 | 431 | buf 432 | }) 433 | .await; 434 | 435 | assert_eq!(&output[..], input); 436 | } 437 | 438 | #[tokio::test] 439 | async fn writes_streams() { 440 | let mut req: MultipartRequest = MultipartRequest::new("AaB03x"); 441 | 442 | let stream = ByteStream::new(b"Lorem Ipsum\n"); 443 | 444 | req.add_stream("file", "test.txt", "text/plain", stream); 445 | 446 | let input: &[u8] = b"--AaB03x\r\n\ 447 | Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\r\n\ 448 | Content-Type: text/plain\r\n\ 449 | \r\n\ 450 | Lorem Ipsum\n\r\n\ 451 | --AaB03x--\r\n"; 452 | 453 | let output = req 454 | .fold(BytesMut::new(), |mut buf, result| async { 455 | if let Ok(bytes) = result { 456 | buf.extend_from_slice(&bytes); 457 | }; 458 | 459 | buf 460 | }) 461 | .await; 462 | 463 | assert_eq!(&output[..], input); 464 | } 465 | 466 | #[tokio::test] 467 | async fn writes_streams_and_fields() { 468 | let mut req: MultipartRequest = MultipartRequest::new("AaB03x"); 469 | 470 | let stream = ByteStream::new(b"Lorem Ipsum\n"); 471 | 472 | req.add_stream("file", "text.txt", "text/plain", stream); 473 | req.add_field("name1", "value1"); 474 | req.add_field("name2", "value2"); 475 | 476 | let input: &[u8] = b"--AaB03x\r\n\ 477 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 478 | Content-Type: text/plain\r\n\ 479 | \r\n\ 480 | Lorem Ipsum\n\r\n\ 481 | --AaB03x\r\n\ 482 | Content-Disposition: form-data; name=\"name1\"\r\n\ 483 | \r\n\ 484 | value1\r\n\ 485 | --AaB03x\r\n\ 486 | Content-Disposition: form-data; name=\"name2\"\r\n\ 487 | \r\n\ 488 | value2\r\n\ 489 | --AaB03x--\r\n"; 490 | 491 | let output = req 492 | .fold(BytesMut::new(), |mut buf, result| async { 493 | if let Ok(bytes) = result { 494 | buf.extend_from_slice(&bytes); 495 | }; 496 | 497 | buf 498 | }) 499 | .await; 500 | 501 | assert_eq!(&output[..], input); 502 | } 503 | } 504 | -------------------------------------------------------------------------------- /src/filestream.rs: -------------------------------------------------------------------------------- 1 | use futures_core::Stream; 2 | use std::path::PathBuf; 3 | use tokio::fs::File; 4 | use tokio_util::codec::{BytesCodec, FramedRead}; 5 | 6 | use std::future::Future; 7 | use std::pin::Pin; 8 | use std::task::{Context, Poll}; 9 | 10 | use std::io::Error; 11 | 12 | use bytes::Bytes; 13 | 14 | /// Convenience wrapper around streaming out files. Requires tokio 15 | /// 16 | /// You can also add this to a `MultipartRequest` using the [`add_file`](../client/struct.MultipartRequest.html#method.add_file) method: 17 | /// ```no_run 18 | /// # use mpart_async::client::MultipartRequest; 19 | /// # fn main() { 20 | /// let mut req = MultipartRequest::default(); 21 | /// req.add_file("file", "/path/to/file"); 22 | /// # } 23 | /// ``` 24 | pub struct FileStream { 25 | inner: Option>, 26 | file: Pin> + Send + Sync>>, 27 | } 28 | 29 | impl FileStream { 30 | /// Create a new FileStream from a file path 31 | pub fn new>(file: P) -> Self { 32 | FileStream { 33 | file: Box::pin(File::open(file.into())), 34 | inner: None, 35 | } 36 | } 37 | } 38 | 39 | impl Stream for FileStream { 40 | type Item = Result; 41 | 42 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 43 | if let Some(ref mut stream) = self.inner { 44 | return Pin::new(stream) 45 | .poll_next(cx) 46 | .map(|val| val.map(|val| val.map(|val| val.freeze()))); 47 | } else if let Poll::Ready(file_result) = self.file.as_mut().poll(cx) { 48 | match file_result { 49 | Ok(file) => { 50 | self.inner = Some(FramedRead::new(file, BytesCodec::new())); 51 | cx.waker().wake_by_ref(); 52 | } 53 | Err(err) => { 54 | return Poll::Ready(Some(Err(err))); 55 | } 56 | } 57 | } 58 | 59 | Poll::Pending 60 | } 61 | } 62 | 63 | #[cfg(test)] 64 | mod tests { 65 | use super::FileStream; 66 | use std::io::Error; 67 | use tokio_stream::StreamExt; 68 | 69 | #[tokio::test] 70 | async fn streams_file() -> Result<(), Error> { 71 | let bytes = FileStream::new("Cargo.toml").next().await.unwrap()?; 72 | 73 | assert_eq!(bytes, &include_bytes!("../Cargo.toml")[..]); 74 | 75 | Ok(()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | //! # Rust Multipart Async 3 | //! 4 | //! This crate allows the creation of client/server `multipart/form-data` streams for use with std futures & async 5 | //! ## Quick Usage 6 | //! 7 | //! With clients, you want to create a [`MultipartRequest`](client/struct.MultipartRequest.html) & add in your fields & files. 8 | //! 9 | //! With server, you want to use a [`MultipartStream`](server/struct.MultipartStream.html) to retrieve a stream of streams. 10 | //! 11 | //! You can also use the lower level [`MultipartParser`](server/struct.MultipartParser.html) to retrieve a stream emitting either Headers or Byte Chunks 12 | //! 13 | //! ### Hyper Client Example 14 | //! 15 | //! Here is an example of how to use the client with hyper: 16 | //! 17 | //! ```no_run 18 | //! use hyper::{header::CONTENT_TYPE, Body, Client, Request}; 19 | //! use hyper::{service::make_service_fn, service::service_fn, Response, Server}; 20 | //! use mpart_async::client::MultipartRequest; 21 | //! 22 | //! type Error = Box; 23 | //! 24 | //! #[tokio::main] 25 | //! async fn main() -> Result<(), Error> { 26 | //! //Setup a mock server to accept connections. 27 | //! setup_server(); 28 | //! 29 | //! let client = Client::new(); 30 | //! 31 | //! let mut mpart = MultipartRequest::default(); 32 | //! 33 | //! mpart.add_field("foo", "bar"); 34 | //! mpart.add_file("test", "Cargo.toml"); 35 | //! 36 | //! let request = Request::post("http://localhost:3000") 37 | //! .header( 38 | //! CONTENT_TYPE, 39 | //! format!("multipart/form-data; boundary={}", mpart.get_boundary()), 40 | //! ) 41 | //! .body(Body::wrap_stream(mpart))?; 42 | //! 43 | //! client.request(request).await?; 44 | //! 45 | //! Ok(()) 46 | //! } 47 | //! 48 | //! fn setup_server() { 49 | //! let addr = ([127, 0, 0, 1], 3000).into(); 50 | //! let make_svc = make_service_fn(|_conn| async { Ok::<_, Error>(service_fn(mock)) }); 51 | //! let server = Server::bind(&addr).serve(make_svc); 52 | //! 53 | //! tokio::spawn(server); 54 | //! } 55 | //! 56 | //! async fn mock(_: Request) -> Result, Error> { 57 | //! Ok(Response::new(Body::from(""))) 58 | //! } 59 | //! ``` 60 | //! 61 | //! ### Warp Server Example 62 | //! 63 | //! Here is an example of using it with the warp server: 64 | //! 65 | //! ```no_run 66 | //! use warp::Filter; 67 | //! 68 | //! use bytes::Buf; 69 | //! use futures_util::TryStreamExt; 70 | //! use futures_core::Stream; 71 | //! use mime::Mime; 72 | //! use mpart_async::server::MultipartStream; 73 | //! use std::convert::Infallible; 74 | //! 75 | //! #[tokio::main] 76 | //! async fn main() { 77 | //! // Match any request and return hello world! 78 | //! let routes = warp::any() 79 | //! .and(warp::header::("content-type")) 80 | //! .and(warp::body::stream()) 81 | //! .and_then(mpart); 82 | //! 83 | //! warp::serve(routes).run(([127, 0, 0, 1], 3030)).await; 84 | //! } 85 | //! 86 | //! async fn mpart( 87 | //! mime: Mime, 88 | //! body: impl Stream> + Unpin, 89 | //! ) -> Result { 90 | //! let boundary = mime.get_param("boundary").map(|v| v.to_string()).unwrap(); 91 | //! 92 | //! let mut stream = MultipartStream::new( 93 | //! boundary, 94 | //! body.map_ok(|mut buf| buf.copy_to_bytes(buf.remaining())), 95 | //! ); 96 | //! 97 | //! while let Ok(Some(mut field)) = stream.try_next().await { 98 | //! println!("Field received:{}", field.name().unwrap()); 99 | //! if let Ok(filename) = field.filename() { 100 | //! println!("Field filename:{}", filename); 101 | //! } 102 | //! 103 | //! while let Ok(Some(bytes)) = field.try_next().await { 104 | //! println!("Bytes received:{}", bytes.len()); 105 | //! } 106 | //! } 107 | //! 108 | //! Ok(format!("Thanks!\n")) 109 | //! } 110 | //! ``` 111 | 112 | /// The Client Module used for sending requests to servers 113 | /// 114 | pub mod client; 115 | /// Server structs for use when parsing requests from clients 116 | pub mod server; 117 | 118 | /// The FileStream is a tokio way of streaming out a file from a given path. 119 | /// 120 | /// This allows you to do `FileStream::new("/path/to/file")` to create a stream of Bytes of the file 121 | #[cfg(feature = "filestream")] 122 | pub mod filestream; 123 | -------------------------------------------------------------------------------- /src/server.rs: -------------------------------------------------------------------------------- 1 | use bytes::{Bytes, BytesMut}; 2 | use futures_core::Stream; 3 | use http::header::{HeaderMap, HeaderName, HeaderValue}; 4 | use httparse::Status; 5 | use log::debug; 6 | use pin_project_lite::pin_project; 7 | use std::borrow::Cow; 8 | use std::error::Error as StdError; 9 | use std::mem; 10 | use std::pin::Pin; 11 | use std::sync::{Arc, Mutex}; 12 | use std::task::{Context, Poll}; 13 | use thiserror::Error; 14 | 15 | use memchr::{memchr, memmem::find, memrchr}; 16 | use percent_encoding::percent_decode_str; 17 | 18 | type AnyStdError = Box; 19 | 20 | /// A single field of a [`MultipartStream`](./struct.MultipartStream.html) which itself is a stream 21 | /// 22 | /// This represents either an uploaded file, or a simple text value 23 | /// 24 | /// Each field will have some headers and then a body. 25 | /// There are no assumptions made when parsing about what headers are present, but some of the helper methods here (such as `content_type()` & `filename()`) will return an error if they aren't present. 26 | /// The body will be returned as an inner stream of bytes from the request, but up to the end of the field. 27 | /// 28 | /// Fields are not concurrent against their parent multipart request. This is because multipart submissions are a single http request and we don't support going backwards or skipping bytes. In other words you can't read from multiple fields from the same request at the same time: you must wait for one field to finish being read before moving on. 29 | 30 | pub struct MultipartField 31 | where 32 | S: Stream> + Unpin, 33 | E: Into, 34 | { 35 | headers: HeaderMap, 36 | state: Arc>>, 37 | } 38 | 39 | impl MultipartField 40 | where 41 | S: Stream> + Unpin, 42 | E: Into, 43 | { 44 | /// Return the headers for the field 45 | /// 46 | /// You can use `self.headers.get("my-header").and_then(|val| val.to_str().ok())` to get out the header if present 47 | pub fn headers(&self) -> &HeaderMap { 48 | &self.headers 49 | } 50 | 51 | /// Return the content type of the field (if present or error) 52 | pub fn content_type(&self) -> Result<&str, MultipartError> { 53 | if let Some(val) = self.headers.get("content-type") { 54 | return val.to_str().map_err(|_| MultipartError::InvalidHeader); 55 | } 56 | 57 | Err(MultipartError::InvalidHeader) 58 | } 59 | 60 | /// Return the filename of the field (if present or error). 61 | /// The returned filename will be utf8 percent-decoded 62 | pub fn filename(&self) -> Result, MultipartError> { 63 | if let Some(val) = self.headers.get("content-disposition") { 64 | let string_val = 65 | std::str::from_utf8(val.as_bytes()).map_err(|_| MultipartError::InvalidHeader)?; 66 | if let Some(filename) = get_dispo_param(string_val, "filename*") { 67 | let stripped = strip_utf8_prefix(filename); 68 | return Ok(stripped); 69 | } 70 | if let Some(filename) = get_dispo_param(string_val, "filename") { 71 | return Ok(filename); 72 | } 73 | } 74 | 75 | Err(MultipartError::InvalidHeader) 76 | } 77 | 78 | /// Return the name of the field (if present or error). 79 | /// The returned name will be utf8 percent-decoded 80 | pub fn name(&self) -> Result, MultipartError> { 81 | if let Some(val) = self.headers.get("content-disposition") { 82 | let string_val = 83 | std::str::from_utf8(val.as_bytes()).map_err(|_| MultipartError::InvalidHeader)?; 84 | if let Some(filename) = get_dispo_param(string_val, "name") { 85 | return Ok(filename); 86 | } 87 | } 88 | 89 | Err(MultipartError::InvalidHeader) 90 | } 91 | } 92 | 93 | fn strip_utf8_prefix(string: Cow) -> Cow { 94 | if string.starts_with("UTF-8''") || string.starts_with("utf-8''") { 95 | let split = string.split_at(7).1; 96 | return Cow::from(split.to_owned()); 97 | } 98 | 99 | string 100 | } 101 | 102 | /// This function will get a disposition param from `content-disposition` header & try to escape it if there are escaped quotes or percent encoding 103 | fn get_dispo_param<'a>(input: &'a str, param: &str) -> Option> { 104 | debug!("dispo param:{input}, field `{param}`"); 105 | if let Some(start_idx) = find(input.as_bytes(), param.as_bytes()) { 106 | debug!("Start idx found:{start_idx}"); 107 | let end_param = start_idx + param.len(); 108 | //check bounds 109 | if input.len() > end_param + 2 && &input[end_param..end_param + 2] == "=\"" { 110 | let start = end_param + 2; 111 | 112 | let mut snippet = &input[start..]; 113 | 114 | // If we encounter a `\"` in the string we need to escape it 115 | // This means that we need to create a new escaped string as it will be discontiguous 116 | let mut escaped_buffer: Option = None; 117 | 118 | while let Some(end) = memchr(b'"', snippet.as_bytes()) { 119 | // if we encounter a backslash before the quote 120 | if end > 0 121 | && snippet 122 | .get(end - 1..end) 123 | .map_or(false, |character| character == "\\") 124 | { 125 | // We get an existing escaped buffer or create an empty string 126 | let mut buffer = escaped_buffer.unwrap_or_default(); 127 | 128 | // push up until the escaped quote 129 | buffer.push_str(&snippet[..end - 1]); 130 | // push in the quote itself 131 | buffer.push('"'); 132 | 133 | escaped_buffer = Some(buffer); 134 | 135 | // Move the buffer ahead 136 | snippet = &snippet[end + 1..]; 137 | continue; 138 | } else { 139 | // we're at the end 140 | 141 | // if we have something escaped 142 | match escaped_buffer { 143 | Some(mut escaped) => { 144 | // tack on the end of the string 145 | escaped.push_str(&snippet[0..end]); 146 | 147 | // Double escape with percent decode 148 | if escaped.contains('%') { 149 | let decoded_val = percent_decode_str(&escaped).decode_utf8_lossy(); 150 | return Some(Cow::Owned(decoded_val.into_owned())); 151 | } 152 | 153 | return Some(Cow::Owned(escaped)); 154 | } 155 | None => { 156 | let value = &snippet[0..end]; 157 | 158 | // Escape with percent decode, if necessary 159 | if value.contains('%') { 160 | let decoded_val = percent_decode_str(value).decode_utf8_lossy(); 161 | 162 | return Some(decoded_val); 163 | } 164 | 165 | return Some(Cow::Borrowed(value)); 166 | } 167 | } 168 | } 169 | } 170 | } 171 | } 172 | 173 | None 174 | } 175 | 176 | //Streams out bytes 177 | impl Stream for MultipartField 178 | where 179 | S: Stream> + Unpin, 180 | E: Into, 181 | { 182 | type Item = Result; 183 | 184 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 185 | let self_mut = &mut self.as_mut(); 186 | 187 | let state = &mut self_mut 188 | .state 189 | .try_lock() 190 | .map_err(|_| MultipartError::InternalBorrowError)?; 191 | 192 | match Pin::new(&mut state.parser).poll_next(cx) { 193 | Poll::Pending => Poll::Pending, 194 | Poll::Ready(Some(Err(err))) => { 195 | Poll::Ready(Some(Err(MultipartError::Stream(err.into())))) 196 | } 197 | Poll::Ready(None) => Poll::Ready(None), 198 | //If we have headers, we have reached the next file 199 | Poll::Ready(Some(Ok(ParseOutput::Headers(headers)))) => { 200 | state.next_item = Some(headers); 201 | Poll::Ready(None) 202 | } 203 | Poll::Ready(Some(Ok(ParseOutput::Bytes(bytes)))) => Poll::Ready(Some(Ok(bytes))), 204 | } 205 | } 206 | } 207 | 208 | //This is our state we use to drive the parser. The `next_item` is there just for headers if there are more in the request 209 | struct MultipartState 210 | where 211 | S: Stream> + Unpin, 212 | E: Into, 213 | { 214 | parser: MultipartParser, 215 | next_item: Option>, 216 | } 217 | 218 | /// The main `MultipartStream` struct which will contain one or more fields (a stream of streams) 219 | /// 220 | /// You can construct this given a boundary and a stream of bytes from a server request. 221 | /// 222 | /// **Please Note**: If you are reading in a field, you must exhaust the field's bytes before moving onto the next field 223 | /// ```no_run 224 | /// # use warp::Filter; 225 | /// # use bytes::{Buf, BufMut, BytesMut}; 226 | /// # use futures_util::TryStreamExt; 227 | /// # use futures_core::Stream; 228 | /// # use mime::Mime; 229 | /// # use mpart_async::server::MultipartStream; 230 | /// # use std::convert::Infallible; 231 | /// # #[tokio::main] 232 | /// # async fn main() { 233 | /// # // Match any request and return hello world! 234 | /// # let routes = warp::any() 235 | /// # .and(warp::header::("content-type")) 236 | /// # .and(warp::body::stream()) 237 | /// # .and_then(mpart); 238 | /// # warp::serve(routes).run(([127, 0, 0, 1], 3030)).await; 239 | /// # } 240 | /// # async fn mpart( 241 | /// # mime: Mime, 242 | /// # body: impl Stream> + Unpin, 243 | /// # ) -> Result { 244 | /// # let boundary = mime.get_param("boundary").map(|v| v.to_string()).unwrap(); 245 | /// let mut stream = MultipartStream::new(boundary, body.map_ok(|mut buf| { 246 | /// let mut ret = BytesMut::with_capacity(buf.remaining()); 247 | /// ret.put(buf); 248 | /// ret.freeze() 249 | /// })); 250 | /// 251 | /// while let Ok(Some(mut field)) = stream.try_next().await { 252 | /// println!("Field received:{}", field.name().unwrap()); 253 | /// if let Ok(filename) = field.filename() { 254 | /// println!("Field filename:{}", filename); 255 | /// } 256 | /// 257 | /// while let Ok(Some(bytes)) = field.try_next().await { 258 | /// println!("Bytes received:{}", bytes.len()); 259 | /// } 260 | /// } 261 | /// # Ok(format!("Thanks!\n")) 262 | /// # } 263 | /// ``` 264 | 265 | pub struct MultipartStream 266 | where 267 | S: Stream> + Unpin, 268 | E: Into, 269 | { 270 | state: Arc>>, 271 | } 272 | 273 | impl MultipartStream 274 | where 275 | S: Stream> + Unpin, 276 | E: Into, 277 | { 278 | /// Construct a MultipartStream given a boundary 279 | pub fn new>(boundary: I, stream: S) -> Self { 280 | Self { 281 | state: Arc::new(Mutex::new(MultipartState { 282 | parser: MultipartParser::new(boundary, stream), 283 | next_item: None, 284 | })), 285 | } 286 | } 287 | } 288 | 289 | impl Stream for MultipartStream 290 | where 291 | S: Stream> + Unpin, 292 | E: Into, 293 | { 294 | type Item = Result, MultipartError>; 295 | 296 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 297 | let self_mut = &mut self.as_mut(); 298 | 299 | let state = &mut self_mut 300 | .state 301 | .try_lock() 302 | .map_err(|_| MultipartError::InternalBorrowError)?; 303 | 304 | if let Some(headers) = state.next_item.take() { 305 | return Poll::Ready(Some(Ok(MultipartField { 306 | headers, 307 | state: self_mut.state.clone(), 308 | }))); 309 | } 310 | 311 | match Pin::new(&mut state.parser).poll_next(cx) { 312 | Poll::Pending => Poll::Pending, 313 | Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), 314 | Poll::Ready(None) => Poll::Ready(None), 315 | 316 | //If we have headers, we have reached the next file 317 | Poll::Ready(Some(Ok(ParseOutput::Headers(headers)))) => { 318 | Poll::Ready(Some(Ok(MultipartField { 319 | headers, 320 | state: self_mut.state.clone(), 321 | }))) 322 | } 323 | Poll::Ready(Some(Ok(ParseOutput::Bytes(_bytes)))) => { 324 | //If we are returning bytes from this stream, then there is some error 325 | Poll::Ready(Some(Err(MultipartError::ShouldPollField))) 326 | } 327 | } 328 | } 329 | } 330 | 331 | #[derive(Error, Debug)] 332 | /// The Standard Error Type 333 | pub enum MultipartError { 334 | /// Given if the boundary is not what is expected 335 | #[error("Invalid Boundary. (expected {expected:?}, found {found:?})")] 336 | InvalidBoundary { 337 | /// The Expected Boundary 338 | expected: String, 339 | /// The Found Boundary 340 | found: String, 341 | }, 342 | /// Given if when parsing the headers they are incomplete 343 | #[error("Incomplete Headers")] 344 | IncompleteHeader, 345 | /// Given if when trying to retrieve a field like name or filename it's not present or malformed 346 | #[error("Invalid Header Value")] 347 | InvalidHeader, 348 | /// Given if in the middle of polling a Field, and someone tries to poll the Stream 349 | #[error( 350 | "Tried to poll an MultipartStream when the MultipartField should be polled, try using `flatten()`" 351 | )] 352 | ShouldPollField, 353 | /// Given if in the middle of polling a Field, but the Mutex is in use somewhere else 354 | #[error("Tried to poll an MultipartField and the Mutex has already been locked")] 355 | InternalBorrowError, 356 | /// Given if there is an error when parsing headers 357 | #[error(transparent)] 358 | HeaderParse(#[from] httparse::Error), 359 | /// Given if there is an error in the underlying stream 360 | #[error(transparent)] 361 | Stream(#[from] AnyStdError), 362 | /// Given if the stream ends when reading headers 363 | #[error("EOF while reading headers")] 364 | EOFWhileReadingHeaders, 365 | /// Given if the stream ends when reading boundary 366 | #[error("EOF while reading boundary")] 367 | EOFWhileReadingBoundary, 368 | /// Given if if the stream ends when reading the body and there is no end boundary 369 | #[error("EOF while reading body")] 370 | EOFWhileReadingBody, 371 | /// Given if there is garbage after the boundary 372 | #[error("Garbage following boundary: {0:02x?}")] 373 | GarbageAfterBoundary([u8; 2]), 374 | } 375 | 376 | pin_project! { 377 | /// A low-level parser which `MultipartStream` uses. 378 | /// 379 | /// Returns either headers of a field or a byte chunk, alternating between the two types 380 | /// 381 | /// Unless there is an issue with using [`MultipartStream`](./struct.MultipartStream.html) you don't normally want to use this struct 382 | #[project = ParserProj] 383 | pub struct MultipartParser 384 | where 385 | S: Stream>, 386 | E: Into, 387 | { 388 | boundary: Bytes, 389 | buffer: BytesMut, 390 | state: State, 391 | #[pin] 392 | stream: S, 393 | } 394 | } 395 | 396 | impl MultipartParser 397 | where 398 | S: Stream>, 399 | E: Into, 400 | { 401 | /// Construct a raw parser from a boundary/stream. 402 | pub fn new>(boundary: I, stream: S) -> Self { 403 | Self { 404 | boundary: boundary.into(), 405 | buffer: BytesMut::new(), 406 | state: State::ReadingBoundary, 407 | stream, 408 | } 409 | } 410 | } 411 | 412 | const NUM_HEADERS: usize = 16; 413 | 414 | fn get_headers(buffer: &[u8]) -> Result, MultipartError> { 415 | let mut headers = [httparse::EMPTY_HEADER; NUM_HEADERS]; 416 | 417 | let idx = match httparse::parse_headers(buffer, &mut headers)? { 418 | Status::Complete((idx, _val)) => idx, 419 | Status::Partial => return Err(MultipartError::IncompleteHeader), 420 | }; 421 | 422 | let mut header_map = HeaderMap::with_capacity(idx); 423 | 424 | for header in headers.iter().take(idx) { 425 | if !header.name.is_empty() { 426 | header_map.insert( 427 | HeaderName::from_bytes(header.name.as_bytes()) 428 | .map_err(|_| MultipartError::InvalidHeader)?, 429 | HeaderValue::from_bytes(header.value).map_err(|_| MultipartError::InvalidHeader)?, 430 | ); 431 | } 432 | } 433 | 434 | Ok(header_map) 435 | } 436 | 437 | impl Stream for MultipartParser 438 | where 439 | S: Stream>, 440 | E: Into, 441 | { 442 | type Item = Result; 443 | 444 | fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 445 | let ParserProj { 446 | boundary, 447 | buffer, 448 | state, 449 | mut stream, 450 | } = self.project(); 451 | 452 | loop { 453 | match state { 454 | State::ReadingBoundary => { 455 | let boundary_len = boundary.len(); 456 | 457 | //If the buffer is too small 458 | if buffer.len() < boundary_len + 4 { 459 | match futures_core::ready!(stream.as_mut().poll_next(cx)) { 460 | Some(Ok(bytes)) => { 461 | buffer.extend_from_slice(&bytes); 462 | continue; 463 | } 464 | Some(Err(e)) => { 465 | return Poll::Ready(Some(Err(MultipartError::Stream(e.into())))) 466 | } 467 | None => { 468 | return Poll::Ready(Some(Err( 469 | MultipartError::EOFWhileReadingBoundary, 470 | ))) 471 | } 472 | } 473 | } 474 | 475 | //If the buffer starts with `--\r\n` 476 | if &buffer[..2] == b"--" 477 | && buffer[2..boundary_len + 2] == *boundary 478 | && &buffer[boundary_len + 2..boundary_len + 4] == b"\r\n" 479 | { 480 | //remove the boundary from the buffer, returning the tail 481 | *buffer = buffer.split_off(boundary_len + 4); 482 | *state = State::ReadingHeader; 483 | 484 | //Update the boundary here to include the \r\n-- preamble for individual fields 485 | let mut new_boundary = BytesMut::with_capacity(boundary_len + 4); 486 | 487 | new_boundary.extend_from_slice(b"\r\n--"); 488 | new_boundary.extend_from_slice(boundary); 489 | 490 | *boundary = new_boundary.freeze(); 491 | 492 | cx.waker().wake_by_ref(); 493 | return Poll::Pending; 494 | } else { 495 | let expected = format!("--{}\\r\\n", String::from_utf8_lossy(boundary)); 496 | let found = 497 | String::from_utf8_lossy(&buffer[..boundary_len + 4]).to_string(); 498 | 499 | let error = MultipartError::InvalidBoundary { expected, found }; 500 | 501 | //There is some error with the boundary... 502 | return Poll::Ready(Some(Err(error))); 503 | } 504 | } 505 | State::ReadingHeader => { 506 | if let Some(end) = find(buffer, b"\r\n\r\n") { 507 | //Need to include the end header bytes for the parse to work 508 | let end = end + 4; 509 | 510 | let header_map = match get_headers(&buffer[0..end]) { 511 | Ok(headers) => headers, 512 | Err(error) => { 513 | *state = State::Finished; 514 | return Poll::Ready(Some(Err(error))); 515 | } 516 | }; 517 | 518 | *buffer = buffer.split_off(end); 519 | *state = State::StreamingContent(buffer.is_empty()); 520 | 521 | cx.waker().wake_by_ref(); 522 | 523 | return Poll::Ready(Some(Ok(ParseOutput::Headers(header_map)))); 524 | } else { 525 | match futures_core::ready!(stream.as_mut().poll_next(cx)) { 526 | Some(Ok(bytes)) => { 527 | buffer.extend_from_slice(&bytes); 528 | continue; 529 | } 530 | Some(Err(e)) => { 531 | return Poll::Ready(Some(Err(MultipartError::Stream(e.into())))) 532 | } 533 | None => { 534 | return Poll::Ready(Some(Err( 535 | MultipartError::EOFWhileReadingHeaders, 536 | ))) 537 | } 538 | } 539 | } 540 | } 541 | 542 | State::StreamingContent(exhausted) => { 543 | let boundary_len = boundary.len(); 544 | 545 | if buffer.is_empty() || *exhausted { 546 | *state = State::StreamingContent(false); 547 | match futures_core::ready!(stream.as_mut().poll_next(cx)) { 548 | Some(Ok(bytes)) => { 549 | buffer.extend_from_slice(&bytes); 550 | continue; 551 | } 552 | Some(Err(e)) => { 553 | return Poll::Ready(Some(Err(MultipartError::Stream(e.into())))) 554 | } 555 | None => { 556 | return Poll::Ready(Some(Err(MultipartError::EOFWhileReadingBody))) 557 | } 558 | } 559 | } 560 | 561 | //We want to check the value of the buffer to see if there looks like there is an `end` boundary. 562 | if let Some(idx) = find(buffer, boundary) { 563 | //Check the length has enough bytes for us 564 | if buffer.len() < idx + 2 + boundary_len { 565 | // FIXME: cannot stop the read successfully here! 566 | *state = State::StreamingContent(true); 567 | continue; 568 | } 569 | 570 | //The start of the boundary is 4 chars. i.e, after `\r\n--` 571 | let end_boundary = idx + boundary_len; 572 | 573 | //We want the chars after the boundary basically 574 | let after_boundary = &buffer[end_boundary..end_boundary + 2]; 575 | 576 | if after_boundary == b"\r\n" { 577 | let mut other_bytes = (*buffer).split_off(idx); 578 | 579 | //Remove the boundary-related bytes from the start of the buffer 580 | other_bytes = other_bytes.split_off(2 + boundary_len); 581 | 582 | //Return the bytes up to the boundary, we're finished and need to go back to reading headers 583 | let return_bytes = Bytes::from(mem::replace(buffer, other_bytes)); 584 | 585 | //Replace the buffer with the extra bytes 586 | *state = State::ReadingHeader; 587 | cx.waker().wake_by_ref(); 588 | 589 | return Poll::Ready(Some(Ok(ParseOutput::Bytes(return_bytes)))); 590 | } else if after_boundary == b"--" { 591 | //We're at the end, just truncate the bytes 592 | buffer.truncate(idx); 593 | *state = State::Finished; 594 | 595 | return Poll::Ready(Some(Ok(ParseOutput::Bytes(Bytes::from( 596 | mem::take(buffer), 597 | ))))); 598 | } else { 599 | return Poll::Ready(Some(Err(MultipartError::GarbageAfterBoundary([ 600 | after_boundary[0], 601 | after_boundary[1], 602 | ])))); 603 | } 604 | } else { 605 | //We need to check for partial matches by checking the last boundary_len bytes 606 | let buffer_len = buffer.len(); 607 | 608 | //Clamp to zero if the boundary length is bigger than the buffer 609 | let start_idx = 610 | (buffer_len as i64 - (boundary_len as i64 - 1)).max(0) as usize; 611 | 612 | let end_of_buffer = &buffer[start_idx..]; 613 | 614 | if let Some(idx) = memrchr(b'\r', end_of_buffer) { 615 | //If to the end of the match equals the same amount of bytes 616 | if end_of_buffer[idx..] == boundary[..(end_of_buffer.len() - idx)] { 617 | *state = State::StreamingContent(true); 618 | 619 | //we return all the bytes except for the start of our boundary 620 | let mut output = buffer.split_off(idx + start_idx); 621 | mem::swap(&mut output, buffer); 622 | 623 | cx.waker().wake_by_ref(); 624 | return Poll::Ready(Some(Ok(ParseOutput::Bytes(output.freeze())))); 625 | } 626 | } 627 | 628 | let output = mem::take(buffer); 629 | return Poll::Ready(Some(Ok(ParseOutput::Bytes(output.freeze())))); 630 | } 631 | } 632 | State::Finished => return Poll::Ready(None), 633 | } 634 | } 635 | } 636 | } 637 | 638 | #[derive(Debug, PartialEq)] 639 | enum State { 640 | ReadingBoundary, 641 | ReadingHeader, 642 | StreamingContent(bool), 643 | Finished, 644 | } 645 | 646 | #[derive(Debug)] 647 | /// The output from a MultipartParser 648 | pub enum ParseOutput { 649 | /// Headers received in the output 650 | Headers(HeaderMap), 651 | /// Bytes received in the output 652 | Bytes(Bytes), 653 | } 654 | 655 | #[cfg(test)] 656 | mod tests { 657 | use super::*; 658 | use crate::client::ByteStream; 659 | use futures_util::StreamExt; 660 | 661 | #[tokio::test] 662 | async fn read_stream() { 663 | let input: &[u8] = b"--AaB03x\r\n\ 664 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 665 | Content-Type: text/plain\r\n\ 666 | \r\n\ 667 | Lorem Ipsum\n\r\n\ 668 | --AaB03x\r\n\ 669 | Content-Disposition: form-data; name=\"name1\"\r\n\ 670 | \r\n\ 671 | value1\r\n\ 672 | --AaB03x\r\n\ 673 | Content-Disposition: form-data; name=\"name2\"\r\n\ 674 | \r\n\ 675 | value2\r\n\ 676 | --AaB03x--\r\n"; 677 | 678 | let mut stream = MultipartStream::new("AaB03x", ByteStream::new(input)); 679 | 680 | if let Some(Ok(mut mpart_field)) = stream.next().await { 681 | assert_eq!(mpart_field.name().ok(), Some(Cow::Borrowed("file"))); 682 | assert_eq!(mpart_field.filename().ok(), Some(Cow::Borrowed("text.txt"))); 683 | 684 | if let Some(Ok(bytes)) = mpart_field.next().await { 685 | assert_eq!(bytes, Bytes::from(b"Lorem Ipsum\n" as &[u8])); 686 | } 687 | } else { 688 | panic!("First value should be a field") 689 | } 690 | } 691 | 692 | #[tokio::test] 693 | async fn read_utf_8_filename() { 694 | let input: &[u8] = b"--AaB03x\r\n\ 695 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"; filename*=\"aous%20.txt\"\r\n\ 696 | Content-Type: text/plain\r\n\ 697 | \r\n\ 698 | Lorem Ipsum\n\r\n\ 699 | --AaB03x--\r\n"; 700 | 701 | let mut stream = MultipartStream::new("AaB03x", ByteStream::new(input)); 702 | 703 | let field = stream.next().await.unwrap().unwrap(); 704 | assert_eq!(field.filename().ok(), Some(Cow::Borrowed("aous .txt"))); 705 | } 706 | 707 | #[test] 708 | fn read_filename() { 709 | let input = "form-data; name=\"file\";\ 710 | filename=\"text%20.txt\";\ 711 | quoted=\"with a \\\" quote and another \\\" quote\";\ 712 | empty=\"\"\ 713 | percent_encoded=\"foo%20%3Cbar%3E\"\ 714 | "; 715 | let name = get_dispo_param(input, "name"); 716 | let filename = get_dispo_param(input, "filename"); 717 | let with_a_quote = get_dispo_param(input, "quoted"); 718 | let empty = get_dispo_param(input, "empty"); 719 | let percent_encoded = get_dispo_param(input, "percent_encoded"); 720 | 721 | assert_eq!(name, Some(Cow::Borrowed("file"))); 722 | assert_eq!(filename, Some(Cow::Borrowed("text .txt"))); 723 | assert_eq!( 724 | with_a_quote, 725 | Some(Cow::Owned("with a \" quote and another \" quote".into())) 726 | ); 727 | assert_eq!(empty, Some(Cow::Borrowed(""))); 728 | assert_eq!(percent_encoded, Some(Cow::Borrowed("foo "))); 729 | } 730 | 731 | #[test] 732 | fn read_filename_umlaut() { 733 | let input = "form-data; name=\"äöüß\";\ 734 | filename*=\"äöü ß%20.txt\";\ 735 | quoted=\"with a \\\" quote and another \\\" quote\";\ 736 | empty=\"\"\ 737 | percent_encoded=\"foo%20%3Cbar%3E\"\ 738 | "; 739 | let name = get_dispo_param(input, "name"); 740 | let filename = get_dispo_param(input, "filename*"); 741 | let with_a_quote = get_dispo_param(input, "quoted"); 742 | let empty = get_dispo_param(input, "empty"); 743 | let percent_encoded = get_dispo_param(input, "percent_encoded"); 744 | 745 | assert_eq!(name, Some(Cow::Borrowed("äöüß"))); 746 | assert_eq!(filename, Some(Cow::Borrowed("äöü ß .txt"))); 747 | assert_eq!( 748 | with_a_quote, 749 | Some(Cow::Owned("with a \" quote and another \" quote".into())) 750 | ); 751 | assert_eq!(empty, Some(Cow::Borrowed(""))); 752 | assert_eq!(percent_encoded, Some(Cow::Borrowed("foo "))); 753 | } 754 | 755 | #[tokio::test] 756 | async fn reads_streams_and_fields() { 757 | let input: &[u8] = b"--AaB03x\r\n\ 758 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 759 | Content-Type: text/plain\r\n\ 760 | \r\n\ 761 | Lorem Ipsum\n\r\n\ 762 | --AaB03x\r\n\ 763 | Content-Disposition: form-data; name=\"name1\"\r\n\ 764 | \r\n\ 765 | value1\r\n\ 766 | --AaB03x\r\n\ 767 | Content-Disposition: form-data; name=\"name2\"\r\n\ 768 | \r\n\ 769 | value2\r\n\ 770 | --AaB03x--\r\n"; 771 | 772 | let mut read = MultipartParser::new("AaB03x", ByteStream::new(input)); 773 | 774 | if let Some(Ok(ParseOutput::Headers(val))) = read.next().await { 775 | println!("Headers:{:?}", val); 776 | } else { 777 | panic!("First value should be a header") 778 | } 779 | 780 | if let Some(Ok(ParseOutput::Bytes(bytes))) = read.next().await { 781 | assert_eq!(&*bytes, b"Lorem Ipsum\n"); 782 | } else { 783 | panic!("Second value should be bytes") 784 | } 785 | 786 | if let Some(Ok(ParseOutput::Headers(val))) = read.next().await { 787 | println!("Headers:{:?}", val); 788 | } else { 789 | panic!("Third value should be a header") 790 | } 791 | 792 | if let Some(Ok(ParseOutput::Bytes(bytes))) = read.next().await { 793 | assert_eq!(&*bytes, b"value1"); 794 | } else { 795 | panic!("Fourth value should be bytes") 796 | } 797 | 798 | if let Some(Ok(ParseOutput::Headers(val))) = read.next().await { 799 | println!("Headers:{:?}", val); 800 | } else { 801 | panic!("Fifth value should be a header") 802 | } 803 | 804 | if let Some(Ok(ParseOutput::Bytes(bytes))) = read.next().await { 805 | assert_eq!(&*bytes, b"value2"); 806 | } else { 807 | panic!("Sixth value should be bytes") 808 | } 809 | 810 | assert!(read.next().await.is_none()); 811 | } 812 | 813 | #[tokio::test] 814 | async fn unfinished_header() { 815 | let input: &[u8] = b"--AaB03x\r\n\ 816 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 817 | Content-Type: text/plain"; 818 | let mut read = MultipartParser::new("AaB03x", ByteStream::new(input)); 819 | 820 | let ret = read.next().await; 821 | 822 | assert!(matches!( 823 | ret, 824 | Some(Err(MultipartError::EOFWhileReadingHeaders)) 825 | ),); 826 | } 827 | 828 | #[tokio::test] 829 | async fn unfinished_second_header() { 830 | let input: &[u8] = b"--AaB03x\r\n\ 831 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 832 | Content-Type: text/plain\r\n\ 833 | \r\n\ 834 | Lorem Ipsum\n\r\n\ 835 | --AaB03x\r\n\ 836 | Content-Disposition: form-data; name=\"name1\""; 837 | 838 | let mut read = MultipartParser::new("AaB03x", ByteStream::new(input)); 839 | 840 | if let Some(Ok(ParseOutput::Headers(val))) = read.next().await { 841 | println!("Headers:{:?}", val); 842 | } else { 843 | panic!("First value should be a header") 844 | } 845 | 846 | if let Some(Ok(ParseOutput::Bytes(bytes))) = read.next().await { 847 | assert_eq!(&*bytes, b"Lorem Ipsum\n"); 848 | } else { 849 | panic!("Second value should be bytes") 850 | } 851 | 852 | let ret = read.next().await; 853 | 854 | assert!(matches!( 855 | ret, 856 | Some(Err(MultipartError::EOFWhileReadingHeaders)) 857 | ),); 858 | } 859 | 860 | #[tokio::test] 861 | async fn invalid_header() { 862 | let input: &[u8] = b"--AaB03x\r\n\ 863 | I am a bad header\r\n\ 864 | \r\n"; 865 | 866 | let mut read = MultipartParser::new("AaB03x", ByteStream::new(input)); 867 | 868 | let val = read.next().await.unwrap(); 869 | 870 | match val { 871 | Err(MultipartError::HeaderParse(err)) => { 872 | //all good 873 | println!("{}", err); 874 | } 875 | val => { 876 | panic!("Expecting Parse Error, Instead got:{:?}", val); 877 | } 878 | } 879 | } 880 | 881 | #[tokio::test] 882 | async fn invalid_boundary() { 883 | let input: &[u8] = b"--InvalidBoundary\r\n\ 884 | Content-Disposition: form-data; name=\"file\"; filename=\"text.txt\"\r\n\ 885 | Content-Type: text/plain\r\n\ 886 | \r\n\ 887 | Lorem Ipsum\n\r\n\ 888 | --InvalidBoundary--\r\n"; 889 | 890 | let mut read = MultipartParser::new("AaB03x", ByteStream::new(input)); 891 | 892 | let val = read.next().await.unwrap(); 893 | 894 | match val { 895 | Err(MultipartError::InvalidBoundary { expected, found }) => { 896 | assert_eq!(expected, "--AaB03x\\r\\n"); 897 | assert_eq!(found, "--InvalidB"); 898 | } 899 | val => { 900 | panic!("Expecting Invalid Boundary Error, Instead got:{:?}", val); 901 | } 902 | } 903 | } 904 | 905 | #[tokio::test] 906 | async fn zero_read() { 907 | use bytes::{BufMut, BytesMut}; 908 | 909 | let input = b"----------------------------332056022174478975396798\r\n\ 910 | Content-Disposition: form-data; name=\"file\"\r\n\ 911 | Content-Type: application/octet-stream\r\n\ 912 | \r\n\ 913 | \r\n\ 914 | \r\n\ 915 | dolphin\n\ 916 | whale\r\n\ 917 | ----------------------------332056022174478975396798--\r\n"; 918 | 919 | let boundary = "--------------------------332056022174478975396798"; 920 | 921 | let mut read = MultipartStream::new(boundary, ByteStream::new(input)); 922 | 923 | let mut part = match read.next().await.unwrap() { 924 | Ok(mf) => { 925 | assert_eq!(mf.name().unwrap(), "file"); 926 | assert_eq!(mf.content_type().unwrap(), "application/octet-stream"); 927 | mf 928 | } 929 | Err(e) => panic!("unexpected: {}", e), 930 | }; 931 | 932 | let mut buffer = BytesMut::new(); 933 | 934 | loop { 935 | match part.next().await { 936 | Some(Ok(bytes)) => buffer.put(bytes), 937 | Some(Err(e)) => panic!("unexpected {}", e), 938 | None => break, 939 | } 940 | } 941 | 942 | let nth = read.next().await; 943 | assert!(nth.is_none()); 944 | 945 | assert_eq!(buffer.as_ref(), b"\r\n\r\ndolphin\nwhale"); 946 | } 947 | 948 | #[tokio::test] 949 | async fn r_read() { 950 | use std::convert::Infallible; 951 | 952 | //Used to ensure partial matches are working! 953 | 954 | #[derive(Clone)] 955 | pub struct SplitStream { 956 | packets: Vec, 957 | } 958 | 959 | impl SplitStream { 960 | pub fn new() -> Self { 961 | SplitStream { packets: vec![] } 962 | } 963 | 964 | pub fn add_packet>(&mut self, bytes: P) { 965 | self.packets.push(bytes.into()); 966 | } 967 | } 968 | 969 | impl Stream for SplitStream { 970 | type Item = Result; 971 | 972 | fn poll_next( 973 | mut self: Pin<&mut Self>, 974 | _cx: &mut Context<'_>, 975 | ) -> Poll> { 976 | if self.as_mut().packets.is_empty() { 977 | return Poll::Ready(None); 978 | } 979 | 980 | Poll::Ready(Some(Ok(self.as_mut().packets.remove(0)))) 981 | } 982 | } 983 | 984 | use bytes::{BufMut, BytesMut}; 985 | 986 | //This is a packet split on the boundary to test partial matching 987 | let input1: &[u8] = b"----------------------------332056022174478975396798\r\n\ 988 | Content-Disposition: form-data; name=\"file\"\r\n\ 989 | Content-Type: application/octet-stream\r\n\ 990 | \r\n\ 991 | \r\r\r\r\r\r\r\r\r\r\r\r\r\ 992 | \r\n\ 993 | ----------------------------332"; 994 | 995 | //This is the rest of the packet 996 | let input2: &[u8] = b"056022174478975396798--\r\n"; 997 | 998 | let boundary = "--------------------------332056022174478975396798"; 999 | 1000 | let mut split_stream = SplitStream::new(); 1001 | 1002 | split_stream.add_packet(&*input1); 1003 | split_stream.add_packet(&*input2); 1004 | 1005 | let mut read = MultipartStream::new(boundary, split_stream); 1006 | 1007 | let mut part = match read.next().await.unwrap() { 1008 | Ok(mf) => { 1009 | assert_eq!(mf.name().unwrap(), "file"); 1010 | assert_eq!(mf.content_type().unwrap(), "application/octet-stream"); 1011 | mf 1012 | } 1013 | Err(e) => panic!("unexpected: {}", e), 1014 | }; 1015 | 1016 | let mut buffer = BytesMut::new(); 1017 | 1018 | loop { 1019 | match part.next().await { 1020 | Some(Ok(bytes)) => buffer.put(bytes), 1021 | Some(Err(e)) => panic!("unexpected {}", e), 1022 | None => break, 1023 | } 1024 | } 1025 | 1026 | let nth = read.next().await; 1027 | assert!(nth.is_none()); 1028 | 1029 | assert_eq!(buffer.as_ref(), b"\r\r\r\r\r\r\r\r\r\r\r\r\r"); 1030 | } 1031 | 1032 | #[test] 1033 | fn test_strip_no_strip_necessary() { 1034 | let name: Cow = Cow::Owned("äöüß.txt".to_owned()); 1035 | 1036 | let res = strip_utf8_prefix(name.clone()); 1037 | 1038 | assert_eq!(res, name); 1039 | } 1040 | 1041 | #[test] 1042 | fn test_strip_uppercase_utf8() { 1043 | let name: Cow = Cow::Owned("UTF-8''äöüß.txt".to_owned()); 1044 | 1045 | let res = strip_utf8_prefix(name); 1046 | 1047 | assert_eq!(res, "äöüß.txt"); 1048 | } 1049 | 1050 | #[test] 1051 | fn test_strip_lowercase_utf8() { 1052 | let name: Cow = Cow::Owned("utf-8''äöüß.txt".to_owned()); 1053 | 1054 | let res = strip_utf8_prefix(name); 1055 | 1056 | assert_eq!(res, "äöüß.txt"); 1057 | } 1058 | } 1059 | --------------------------------------------------------------------------------