├── .gitignore ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── indicatif-reqwest-tokio ├── Cargo.toml ├── README.md └── src │ └── bin │ ├── indicatif-reqwest-tokio-multi.rs │ └── indicatif-reqwest-tokio-single.rs ├── indicatif-tokio ├── Cargo.toml ├── README.md └── src │ └── bin │ ├── multi.rs │ └── single.rs ├── reqwest-tokio-compat ├── Cargo.toml ├── README.md └── src │ └── main.rs ├── reqwest-tokio ├── Cargo.toml ├── README.md └── src │ └── main.rs └── util ├── Cargo.toml └── src ├── error.rs └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | **/target 2 | **/Cargo.lock 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "indicatif-tokio", 5 | "indicatif-reqwest-tokio", 6 | "reqwest-tokio", 7 | "reqwest-tokio-compat", 8 | "util", 9 | ] 10 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2016 Alex Crichton 190 | Copyright (c) 2017 The Tokio Authors 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Alex Crichton 2 | Copyright (c) 2017 The Tokio Authors 3 | 4 | Permission is hereby granted, free of charge, to any 5 | person obtaining a copy of this software and associated 6 | documentation files (the "Software"), to deal in the 7 | Software without restriction, including without 8 | limitation the rights to use, copy, modify, merge, 9 | publish, distribute, sublicense, and/or sell copies of 10 | the Software, and to permit persons to whom the Software 11 | is furnished to do so, subject to the following 12 | conditions: 13 | 14 | The above copyright notice and this permission notice 15 | shall be included in all copies or substantial portions 16 | of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 19 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 20 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 21 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 22 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 23 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 25 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 26 | DEALINGS IN THE SOFTWARE. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Async Applied 2 | 3 | Rust's async ecosystem continues to evolve rapidly -- faster than my free time permits me to write examples! As a result, **this repository is not actively maintained.** Please feel free to explore the old code here, and consider searching for more recent examples. 4 | 5 | ## Introduction 6 | 7 | The Rust programming language supports the powerful concurrency concept of _tasks_, non-blocking computations that model cooperative concurrency. Tasks are implemented in rust using [futures](https://doc.rust-lang.org/std/future/trait.Future.html), [async](https://doc.rust-lang.org/std/keyword.async.html), and [await](https://doc.rust-lang.org/std/keyword.await.html). These features were [stabilized quite recently](https://blog.rust-lang.org/2019/11/07/Async-await-stable.html) on 11/7/2019. Consequently, the ecosystem of crates that depend on them is still very much in flux, and there are not a lot of examples of how to use async/await in practice. 8 | 9 | This repository is _not_ intended as yet another tutorial or primer on async/await. This repository _is_ intended to showcase examples demonstrating practical use of async/await applied to common, real-world problems. You should familiarize yourself with async/await by reading one of the [excellent](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html) [existing](https://book.async.rs/) [tutorials](https://tokio.rs/docs/getting-started/hello-world/) before you attempt the examples herein. 10 | 11 | You are welcome to contribute to examples to or steal examples from this repository. 12 | 13 | ## Examples 14 | 15 | This repository contains examples of practical async code use that you can download, build, and experiment with. There are admittedly not many examples right now, but I hope to gradually add more as time permits. 16 | 17 | * [reqwest-tokio](./reqwest-tokio/README.md) download a file using [reqwest](https://github.com/seanmonstar/reqwest) and [tokio](https://tokio.rs). 18 | * [reqwest-tokio-compat](./reqwest-tokio-compat/README.md) download a file using [reqwest](https://github.com/seanmonstar/reqwest) and [tokio](https://tokio.rs) using [`tokio_util::compat`](https://github.com/tokio-rs/tokio/blob/master/tokio-util/src/compat.rs) to harmonize traits between [futures](https://github.com/rust-lang/futures-rs) and tokio. 19 | * [indicatif-tokio](./indicatif-tokio/README.md) shows the usage of progress bars with [indicatif](https://github.com/mitsuhiko/indicatif) for iterable asynchronous tasks, single and concurrent multi examples are given 20 | * [indicatif-reqwest-tokio](./indicatif-reqwest-tokio/README.md) is a combination of `reqwest-tokio` and `indicatif-tokio` 21 | 22 | ## Async is Not Threads 23 | 24 | For those who inevitably are not going to familiarize themselves with async/await before reading further, it is important to understand that **async is not threads**. 25 | 26 | Async is _related to_ threads. When most programmers think of concurrency they think of threads. Threads are lightweight processes managed by the operating system that share memory and may run concurrently on multiple physical processing units (i.e. cpu cores). Async, await, and futures are about _tasks_. Tasks are abstract units of computation that may be run concurrently. Tasks may be scheduled to run on different threads, or they may run interleaved on the same thread, or some combination of the above. 27 | 28 | The most important distinction between _tasks_ and _threads_ is that tasks written with **async should not block**. Underneath the hood, tasks (modeled as [futures](https://doc.rust-lang.org/std/future/trait.Future.html)) implement a method `poll()`. Each time `poll()` is called the task performs a little bit of computation. Something called the _runtime_ will call `poll()` repeatedly until the task completes all of its computations. Each call to `poll()` will take some small amount of time to return, but the task should not _block_ other tasks by making `poll()` take a really long time to return. 29 | 30 | For example, if writing an async task to compute the first one million numbers in the fibonnaci sequence, each call to `poll()` should perform just a little bit of the computation (e.g. compute just the next number). The task should not block by having `poll()` compute all one million numbers all at once. 31 | 32 | **If you write tasks that block your async programs will not behave as expected.** The Rust compiler cannot know if your task is going to block or not, so it is up to you the programmer to write non-blocking tasks. If you do not understand the difference between tasks and threads, or if you do not understand the concept of an asynchronous, non-blocking task, **stop**. Go back and read an async/await tutorial before you proceed. 33 | 34 | ## The Async Ecosystem 35 | 36 | Because you have familiarized yourself with async/await by reading other tutorials, you know that async/await syntax requires the use of a runtime (a.k.a. executor) to schedule async tasks and drive them to completion. You also know that writing an async/await task that performs io (input/output) is impossible with the primitives in [std::io](https://doc.rust-lang.org/std/io/) because the standard library primitives block. Therefore, in addition to a runtime, any non-trivial async task that performs io needs non-blocking, asynchronous io primitives. At the time of this writing _none_ of these features are provided by the Rust standard library. They all exist in crates within the async ecosystem. 37 | 38 | ### The Vocabulary 39 | 40 | At the bottom of the async ecosystem is a set of traits that serve as a common vocabulary to describe how the different parts of the ecosystem interact with each other. Currently these are in: 41 | 42 | * [`std::future`](https://doc.rust-lang.org/std/future), the parts of the async ecosystem that have been stabilized, most notably: 43 | - [`Future`](https://doc.rust-lang.org/std/future/trait.Future.html), the fundamental trait describing an asynchronous unit of computation. 44 | * [futures](https://github.com/rust-lang/futures-rs) 0.3, a crate that defines all the traits which _haven't_ been stabilized... yet. Expect that many of the traits in futures will eventually find their way into `std::future`. These include: 45 | - [`AsyncRead`](https://docs.rs/futures/0.3.4/futures/io/trait.AsyncRead.html), an asynchronous version of [`std::io::Read`](https://doc.rust-lang.org/std/io/trait.Read.html). 46 | - [`AsyncRead`](https://docs.rs/futures/0.3.4/futures/io/trait.AsyncWrite.html), an asynchronous version of [`std::io::Write`](https://doc.rust-lang.org/std/io/trait.Write.html). 47 | - [`Stream`](https://docs.rs/futures/0.3.4/futures/stream/trait.Stream.html), an asynchronous analog to [`std::iter::Iterator`](https://doc.rust-lang.org/std/iter/trait.Iterator.html). 48 | 49 | Note that in the beginning, before async/await was stabilized, the [futures](https://github.com/rust-lang/futures-rs) crate defined everything async and even provided macros like `await!` that are mostly not needed now that async and await have been stabilized. To avoid encountering such fossils you should make sure to use futures version 0.3 or greater. 50 | 51 | ### The Foundation 52 | 53 | You are free to implement the async traits yourself, and in fact it is not that hard to implement a `Future` to perform an asynchronous computation (e.g. fibbonaci sequence). Asynchronous input/output (io) operations are much more complicated to implement because they require support from the operating system. The foundation of async is a set of io-related crates upon which the rest of the async ecosystem rests. They are important and so you should know they exist -- but it is unlikely you will ever need to use them directly. 54 | 55 | * [mio](https://github.com/tokio-rs/mio) interfaces with the operating system to define asynchronous versions of file, socket, etc primitives. 56 | * [hyper](https://github.com/hyperium/hyper) is an HTTP protocol implementation with an asynchronous design. 57 | 58 | ### The Building Blocks 59 | 60 | The foundational crates above are quite primitive, and it would not be very ergonomic to use them directly. The following crates build upon that foundation to implement traits like `AsyncRead` on, for example, a file. They define the modules and structures you will interact with most often in the async ecosystem. Note that some of these crates are listed again under Runtimes below. 61 | 62 | * [async-std](https://async.rs) 1.5 implements async traits on files, sockets, etc. It aims to be easy to learn and hew closely to the the style of its synchronous counterparts in `std::io`. It is compatible with all the foundational traits in [futures](https://github.com/rust-lang/futures-rs). 63 | * [tokio](https://tokio.rs) 0.2 implements async traits on files, sockets, etc, making it very similar to async-std. Although it is "only" version 0.2, tokio has actually been around for longer than async-std, is used in more projects, and is arguably more stable. However, the async ecosystem is young and it's not clear which crate is "better." One current disadvantage of using tokio is that [`tokio::io::AsyncRead`](https://docs.rs/tokio/0.2.13/tokio/io/trait.AsyncRead.html) are not compatible with [`futures::io::AsyncRead`](https://docs.rs/futures/0.3.4/futures/io/trait.AsyncRead.html) et al, although there is a compatibility layer. The reasons for this difference are summarized [here](https://www.reddit.com/r/rust/comments/enn3ax/strategies_for_futuresioasyncread_vs/) and [here](https://github.com/rust-lang/futures-rs/pull/1826). 64 | * [tokio-util](https://github.com/tokio-rs/tokio/tree/master/tokio-util) contains some additional functionality built around tokio. As of version 0.3.0 it includes a [compatibility layer between tokio and futures](https://docs.rs/tokio-util/0.3.0/tokio_util/compat/index.html). 65 | * [reqwest](https://github.com/seanmonstar/reqwest) is a crate for async HTTP/TLS (think curl, wget, etc). Don't let the strange name fool you; it is the best and only choice for async web requests. 66 | * [warp](https://github.com/seanmonstar/warp) is like reqwest but for the server side. 67 | * [rusoto](https://github.com/rusoto/rusoto) 0.43 is an async API for Amazon Web Services (AWS) including S3. You will need at least version 0.43, which is [currently beta](https://linuxwit.ch/blog/2020/02/the-future-of-rusoto/). 68 | 69 | ### The Runtimes 70 | 71 | The runtime (also called executor or reactor) is what schedules your async tasks and drives them to completion. Although it may seem counterintuitive at first, just as there are multiple building blocks that implement the same trait different use cases (e.g. file access vs web access), there are also multiple runtimes with slightly different performance objective and use cases. These include: 72 | 73 | * `std::runtime` -- just kidding! There is no runtime in the Rust standard library. That's right, even though async/await are stabilized you will still need to use one of the runtime crates below to actually run any tasks concurrently. This is because, as per above, there is no unifying runtime use case or obviously best runtime to be standardized. 74 | * [async-std](https://async.rs) 1.5 is a runtime bundled with async-std. On the backend, the runtime has to keep track of the state of its various io operations to work efficiently, so it makes sense for the runtime and io drivers to live in the same crate. 75 | * [tokio](https://tokio.rs) 0.2 is a runtime bundled with tokio. You can use it with futures-compatible traits via [`tokio_util::compat`](https://github.com/tokio-rs/tokio/blob/master/tokio_util/src/compat.rs). 76 | * [futures](https://github.com/rust-lang/futures-rs) provides its own runtime too, but at least for now it is meant more for trivially simple tasks and internal testing. You will likely want to pick one of async-std or tokio to use in production code. 77 | * [rayon](https://github.com/rayon-rs/rayon) is **not** part of the async ecosystem but deserves honorable mention for being an excellent thread scheduling pool for implementing parallel iterators on CPU-bound, blocking computations. 78 | 79 | ### The High-Level Stuff 80 | 81 | As Rusts's async ecosystem matures we are starting to see high-level frameworks built on top of tokio and other mid-level crates. Chief among these is the appropriately-named: 82 | 83 | * [tower](https://github.com/tower-rs/tower), a network services framework. 84 | 85 | ## License 86 | 87 | This project is licensed under either of 88 | 89 | * Apache License, Version 2.0, (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) 90 | * MIT license (LICENSE-MIT or http://opensource.org/licenses/MIT) 91 | 92 | at your option. -------------------------------------------------------------------------------- /indicatif-reqwest-tokio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "indicatif-reqwest-tokio" 3 | version = "0.1.0" 4 | authors = ["Beh "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | futures = "0.3" 11 | indicatif = "^0" 12 | rand = "^0" 13 | # For a truly multithreaded tokio runtime (overkill for this example), 14 | # replace rt-core with rt-threaded. 15 | tokio = { version = "0.2", features = ["macros", "rt-threaded", "time", "stream", "blocking", "fs"] } 16 | reqwest = "0.10" 17 | util = { path = "../util" } -------------------------------------------------------------------------------- /indicatif-reqwest-tokio/README.md: -------------------------------------------------------------------------------- 1 | This example is part of a larger repository of examples, [async-applied](../README.md). 2 | 3 | # indicatif-reqwest-tokio 4 | 5 | This is basically a combined example of [`indicatif-tokio`](../indicatif-tokio/README.md) and [`reqwest-tokio`](../reqwest-tokio/README.md). If you want pretty downloads with indicatif, this is probably what you want. 6 | 7 | The files downloaded are from [https://file-examples.com](https://file-examples.com). -------------------------------------------------------------------------------- /indicatif-reqwest-tokio/src/bin/indicatif-reqwest-tokio-multi.rs: -------------------------------------------------------------------------------- 1 | // Downloads multiple files from https://file-examples.com. 2 | // Demonstrates basic use of reqwest for async http(s) requests and showing an indicatif status bar for the downloads. 3 | // This is a combination of reqwest-tokio and indicatif-tokio 4 | 5 | use std::sync::Arc; 6 | 7 | use futures::{stream, StreamExt}; 8 | use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; 9 | use reqwest::{header, Client, Url}; 10 | use tokio::{io::AsyncWriteExt}; 11 | 12 | async fn download_task( 13 | download_url: &str, 14 | multibar: Arc, 15 | ) -> Result<(), util::BoxError> { 16 | // Parse URL into Url type 17 | let url = Url::parse(download_url)?; 18 | 19 | // Create a reqwest Client 20 | let client = Client::new(); 21 | 22 | // We need to determine the file size before we download, so we can create a ProgressBar 23 | // A Header request for the CONTENT_LENGTH header gets us the file size 24 | let download_size = { 25 | let resp = client.head(url.as_str()).send().await?; 26 | if resp.status().is_success() { 27 | resp.headers() // Gives us the HeaderMap 28 | .get(header::CONTENT_LENGTH) // Gives us an Option containing the HeaderValue 29 | .and_then(|ct_len| ct_len.to_str().ok()) // Unwraps the Option as &str 30 | .and_then(|ct_len| ct_len.parse().ok()) // Parses the Option as u64 31 | .unwrap_or(0) // Fallback to 0 32 | } else { 33 | // We return an Error if something goes wrong here 34 | return Err( 35 | format!("Couldn't download URL: {}. Error: {:?}", url, resp.status(),).into(), 36 | ); 37 | } 38 | }; 39 | 40 | // Parse the filename from the given URL 41 | let filename = url 42 | .path_segments() // Splits into segments of the URL 43 | .and_then(|segments| segments.last()) // Retrieves the last segment 44 | .unwrap_or("file.download"); // Fallback to generic filename 45 | 46 | // Here we build the actual Request with a RequestBuilder from the Client 47 | let request = client.get(url.as_str()); 48 | 49 | // Create the ProgressBar with the aquired size from before 50 | // and add it to the multibar 51 | let progress_bar = multibar.add(ProgressBar::new(download_size)); 52 | 53 | // Set Style to the ProgressBar 54 | progress_bar.set_style( 55 | ProgressStyle::default_bar() 56 | .template("[{bar:40.cyan/blue}] {bytes}/{total_bytes} - {msg}") 57 | .progress_chars("#>-"), 58 | ); 59 | 60 | // Set the filename as message part of the progress bar 61 | progress_bar.set_message(&filename); 62 | 63 | // Create the output file with tokio's async fs lib 64 | let mut outfile = tokio::fs::File::create(&filename).await?; 65 | 66 | // Do the actual request to download the file 67 | let mut download = request.send().await?; 68 | 69 | // Do an asynchronous, buffered copy of the download to the output file. 70 | // 71 | // We use the part from the reqwest-tokio example here on purpose 72 | // This way, we are able to increase the ProgressBar with every downloaded chunk 73 | while let Some(chunk) = download.chunk().await? { 74 | progress_bar.inc(chunk.len() as u64); // Increase ProgressBar by chunk size 75 | outfile.write(&chunk).await?; // Write chunk to output file 76 | } 77 | 78 | // Finish the progress bar to prevent glitches 79 | progress_bar.finish(); 80 | 81 | // Must flush tokio::io::BufWriter manually. 82 | // It will *not* flush itself automatically when dropped. 83 | outfile.flush().await?; 84 | 85 | Ok(()) 86 | } 87 | 88 | #[tokio::main] 89 | async fn main() -> Result<(), util::BoxError> { 90 | // A vector containing all the URLs to download 91 | let download_links = vec![ 92 | "https://file-examples-com.github.io/uploads/2017/11/file_example_WAV_10MG.wav", // 10MB WAV audio file 93 | "https://file-examples-com.github.io/uploads/2017/11/file_example_OOG_2MG.ogg", // 2MB OGG audio file 94 | "https://file-examples-com.github.io/uploads/2017/10/file_example_PNG_3MB.png", // 3MB PNG image 95 | "https://file-examples-com.github.io/uploads/2017/10/file_example_JPG_1MB.jpg", // 1MB JPG image 96 | ]; 97 | 98 | // Set up a new multi-progress bar. 99 | // The bar is stored in an `Arc` to facilitate sharing between threads. 100 | let multibar = std::sync::Arc::new(indicatif::MultiProgress::new()); 101 | 102 | // Add an overall progress indicator to the multibar. 103 | // It has as many steps as the download_links Vector and will increment on completion of each task. 104 | let main_pb = std::sync::Arc::new( 105 | multibar 106 | .clone() 107 | .add(indicatif::ProgressBar::new(download_links.len() as u64)), 108 | ); 109 | main_pb.set_style( 110 | indicatif::ProgressStyle::default_bar().template("{msg} {bar:10} {pos}/{len}"), 111 | ); 112 | main_pb.set_message("total "); 113 | 114 | // Make the main progress bar render immediately rather than waiting for the 115 | // first task to finish. 116 | main_pb.tick(); 117 | 118 | // Convert download_links Vector into stream 119 | // This is basically a async compatible iterator 120 | let stream = stream::iter(download_links); 121 | 122 | // Set up a future to iterate over tasks and run up to 2 at a time. 123 | let tasks = stream 124 | .enumerate() 125 | .for_each_concurrent(Some(2), |(_i, download_link)| { 126 | // Clone multibar and main_pb. We will move the clones into each task. 127 | let multibar = multibar.clone(); 128 | let main_pb = main_pb.clone(); 129 | async move { 130 | // Spawn a new tokio task for the current download link 131 | // We need to hand over the multibar, so the ProgressBar for the task can be added 132 | let _task = tokio::task::spawn(download_task(download_link, multibar)).await; 133 | 134 | // Increase main ProgressBar by 1 135 | main_pb.inc(1); 136 | } 137 | }); 138 | 139 | // Set up a future to manage rendering of the multiple progress bars. 140 | let multibar = { 141 | // Create a clone of the multibar, which we will move into the task. 142 | let multibar = multibar.clone(); 143 | 144 | // multibar.join() is *not* async and will block until all the progress 145 | // bars are done, therefore we must spawn it on a separate scheduler 146 | // on which blocking behavior is allowed. 147 | tokio::task::spawn_blocking(move || multibar.join()) 148 | }; 149 | 150 | // Wait for the tasks to finish. 151 | tasks.await; 152 | 153 | // Change the message on the overall progress indicator. 154 | main_pb.finish_with_message("done"); 155 | 156 | // Wait for the progress bars to finish rendering. 157 | // The first ? unwraps the outer join() in which we are waiting for the 158 | // future spawned by tokio::task::spawn_blocking to finish. 159 | // The second ? unwraps the inner multibar.join(). 160 | multibar.await??; 161 | 162 | Ok(()) 163 | } 164 | -------------------------------------------------------------------------------- /indicatif-reqwest-tokio/src/bin/indicatif-reqwest-tokio-single.rs: -------------------------------------------------------------------------------- 1 | // Downloads a 10MB video example file from https://file-examples.com. 2 | // Demonstrates basic use of reqwest for async http(s) requests and showing an indicatif status bar for the download. 3 | 4 | // Needed to be able to call flush() on a tokio::io::AsyncWrite. 5 | use indicatif::{ProgressBar, ProgressStyle}; 6 | use reqwest::{Client, Url, header}; 7 | use tokio::io::AsyncWriteExt; 8 | 9 | // tokio::main macro automatically sets up the tokio runtime. 10 | #[tokio::main] 11 | async fn main() -> Result<(), util::BoxError> { 12 | // Set the URL of the file to download, we us a 10MB example video here 13 | let download_url_str = "https://file-examples-com.github.io/uploads/2017/04/file_example_MP4_1280_10MG.mp4"; 14 | 15 | // Parse URL into Url type 16 | let url = Url::parse(download_url_str)?; 17 | 18 | // Create a reqwest Client 19 | let client = Client::new(); 20 | 21 | // We need to determine the file size before we download so we can create a ProgressBar 22 | // A Header request for the CONTENT_LENGTH header gets us the file size 23 | let download_size = { 24 | let resp = client.head(url.as_str()).send().await?; 25 | if resp.status().is_success() { 26 | resp.headers() // Gives is the HeaderMap 27 | .get(header::CONTENT_LENGTH) // Gives us an Option containing the HeaderValue 28 | .and_then(|ct_len| ct_len.to_str().ok()) // Unwraps the Option as &str 29 | .and_then(|ct_len| ct_len.parse().ok()) // Parses the Option as u64 30 | .unwrap_or(0) // Fallback to 0 31 | } else { 32 | // We return an Error if something goes wrong here 33 | return Err(format!( 34 | "Couldn't download URL: {}. Error: {:?}", 35 | url, 36 | resp.status(), 37 | ) 38 | .into()); 39 | } 40 | }; 41 | 42 | // Parse the filename from the given URL 43 | let filename = url 44 | .path_segments() // Splits into segments of the URL 45 | .and_then(|segments| segments.last()) // Retrieves the last segment 46 | .unwrap_or("video.mp4"); // Fallback to generic filename 47 | 48 | // Here we build the actual Request with a RequestBuilder from the Client 49 | let request = client.get(url.as_str()); 50 | 51 | // Create the ProgressBar with the aquired size from before 52 | let progress_bar = ProgressBar::new(download_size); 53 | 54 | // Set Style to the ProgressBar 55 | progress_bar.set_style( 56 | ProgressStyle::default_bar() 57 | .template("[{bar:40.cyan/blue}] {bytes}/{total_bytes} - {msg}") 58 | .progress_chars("#>-"), 59 | ); 60 | 61 | // Set the filename as message part of the progress bar 62 | progress_bar.set_message(&filename); 63 | 64 | // Create the output file 65 | let mut outfile = tokio::fs::File::create(&filename).await?; 66 | 67 | // Do the actual request to download the file 68 | let mut download = request.send().await?; 69 | 70 | // Do an asynchronous, buffered copy of the download to the output file. 71 | // 72 | // We use the part from the reqwest-tokio example here on purpose 73 | // This way, we are able to increase the ProgressBar with every downloaded chunk 74 | while let Some(chunk) = download.chunk().await? { 75 | progress_bar.inc(chunk.len() as u64); // Increase ProgressBar by chunk size 76 | outfile.write(&chunk).await?; // Write chunk to output file 77 | } 78 | 79 | // Finish the progress bar to prevent glitches 80 | progress_bar.finish(); 81 | 82 | // Must flush tokio::io::BufWriter manually. 83 | // It will *not* flush itself automatically when dropped. 84 | outfile.flush().await?; 85 | 86 | Ok(()) 87 | } -------------------------------------------------------------------------------- /indicatif-tokio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "indicatif-tokio" 3 | description = "Progress bars with indicatif and the tokio runtime." 4 | version = "0.1.0" 5 | authors = ["Benjamin Kay "] 6 | edition = "2018" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | futures = "0.3" 12 | indicatif = "^0" 13 | rand = ">= 0.8, < 1.0" 14 | # For a truly multithreaded tokio runtime (overkill for this example), 15 | # replace rt-core with rt-threaded. 16 | tokio = { version = "0.2", features = ["macros", "rt-threaded", "time", "stream", "blocking"] } 17 | util = { path = "../util" } 18 | -------------------------------------------------------------------------------- /indicatif-tokio/README.md: -------------------------------------------------------------------------------- 1 | This example is part of a larger repository of examples, [async-applied](../README.md). 2 | 3 | # indicatif-tokio 4 | 5 | [indicatif](https://github.com/mitsuhiko/indicatif) is a crate for rendering progress bars in the terminal. Indicatif does not support async/await syntax per se, but it does support concurrency well enough that you can use it to display the progress of async tasks. -------------------------------------------------------------------------------- /indicatif-tokio/src/bin/multi.rs: -------------------------------------------------------------------------------- 1 | use futures::stream::StreamExt; 2 | use rand::Rng; 3 | 4 | // tokio::main macro automatically sets up the tokio runtime. 5 | #[tokio::main] 6 | async fn main() -> Result<(), util::BoxError> { 7 | // Generate a stream of 10 tasks each with 10 steps. 8 | // Each task progresses at a random rate, 9 | // with a delay between 500 and 1500 ms. 10 | let task_steps = 10; 11 | let task_delay_millis: (u64, u64) = (500, 1500); 12 | let ntasks = 10; 13 | let mut rng = rand::thread_rng(); 14 | let tasks = futures::stream::repeat(()) 15 | .map(|_| tokio::time::interval( 16 | tokio::time::Duration::from_millis( 17 | rng.gen_range(task_delay_millis.0.. 18 | task_delay_millis.1 19 | ) 20 | )) 21 | .take(task_steps)).take(ntasks); 22 | 23 | // Set up a new multi-progress bar. 24 | // The bar is stored in an `Arc` to facilitate sharing between threads. 25 | let multibar = std::sync::Arc::new(indicatif::MultiProgress::new()); 26 | 27 | // Add an overall progress indicator to the multibar. 28 | // It has 10 steps and will increment on completion of each task. 29 | let main_pb = std::sync::Arc::new(multibar.clone().add(indicatif::ProgressBar::new(ntasks as u64))); 30 | main_pb.set_style( 31 | indicatif::ProgressStyle::default_bar() 32 | .template("{msg} {bar:10} {pos}/{len}") 33 | ); 34 | main_pb.set_message("total "); 35 | 36 | // Make the main progress bar render immediately rather than waiting for the 37 | // first task to finish. 38 | main_pb.tick(); 39 | 40 | // Set up a future to iterate over tasks and run up to 3 at a time. 41 | let tasks = tasks.enumerate().for_each_concurrent(Some(3), |(i, interval)| { 42 | // Clone multibar and main_pb. We will move the clones into each task. 43 | let multibar = multibar.clone(); 44 | let main_pb = main_pb.clone(); 45 | async move { 46 | // Add a new progress indicator to the multibar. 47 | let task_pb = multibar.add(indicatif::ProgressBar::new(task_steps as u64)); 48 | task_pb.set_style( 49 | indicatif::ProgressStyle::default_bar() 50 | .template("task {msg} {bar:10} {pos}/{len}") 51 | ); 52 | task_pb.set_message(&format!("{:>2}", i+1)); 53 | 54 | // Increment this task's progress indicator. 55 | interval.for_each(|_| async { task_pb.inc(1) }).await; 56 | 57 | // Increment the overall progress indicator. 58 | main_pb.inc(1); 59 | 60 | // Clear this tasks's progress indicator. 61 | task_pb.finish_and_clear(); 62 | } 63 | }); 64 | 65 | // Set up a future to manage rendering of the multiple progress bars. 66 | let multibar = { 67 | // Create a clone of the multibar, which we will move into the task. 68 | let multibar = multibar.clone(); 69 | 70 | // multibar.join() is *not* async and will block until all the progress 71 | // bars are done, therefore we must spawn it on a separate scheduler 72 | // on which blocking behavior is allowed. 73 | tokio::task::spawn_blocking(move || { multibar.join() }) 74 | }; 75 | 76 | // Wait for the tasks to finish. 77 | tasks.await; 78 | 79 | // Change the message on the overall progress indicator. 80 | main_pb.finish_with_message("done"); 81 | 82 | // Wait for the progress bars to finish rendering. 83 | // The first ? unwraps the outer join() in which we are waiting for the 84 | // future spawned by tokio::task::spawn_blocking to finish. 85 | // The second ? unwraps the inner multibar.join(). 86 | multibar.await??; 87 | 88 | Ok(()) 89 | } 90 | -------------------------------------------------------------------------------- /indicatif-tokio/src/bin/single.rs: -------------------------------------------------------------------------------- 1 | use futures::stream::StreamExt; 2 | 3 | // tokio::main macro automatically sets up the tokio runtime. 4 | #[tokio::main] 5 | async fn main() -> Result<(), util::BoxError> { 6 | // Initialize a progress bar with 10 steps. 7 | let steps = 10; 8 | let pb = indicatif::ProgressBar::new(steps); 9 | pb.set_style( 10 | indicatif::ProgressStyle::default_bar() 11 | .template("for_each {bar:10} {pos}/{len}") 12 | ); 13 | 14 | // Stream that yields values no more than once every second. 15 | // FYI, the value yielded by the stream is a tokio::time::Instant, which is 16 | // a timestamp, but we do not use it in this example. 17 | let interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); 18 | 19 | // Iterate over the first 10 values in the stream. 20 | // Increment the progress bar by one for each item. 21 | interval.take(steps as usize).for_each(|_| async { pb.inc(1) } ).await; 22 | 23 | // Finish the progress bar. 24 | pb.finish(); 25 | 26 | // Same as above, but using for_each_concurrent. 27 | // The first parameter is a limit, which limits the number of futures that 28 | // can be run concurrently. In this case the limit is 1, making the 29 | // behavior the same as if we had simply called for_each. 30 | let pb = indicatif::ProgressBar::new(steps); 31 | pb.set_style( 32 | indicatif::ProgressStyle::default_bar() 33 | .template("for_each_concurrent, limit 1 {bar:10} {pos}/{len}") 34 | ); 35 | let interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); 36 | interval.take(steps as usize).for_each_concurrent( 37 | Some(1), 38 | |_| async { pb.inc(1) } 39 | ).await; 40 | pb.finish(); 41 | 42 | // Now with the limit set to None, which will attempt to run all the futures 43 | // concurrently. The point here is to show that indicatif can handle 44 | // concurrency. However, due to the nature of tokio::time::Interval, the 45 | // progress bar will not actually advance any faster. 46 | let pb = indicatif::ProgressBar::new(steps); 47 | pb.set_style( 48 | indicatif::ProgressStyle::default_bar() 49 | .template("for_each_concurrent, no limit {bar:10} {pos}/{len}") 50 | ); 51 | let interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); 52 | interval.take(steps as usize).for_each_concurrent( 53 | None, 54 | |_| async { pb.inc(1) } 55 | ).await; 56 | pb.finish(); 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /reqwest-tokio-compat/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "reqwest-tokio-compat" 3 | description = "Download files from the Internet the async way." 4 | version = "0.1.0" 5 | authors = ["Benjamin Kay "] 6 | edition = "2018" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | futures = "0.3" 12 | reqwest = { version = "0.10", features = ["stream"] } 13 | # For a truly multithreaded tokio runtime (overkill for this example), 14 | # replace rt-core with rt-threaded. 15 | tokio = { version = "0.2", features = ["macros", "rt-core", "fs", "io-util", "io-driver"] } 16 | tokio-util = { version = "~0.3.0", features = ["compat"] } 17 | util = { path = "../util" } 18 | 19 | -------------------------------------------------------------------------------- /reqwest-tokio-compat/README.md: -------------------------------------------------------------------------------- 1 | This example is part of a larger repository of examples, [async-applied](../README.md). 2 | 3 | # reqwest-tokio-compat 4 | 5 | [reqwest](https://github.com/seanmonstar/reqwest) is an excellent crate for making HTTP requests in the vein of wget, curl, etc. [tokio](https://tokio.rs) is the de facto Rust async runtime, especially for io-driven tasks. This example builds on the [reqwest-tokio](../reqwest-tokio/README.md) example. In that example we wondered why we could not call [`tokio::io::copy`](https://docs.rs/tokio/0.2.11/tokio/io/fn.copy.html) to copy the body of the [`reqwest::Response`](https://docs.rs/reqwest/0.10.1/reqwest/struct.Response.html) into `outfile`? 6 | 7 | For [reasons related to good API design](https://github.com/seanmonstar/reqwest/issues/482) a `request::Response` does not implement `AsyncRead` directly. It is, however, possible to convert it to a [`futures::io::Stream`](https://docs.rs/futures/0.3.4/futures/stream/trait.Stream.html) using [`bytes_stream()`](https://docs.rs/reqwest/0.10.1/reqwest/struct.Response.html#method.bytes_stream). A `Stream` is naught but an async iterator, analagous to how we called `while let Some(chunk) = download.chunk().await?` in the [prior example](../reqwest-tokio/README.md). It is therefore not surprising that there is an [stream extensions trait](https://docs.rs/futures/0.3.4/futures/stream/trait.TryStreamExt.html) that allows us to go from a stream to a [`futures::io::AsyncRead`](https://docs.rs/futures/0.3.4/futures/io/trait.AsyncRead.html). Note that, because `AsyncRead` uses [`futures::io::Error`](https://docs.rs/futures/0.3.4/futures/io/struct.Error.html), we must [map](https://docs.rs/futures/0.3.4/futures/stream/trait.TryStreamExt.html#method.map_err) from [`request::Error`](https://docs.rs/reqwest/0.10.1/reqwest/struct.Error.html) in the process. 8 | 9 | Having turned our `reqwest::Response` into an `AsyncRead` it should now be straightforward to invoke [`tokio::io::copy`](https://docs.rs/tokio/0.2.13/tokio/io/fn.copy.html)... but it is not. For reasons described [here](https://www.reddit.com/r/rust/comments/enn3ax/strategies_for_futuresioasyncread_vs/) and [here](https://github.com/rust-lang/futures-rs/pull/1826), [`futures::io::AsyncRead`](https://docs.rs/futures/0.3.4/futures/io/trait.AsyncRead.html) is not compatible with [`tokio::io::AsyncRead`](https://docs.rs/tokio/0.2.11/tokio/io/trait.AsyncRead.html). Bummer. Fortunately there is a [compatibility layer between tokio and futures](https://docs.rs/tokio-util/0.3.0/tokio_util/compat/index.html)! 10 | 11 | The compatibility layer is found in a separate crate, [tokio-util](https://github.com/tokio-rs/tokio/tree/master/tokio-util). To use it we pull in the appropriate extension trait: 12 | 13 | ``` 14 | use tokio_util::compat::FuturesAsyncReadCompatExt; 15 | ``` 16 | 17 | And then simply call `compat()` in the `futures::io::AsyncRead` to make it into a `tokio::io::AsyncRead`. Although it is annoying that futures and tokio have chosen not to use the same traits (at least for now), converting from one to the other really is not that hard. 18 | 19 | > Note: To use the compatibility layer you will need to use tokio-util version 0.3 or greater in your `Cargo.toml`. -------------------------------------------------------------------------------- /reqwest-tokio-compat/src/main.rs: -------------------------------------------------------------------------------- 1 | // Downloads a picture of Ferris, the Rust mascot, from the Internet. 2 | // Demonstrates basic use of reqwest for async http(s) requests and downloading. 3 | // Demonstrates the futures <--> tokio compatibility layer for AsyncRead. 4 | 5 | // Compatibility trait lets us call `compat()` on a futures::io::AsyncRead 6 | // to convert it into a tokio::io::AsyncRead. 7 | use tokio_util::compat::FuturesAsyncReadCompatExt; 8 | 9 | // Lets us call into_async_read() to convert a futures::stream::Stream into a 10 | // futures::io::AsyncRead. 11 | use futures::stream::TryStreamExt; 12 | 13 | // tokio::main macro automatically sets up the tokio runtime. 14 | #[tokio::main] 15 | async fn main() -> Result<(), util::BoxError> { 16 | // Attempt to download ferris.. 17 | let download = reqwest::get("https://rustacean.net/assets/rustacean-orig-noshadow.png") 18 | .await? // await server response 19 | .error_for_status()?; // generate an error if server didn't respond OK 20 | 21 | // Convert the body of the response into a futures::io::Stream. 22 | let download = download.bytes_stream(); 23 | 24 | // Convert the stream into an futures::io::AsyncRead. 25 | // We must first convert the reqwest::Error into an futures::io::Error. 26 | let download = download 27 | .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) 28 | .into_async_read(); 29 | 30 | // Convert the futures::io::AsyncRead into a tokio::io::AsyncRead. 31 | let mut download = download.compat(); 32 | 33 | // Create an output file into which we will save ferris. 34 | let mut outfile = tokio::fs::File::create("ferris.png").await?; 35 | 36 | // Invoke tokio::io::copy to actually perform the download. 37 | tokio::io::copy(&mut download, &mut outfile).await?; 38 | 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /reqwest-tokio/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "reqwest-tokio" 3 | description = "Download files from the Internet the async way." 4 | version = "0.1.0" 5 | authors = ["Benjamin Kay "] 6 | edition = "2018" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | reqwest = "0.10" 12 | # For a truly multithreaded tokio runtime (overkill for this example), 13 | # replace rt-core with rt-threaded. 14 | tokio = { version = "0.2", features = ["macros", "rt-core", "fs", "io-util", "io-driver"] } 15 | util = { path = "../util" } 16 | -------------------------------------------------------------------------------- /reqwest-tokio/README.md: -------------------------------------------------------------------------------- 1 | This example is part of a larger repository of examples, [async-applied](../README.md). 2 | 3 | # reqwest-tokio 4 | 5 | [reqwest](https://github.com/seanmonstar/reqwest) is an excellent crate for making HTTP requests in the vein of wget, curl, etc. [tokio](https://tokio.rs) is the de facto Rust async runtime, especially for io-driven tasks. This example demonstrates the simplest possible use of these two crates together to download a picture of [the Rust mascot, Ferris](https://rustacean.net/). 6 | 7 | If you have done much reading in the [tokio documentation](https://docs.rs/tokio) you may wonder why we use a `while let` loop to drive the download in "chunks" rather than calling [`tokio::io::copy`](https://docs.rs/tokio/0.2.13/tokio/io/fn.copy.html)? The answer is in the [reqwest-tokio-compat](../reqwest-tokio-compat/README.md) example. -------------------------------------------------------------------------------- /reqwest-tokio/src/main.rs: -------------------------------------------------------------------------------- 1 | // Downloads a picture of Ferris, the Rust mascot, from the Internet. 2 | // Demonstrates basic use of reqwest for async http(s) requests and downloading. 3 | 4 | // Needed to be able to call flush() on a tokio::io::AsyncWrite. 5 | use tokio::io::AsyncWriteExt; 6 | 7 | // tokio::main macro automatically sets up the tokio runtime. 8 | #[tokio::main] 9 | async fn main() -> Result<(), util::BoxError> { 10 | // Attempt to download ferris.. 11 | let mut download = reqwest::get("https://rustacean.net/assets/rustacean-orig-noshadow.png") 12 | .await? // await server response 13 | .error_for_status()?; // generate an error if server didn't respond OK 14 | 15 | // Create an output file into which we will save ferris. 16 | let mut outfile = tokio::fs::File::create("ferris.png").await?; 17 | 18 | // Do an asynchronous, buffered copy of the download to the output file. 19 | // 20 | // Note that in some sense this is a workaround for being unable to use 21 | // tokio::io::copy as in the reqwest-tokio-compat example, but on the other 22 | // hand this method has no performance penalty and can actually be 23 | // preferable in some cases because it gives us more control. 24 | while let Some(chunk) = download.chunk().await? { 25 | outfile.write(&chunk).await?; 26 | } 27 | 28 | // Must flush tokio::io::BufWriter manually. 29 | // It will *not* flush itself automatically when dropped. 30 | outfile.flush().await?; 31 | 32 | Ok(()) 33 | } 34 | -------------------------------------------------------------------------------- /util/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "util" 3 | description = "Common utilities shared by the other examples." 4 | version = "0.1.0" 5 | authors = ["Benjamin Kay "] 6 | edition = "2018" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | -------------------------------------------------------------------------------- /util/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Custom types for error handling. 2 | 3 | /// Type-erased error that can be moved between threads. Example: 4 | /// 5 | /// ```ignore 6 | /// fn my_func() -> Result<(), BoxError> { 7 | /// do_something_fallible()?; 8 | /// Ok(()) 9 | /// } 10 | /// ``` 11 | /// 12 | /// See https://github.com/rust-lang/rfcs/pull/2820 13 | pub type BoxError = std::boxed::Box; 18 | 19 | /// Generic error type that stores a message `what` and optionally wraps another 20 | /// causative error `source`. Example: 21 | /// 22 | /// ```ignore 23 | /// fn my_func() -> Result<(), BoxError> { 24 | /// match do_something_fallible() { 25 | /// Ok(_) => (), 26 | /// Err(e) => return Err(Error{ 27 | /// what: "There was a problem doing something.".to_string(), 28 | /// source: Some(e), 29 | /// }.into()), 30 | /// }; 31 | /// 32 | /// if did_it_work() == false { 33 | /// return Err(Error{ 34 | /// what: "It didn't work.".to_string(), 35 | /// source: None, 36 | /// }.into()); 37 | /// } 38 | /// 39 | /// Ok(()) 40 | /// } 41 | /// ``` 42 | #[derive(Debug)] 43 | pub struct Error { 44 | /// What went wrong? 45 | pub what: String, 46 | /// What was the source/cause of this error, if any? 47 | pub source: Option 48 | } 49 | impl std::fmt::Display for Error { 50 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 51 | write!(f, "{}", self.what)?; 52 | if let Some(error) = &self.source { 53 | write!(f, "\nCaused by: {}", error)?; 54 | } 55 | Ok(()) 56 | } 57 | } 58 | impl std::error::Error for Error { 59 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 60 | match &self.source { 61 | Some(error) => Some(error.as_ref()), 62 | None => None 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /util/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Crate containing common utilities shared by the other examples. 2 | 3 | mod error; 4 | /// Type-erased error that can be moved between threads. 5 | pub use error::BoxError; 6 | /// Generic error type that stores a message and can wrap other errors. 7 | pub use error::Error; 8 | --------------------------------------------------------------------------------