├── .github ├── FUNDING.yml └── workflows │ └── build.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── EDITOR_GUIDELINES.md ├── LICENSE ├── README.md ├── content ├── issue-1 │ ├── 2022-review-the-adoption-of-rust-in-business-zh.md │ ├── 2022-review-the-adoption-of-rust-in-business.md │ ├── announcing-zh.md │ ├── announcing.md │ ├── contribute-to-rustc.md │ ├── hidden-control-flow-zh.md │ ├── hidden-control-flow.md │ ├── intro.md │ ├── minilsm.md │ ├── rust-for-linux-brief-introduction.md │ ├── vecdeque-resize-optimization.md │ ├── weihanglo.md │ └── zine.toml ├── issue-2 │ ├── applicative-parsing.md │ ├── demystifying-rust-lifetime-with-chatgpt.md │ ├── how-i-designed-the-api-for-backon-a-user-friendly-retry-crate.md │ ├── intro.md │ ├── optimize-binary-search.md │ ├── optimize-rust-comptime-zh.md │ ├── optimize-rust-comptime.md │ ├── sunli.md │ ├── write-a-sql-optimizer-using-egg-zh.md │ ├── write-a-sql-optimizer-using-egg.md │ └── zine.toml ├── issue-3 │ ├── bridging-async-and-sync-in-rust-zh.md │ ├── bridging-async-and-sync-in-rust.md │ ├── how-rse-index-top-20k-crates.md │ ├── intro.md │ ├── is-zig-safer-than-unsafe-rust.md │ ├── javascript-compiler.md │ ├── optimizing-compilation-for-databend-zh.md │ ├── optimizing-compilation-for-databend.md │ ├── robjtede.md │ ├── task-stats-alloc.md │ ├── understand-unsafe-rust.md │ └── zine.toml └── issue-4 │ ├── a-brief-discussion-on-invariants-in-rust-zh.md │ ├── a-brief-discussion-on-invariants-in-rust.md │ ├── how-tokio-schedule-tasks-zh.md │ ├── how-tokio-schedule-tasks.md │ ├── jhpratt.md │ ├── show.md │ └── zine.toml ├── footer.json ├── pages ├── about.md ├── contribution.md ├── faq.md └── sponsor.md ├── sponsors.json ├── static ├── avatar.png ├── avatar │ ├── chatgpt.svg │ ├── databend.svg │ ├── folyd.jpeg │ ├── jhpratt.png │ ├── m-ou-se.jpeg │ ├── malagant.png │ ├── psiace.jpg │ ├── robjtede.jpeg │ ├── skyzh.jpeg │ ├── sunli.jpeg │ ├── wangrunji.jpeg │ ├── weihanglo.png │ ├── xuanwo.jpeg │ └── xxchan.png ├── bg.jpeg ├── discord.svg ├── favicon.ico ├── favicon │ ├── 16x16.png │ ├── 32x32.png │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── apple-touch-icon.png │ └── site.webmanifest ├── ferris │ ├── 1.jpeg │ ├── 2.jpeg │ ├── 3.jpg │ ├── 4.jpeg │ ├── rustacean-flat-happy.svg │ └── sunshine.png ├── font.css ├── footer.css ├── github.svg ├── issue-2 │ ├── bs152.png │ ├── egg │ │ ├── alphago.png │ │ ├── constant-analysis.png │ │ ├── constant-folding.png │ │ ├── egg-official.png │ │ ├── egraph.png │ │ ├── join-reordering.png │ │ ├── join-swap.png │ │ ├── pushdown-filter.png │ │ └── rewrite.png │ ├── instruction-pipeline.png │ ├── opt1.png │ └── opt2.png ├── issue-3 │ ├── optimizing-compilation-for-databend │ │ ├── 1.png │ │ ├── 10.png │ │ ├── 12.png │ │ ├── 13.png │ │ ├── 14.png │ │ ├── 2.png │ │ ├── 3.png │ │ ├── 4.png │ │ ├── 5.png │ │ ├── 6.png │ │ ├── 8.png │ │ └── 9.png │ ├── robjtede.jpeg │ ├── rust-search-extension.gif │ ├── task-stats-alloc │ │ └── scope-meta.png │ └── update-index.png ├── issue-4 │ ├── ceresdb-task1.png │ ├── ceresdb-task2.png │ ├── cpp-loop-ub.jpg │ └── jacob-in-training.jpg ├── theme.css └── twitter.svg ├── tailwind.config.js ├── templates ├── article-extend.html ├── footer.html └── head.html ├── zine-data.json └── zine.toml /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # github: rustmagazine 2 | open_collective: rustmagazine 3 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build magazine 2 | 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the "main" branch 5 | push: 6 | branches: [ "main" ] 7 | 8 | # Allows you to run this workflow manually from the Actions tab 9 | workflow_dispatch: 10 | 11 | env: 12 | ZINE_VERSION: v0.16.0 13 | 14 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 15 | permissions: 16 | contents: read 17 | pages: write 18 | id-token: write 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | 24 | # Steps represent a sequence of tasks that will be executed as part of the job 25 | steps: 26 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 27 | - uses: actions/checkout@v3 28 | # - uses: dtolnay/rust-toolchain@stable 29 | # - name: Install zine 30 | # run: cargo install zine 31 | - name: Build magazine 32 | run: | 33 | wget https://github.com/zineland/zine/releases/download/$ZINE_VERSION/zine-x86_64-unknown-linux-gnu.tar.gz -O zine.tar.gz 34 | tar xvf zine.tar.gz 35 | ./zine build 36 | - name: Upload artifact 37 | uses: actions/upload-pages-artifact@v1 38 | with: 39 | path: ./build 40 | 41 | deploy: 42 | environment: 43 | name: github-pages 44 | url: ${{ steps.deployment.outputs.page_url }} 45 | runs-on: ubuntu-latest 46 | needs: build 47 | steps: 48 | - name: Deploy to GitHub Pages 49 | id: deployment 50 | uses: actions/deploy-pages@v1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | /build 12 | 13 | .DS_Store -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | All contributors are expected to follow the [Rust Community Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct). -------------------------------------------------------------------------------- /EDITOR_GUIDELINES.md: -------------------------------------------------------------------------------- 1 | # Editor Guidelines for Rust Magazine 2 | 3 | As an editor for Rust Magazine, you play a critical role in shaping the content and direction of this publication. The goal of this document is to provide you with a clear understanding of the principles, responsibilities, and expectations of your role, as well as the procedures for joining or exiting the editorial team. 4 | 5 | ## Principle 6 | 7 | The Rust Magazine editorial team is committed to the following principles: 8 | 9 | - Providing accurate, informative, and engaging content that serves the Rust community 10 | - Encouraging diversity and inclusiveness in our content and authors 11 | - Ensuring that all content is of high quality and adheres to editorial standards 12 | - Promoting the growth and advancement of the Rust programming language 13 | 14 | ## Responsibilities 15 | 16 | As an editor for Rust Magazine, your responsibilities will include: 17 | 18 | - Call for articles from the community for each issue 19 | - Writing articles for the issue if you have an idea 20 | - Reviewing and editing articles submitted by authors 21 | - Assisting authors with improving the quality and clarity of their content 22 | - Ensuring as far as reasonably possible that all articles are accurate, informative, and engaging 23 | - Collaborating with other editors and the Rust Magazine team to plan and execute editorial initiatives 24 | - Promoting Rust Magazine and its content on social media and other channels (prefer) 25 | - Staying up-to-date with the Rust community (prefer) 26 | 27 | ## Joining 28 | 29 | If you are interested in joining the Rust Magazine editorial team, please fill this [Google form](https://xxx) with the following information: 30 | 31 | - Your name and brief background in Rust programming 32 | - Your experience and qualifications as an editor 33 | - Your availability and time commitment (optional) 34 | 35 | ## Exit 36 | 37 | In the event that you need to step down from your role as an editor for Rust Magazine, we understand that life circumstances can change. We kindly request that you inform us of your intentions by sending a PR to move your profile into the alumni section. After being inactive for more than **three months** as the editor, we will change you to the **Reviewer** role. 38 | 39 | We are grateful for your contributions to Rust Magazine and would like to take this opportunity to thank you for your time and dedication to the publication. We wish you all the best in your future endeavors and hope that you will continue to be a part of the Rust community. 40 | 41 | ## How to maintain Rust Magazine 42 | 43 | ### Zine 44 | 45 | Rust Magazine is powered by [zine], a user-friendly static site generator designed specifically for magazines. Familiarity with zine is a valuable asset in maintaining the magazine as it offers built-in code blocks, such as the `urlpreview` to display an OGP preview of a URL, and `quote` to highlight someone's sentence. This is a good use case for an interview article. The repository and accompanying [documentation](https://zineland.github.io/) are great resources for getting started with zine. 46 | 47 | ### Tracking issue 48 | 49 | For each magazine issue, it is recommended to create a dedicated GitHub tracking issue, such as [#4]. This gives us a birds-eye view of the progress made in the current issue, and it can be closed after the latest issue has been published. 50 | 51 | ### Topic 52 | 53 | Each article can have zero, one, or several topics. Each topic should be declared in the root `zine.toml` in advance. As editors, we should provide a description of the topic and help the authors categorize their articles. 54 | 55 | ### Article 56 | 57 | - **path** 58 | 59 | We prefer to maintain control over article paths and do not generally allow authors to set their own `path` in the article metadata. This helps to keep the article paths organized under the issue slug. 60 | 61 | ```toml 62 | [[article]] 63 | path = "/cool-article" 64 | file = "cool-article.md" 65 | ... 66 | ``` 67 | 68 | For example, the article above will render to `https://rustmagazine.org/cool-article` instead of `https://rustmagazine.org/issue-1/cool-article`. We should request the author to remove the `path` property. 69 | 70 | In some significant circumstances, such as the announcing article: `https://rustmagazine.org/announcing/`, customizing the path may be permitted. 71 | 72 | ```toml 73 | [[article]] 74 | path = "/announcing" 75 | file = "announcing.md" 76 | title = "Announcing Rust Magazine" 77 | author = "rust-magazine" 78 | topic = ["announcement"] 79 | ... 80 | ``` 81 | 82 | - **canonical** 83 | 84 | When articles are reposted from their original source, it is important to include a `canonical` property to help prevent duplicate content issues for search engines. 85 | 86 | ```toml 87 | [[article]] 88 | file = "hidden-control-flow.md" 89 | title = "The Hidden Control Flow — Some Insights on an Async Cancellation Problem in Rust" 90 | canonical = "https://greptime.com/blogs/2023-01-12-hidden-control-flow.html" 91 | ... 92 | ``` 93 | 94 | > For more information, please refer to the [Canonical link element 95 | > ](https://en.wikipedia.org/wiki/Canonical_link_element) on Wikipedia. 96 | 97 | ### Issue publishing 98 | 99 | Before publishing an issue, all articles should have `publish = false` set in their metadata. A dedicated PR can be submitted to publish all articles at once, as seen in [#31]. After the issue has been published, a git tag (such as the [`issue-1` tag]) should be created for the last commit of the issue. 100 | 101 | > Pro Tip: It's not uncommon to receive typo-fixing PRs a few days after an issue has been published, so waiting a day or two before creating a tag may be a good strategy to catch any last minute updates. 102 | 103 | [zine]: https://github.com/zineland/zine 104 | [#4]: https://github.com/RustMagazine/rustmagazine/issue/4/ 105 | [#31]: https://github.com/RustMagazine/rustmagazine/pull/31/ 106 | [`issue-1` tag]: https://github.com/RustMagazine/rustmagazine/releases/tag/issue-1 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Rust Magazine 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rust Magazine 2 | 3 | [![license-mit](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/RustMagazine/rustmagazine/blob/main/LICENSE) 4 | [![Discord](https://img.shields.io/discord/1078667135309594654?label=chat&logo=discord)](https://discord.gg/DdwgBuReJe) 5 | 6 | A publication dedicated to the Rust programming language. 7 | 8 | https://rustmagazine.org 9 | 10 | ## Sponsor us 11 | 12 | [![](https://opencollective.com/rustmagazine/tiers/reader.svg?avatarHeight=120)](https://opencollective.com/rustmagazine) 13 | 14 | ## License 15 | 16 | - The source code of the magazine is licensed under the [MIT LICENSE](./LICENSE). 17 | 18 | - The content of the magazine is licensed under the [CC BY-NC-SA 4.0 LICENSE](https://creativecommons.org/licenses/by-nc-sa/4.0/) by default unless otherwise specified. 19 | -------------------------------------------------------------------------------- /content/issue-1/announcing-zh.md: -------------------------------------------------------------------------------- 1 | 2 | ![](/static/bg.jpeg) 3 | 4 | 我们非常兴奋地宣布推出 Rust 杂志——一本专门介绍 Rust 编程语言的出版物。 5 | 6 | 在 Rust 杂志，我们的目标是为 Rust 社区提供一个平台，以分享他们在使用该语言方面的知识、经验和见解。 我们的杂志将刊登来自专家和爱好者的文章，主题广泛，包括最佳实践、新功能和 Rust 的实际应用示例。 7 | 8 | 除了文章，Rust 杂志还将包括教程、访谈和其他内容，以帮助读者了解 Rust 世界的最新发展。 我们还将重点介绍 Rust 生态系统中的著名项目和社区。 9 | 10 | 我们很高兴推出 Rust 杂志，并期待为 Rust 社区提供宝贵的资源。 无论您是经验丰富的 Rustacean 还是刚刚开始使用这门语言，我们都希望您能加入我们这一激动人心的旅程。 -------------------------------------------------------------------------------- /content/issue-1/announcing.md: -------------------------------------------------------------------------------- 1 | 2 | ![](/static/bg.jpeg) 3 | 4 | It is with great excitement that we announce the launch of Rust magazine – a publication dedicated to the Rust programming language. 5 | 6 | At Rust magazine, our goal is to provide a platform for the Rust community to share their knowledge, experiences, and insights on using the language. Our magazine will feature articles from experts and enthusiasts on a wide range of topics, including best practices, new features, and real-world examples of Rust in action. 7 | 8 | In addition to articles, Rust magazine will also include tutorials, interviews, and other content to help readers stay up-to-date on the latest developments in the Rust world. We will also be highlighting notable projects and communities within the Rust ecosystem. 9 | 10 | We are thrilled to be launching Rust magazine and look forward to providing a valuable resource for the Rust community. Whether you are a seasoned Rustacean or just getting started with the language, we hope you will join us on this exciting journey. -------------------------------------------------------------------------------- /content/issue-1/contribute-to-rustc.md: -------------------------------------------------------------------------------- 1 | 2 | I spent a lot of my spare time contributing to the Rust compiler in the second half of 2022, and I learned a lot from such a friendly and vibrant community, not only about Rust, but about the core skills of software engineering and, most importantly, I got a lot of fun from programming in Rust. 3 | 4 | I finished about [70+ PRs](https://github.com/rust-lang/rust/pulls?q=is%3Apr+author%3Achenyukang+is%3Aclosed) in this period of time, and my recent milestone was receiving a [Project Grant from the Rust Foundation](https://foundation.rust-lang.org/news/community-grants-program-awards-announcement-introducing-our-latest-project-grantees/). 5 | 6 | Here, I'd like to share my Rust experience with you. 7 | 8 | # My Background and How I started 9 | 10 | I have more than ten years of programming experience, the book [Essentials of Programming Languages](https://github.com/chenyukang/eopl) triggered my interests in programming languages and implementation. 11 | 12 | In 2014, I found Rust in GitHub, in that period of time, I was interested in OCaml, and Rust compiler was initially implemented in OCaml. 13 | 14 | I spent some time to play with it, and then I begin to write some simple programs in Rust. I remember my first impression of Rust was that the language was too complex, with several kinds of pointers! 15 | 16 | From 2015 to 2020, I didn't spend much time on Rust, but I didn't completely ignore it either. Because I was using Ruby in my daily work and I like OCaml, the syntax of Rust is familiar to me. I've used Rust for some small personal hobby projects and practiced Rust with [exercism](https://exercism.org/). 17 | 18 | In 2020, when I started another personal project called [gomoku](https://github.com/chenyukang/gomoku), I just realized that Rust's toolchain had become so great, and programming in Rust was a joy, and I only need to focus on the coding logic, the dependency management, testing, and documentation tools were all ready for me. From that time, I began to contribute to Rust projects in WebAssembly and containers, such as [wasmerio/wasmer](https://github.com/wasmerio/wasmer),[containers/youki](https://github.com/containers/youki),[second-state/dapr-wasm](https://github.com/second-state/dapr-wasm). 19 | 20 | My [first PR](https://github.com/rust-lang/rust/pull/88493) for Rust compiler was in September 2021, this was a compiler issue that I found in development, some diagnostics from compiler may duplicated. I was curious about the root cause, so I clone the compiler's code and begin to debug. In fact, I underestimated the difficulty of getting started, this problem took me almost a week of my spare time, I spent a lot of time reading documents, I also tried to use gdb to follow some parts of code. 21 | 22 | Finishing my first PR was fulfilling, but then I went on to other things. It wasn't until August 2022 that I inadvertently clicked on the link [Rust is Beautiful #100000](https://github.com/rust-lang/rust/issues/100000), I stopped by to look at a few issues and found some issues related to the parser and error messages that seemed easy to solved, so I started contributing to the compiler again. The next two weeks were Microsoft hackthon, where we could learn new things that interested me, so I spent most of my time on Rust, and now I contribute to regularly. 23 | 24 | # You don't need to be a compiler expert 25 | 26 | Contributing to a compiler is not easy, but it is not as difficult as many people think. Rust was created and developed on Github, and the design documents and all code are public, in a transparent manner. Rust was not designed by a small elite group they call a standardization committee, instead Rust was built by people from different backgrounds, and everyone can submit a PR or RFC to change it. 27 | 28 | Furthermore, Rust has a very active open-source community, a group of friendly Rustaceans, so in my experience, contributing to Rust is not as difficult as other widely used programming language. 29 | 30 | Of course, it's harder to change the semantics of the language, but there are enough engineering challenges within the compiler itself for beginners. 31 | 32 | You don't have to be an expert in Rust, as long as you can understand most of Rust's syntax and have enough patience, you can have a try to read the source code of Rust compiler, or try to make any change which may make it better. 33 | 34 | However, knowing some aspects about compilers will help you a lot, and if you are interested in compilers, these books and projects will help you: 35 | 36 | - [Crafting Interpreters](http://craftinginterpreters.com/) 37 | - [Essentials of Programming Languages](https://eopl3.com/) 38 | - [9cc: A Small C Compiler](https://github.com/rui314/9cc) 39 | - [r9cc: Toy C compiler](https://github.com/utam0k/r9cc) 40 | 41 | # The Workflow 42 | 43 | The Rust compiler repository contains tens of thousands of test cases. Whether you are fixing bugs or working on some new feature, you'd better prepare some minimal test cases first, then keep modifying the code, compiling, and testing, repeat the iteration until all the cases pass. 44 | 45 | ![](https://catcoding.me/images/ob_pasted-image-20230117125646.png) 46 | 47 | The biggest disadvantage of compiled languages is that we lose the fun of interactive programming, especially with Rust, which is a particularly time-consuming compiled language. The good thing is that after the coding phase is done, we don't have any more errors. 48 | 49 | To reduce the number of compilations, my solution is to add more logging. 50 | 51 | The most time-consuming part of the process is probably the code review phase, PR may need several weeks for reviewing. The community is currently discussing how to solve this problem [Did we start reviewing PRs slower?](https://internals.rust-lang.org/t/did-we-start-reviewing-prs-slower/18033). 52 | 53 | You need to spend a lot of time communicating and discussing with others, so writing skills are important. 54 | 55 | # How to begin 56 | 57 | Start with the [Guide to Rustc Development](https://rustc-dev-guide.rust-lang.org/getting-started.html), this document may not be up to date, but it is indispensable to get an overview of the various topics of the Rust compiler. 58 | 59 | At first, you can find some relatively simple issues to solve from labels like [E-mentor](https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AE-mentor), [E-easy](https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AE-easy), [E-help-wanted](https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AE-help-wanted). But if you can't find the right one, you can also find random problems that you can understand and try to solve them. 60 | 61 | Issues labeled with [A-diagnostics](https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AA-diagnostics) are mostly about improving or fixing error diagnostics from compiler, which may suitable for beginners. A large amount of code exists in the Rust compiler to provide developers with the most useful and friendly diagnostics, Rust's focus and investment in this area is more than most programming languages, as [estebank](https://github.com/estebank) writes: 62 | 63 | > We spent decades trying to invent a sufficiently smart compiler when we should have been inventing a sufficiently empathetic one. 64 | 65 | To solve such kind of issue, we need to understand the AST, infer the user's coding intent, and dig all the useful things from the type system, as well as the various data structures in the compiler, to provide the best possible help, even pretty error messages. We can find the code corresponding to the error message by searching for keywords, but not all of these issues are easy to solve. 66 | 67 | Some [ICE](https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AI-ICE) issues are also relatively easy to solve, such as some corner cases that have not been covered, usually there is a simple and direct solution, but choosing the best one from various fixing solutions is also an art. 68 | 69 | An important viewpoint shift in doing compiler development is to move from being a user of a language to being a contributor. 70 | 71 | I saw this on Twitter the other day, and my first question was why the compiler would have two suggestions at the same time, the first one apparently only for cases that care about returning result: 72 | 73 | ![](https://catcoding.me/images/ob_pasted-image-20230117005319.png) 74 | 75 | So I created an issue to track it and subsequently landed a PR to fix it [Properly handle postfix inc/dec](https://github.com/rust-lang/rust/pull/104875). 76 | 77 | I also try to fix those issues that I encounter during development. For example, I found that if I missed a '}' in typing, the compiler may not be able to point out the delimiter mismatch in right span, and if the source file is long, it may report a lot of unrelated errors, so I made another [PR to fix it](https://github.com/rust-lang/rust/pull/104012). 78 | 79 | # Some random tips 80 | 81 | Rust has a not-quite-complete [Reference](https://doc.rust-lang.org/reference/), which is helpful if you can't understand some part of the code, probably because you don't know the terminology. 82 | 83 | In the development of rustc, `x.py` is a frequently used command, I wrote my requently used commands as a [justfile configuration file](https://github.com/chenyukang/share/blob/main/justfile), so it will be easier to run testing, check logs, rebase, etc. 84 | 85 | Logging and reading code is more useful than `gdb` during debugging. By reading the code, printing logs at key steps, and verifying our guesses with runtime logs, we can get the flow and structure of the code in our mind, whereas it is easy to get lost in the specifics with gdb tracing. 86 | 87 | Reading Rust code is easy with `rust-analyzer`, and the `show call hierarchy` is particularly useful for understanding function calls. 88 | 89 | If you run into trouble in development, you can post a topic at [t-compiler/help](https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp). `zulip` is a discussion tool used by Rust compiler developers, and the Rust experts are friendly for newcomers. 90 | 91 | Another great way to learn is to review other's PRs. It's okay if you can't understand every change at first, but you can try to grasp the general idea. If I'm interested in an issue but don't have time to work on it, or I would like to learn something in that topic, I'll subscribe to notifications so if the issue is closed I'll get an Email and I'll probably go to have a look at the PR. 92 | 93 | # Summary 94 | 95 | Rust is growing rapidly in Infra, WebAssembly and Web development, embedded, games, security, and more. 96 | 97 | I'm not an expert on Rust, but I've found that contributing to the Rust compiler is the best way to learn Rust. This may not be for everyone, but maybe you can try it too. 98 | 99 | And contributing to Rust is not limited to programming, reporting issues, improving the documentation and translations, and joining discussions are all community contributions. 100 | 101 | Hope my sharing can help you, Happy Coding! 🦀 102 | 103 | -------------------------------------------------------------------------------- /content/issue-1/hidden-control-flow-zh.md: -------------------------------------------------------------------------------- 1 | 本文主要介绍了一个在 [GreptimeDB](https://github.com/GreptimeTeam/greptimedb) 中遇到的一个关于异步取消 (async cancellation) 的“奇怪”[问题](https://github.com/GreptimeTeam/greptimedb/issues/350)。 2 | 3 | # The Issue 4 | 5 | 针对这个问题，我们首先描述一个简化的场景：在一个长时间运行的测试中存在元信息损坏的问题，有一个应该单调递增的序列号出现了重复。 6 | 7 | 序列号的更新逻辑非常简单：从一个原子变量中读取当前值，然后通过异步 I/O 方法  `persist_number()`  将新值写入文件里，最后更新这个原子变量。整个流程都是串行化的（`file`  是一个独占引用）。 8 | 9 | ```rust 10 | async fn update_metadata(file: &mut File, counter: AtomicU64) -> Result<()> { 11 | let next_number = counter.load(Ordering::Relaxed) + 1; 12 | persist_number(file, next_number).await?; 13 | counter.fetch_add(1, Ordering::Relaxed); 14 | } 15 | ``` 16 | 17 | 由于一些原因，我们在这里使用了 `load` 函数而非 `fetch_add`（虽然单就这里来说可以用 `fetch_add`，并且用了还不会引发这次的问题 🤪）。当这个更新流程在中间出现错误时，我们不希望更新内存中的计数器。我们清楚如果 `persist_number()` 写入文件时失败就能够从 ? 提前返回，并且会提早结束执行来传播错误，所以编码的时候会注意这些问题。 18 | 19 | 但是到了 `.await` 这里事情就变得奇妙了起来，因为 async cancellation 带来了一个隐藏的控制流。 20 | 21 | # Async Cancellation 22 | 23 | ## async task and runtime 24 | 25 | 如果这时候你已经猜到了到底是什么引发了这个问题，可以跳过这一章节。如果没有，就让我从一些伪代码开始解释在 await point 那里到底发生了什么，以及 runtime 是如何参与其中的。 26 | 27 | ### poll_future 28 | 29 | 首先是 `poll_future` ，对应到 `Future` 的 [`poll`](https://doc.rust-lang.org/std/future/trait.Future.html#tymethod.poll) 方法。我们写的异步方法都会被转化成类似这样子的一个匿名的 `Future` 实现。 30 | 31 | ```rust 32 | fn poll_future() -> FutureOutput { 33 | match status_of_the_task { 34 | Ready(output) => { 35 | // the task is finished, and we have it output. 36 | // some logic 37 | return our_output; 38 | }, 39 | Pending => { 40 | // it is not ready, we don't have the output. 41 | // thus we cannot make progress and need to wait 42 | return Pending; 43 | } 44 | } 45 | } 46 | ``` 47 | 48 | `async` 块通常包含其他的异步方法，比如 `update_metadata` 和 `persist_number`。这里把 `persist_number` 称为 `update_metadata` 的子异步任务。每个 `.await` 都会被展开成类似 `poll_future` 的东西，等待子任务的结果并继续执行。在这个例子中就是等待 `persist_number` 的结果返回 `Ready` 再更新计数器，否则不更新。 49 | 50 | ### runtime 51 | 52 | 第二段伪代码是一个简化的 runtime，它负责轮询 (poll) 异步任务直到它们完成（考虑到接下来的文章内容，“直到……完成”这种表述并不适合于所有情况）。在 GreptimeDB 中我们使用 [`tokio`](https://docs.rs/tokio/latest/tokio/) 作为 runtime。现在的异步 runtime 可能有很多特性和功能，其中最基础的就是轮询这些任务。 53 | 54 | ```rust 55 | fn runtime(&self) { 56 | loop { 57 | let future_tasks: Vec = self.get_tasks(); 58 | for task in tasks { 59 | match task.poll_future(){ 60 | Ready(output) => { 61 | // this task is finished. wake it with the result 62 | task.wake(output); 63 | }, 64 | Pending => { 65 | // this task needs some time to run. poll it later 66 | self.poll_later(task); 67 | } 68 | } 69 | } 70 | } 71 | } 72 | ``` 73 | 74 | 通过结合上述两个简化的 future 和 runtime 模型，我们得到如下这个循环（真实的 runtime 非常复杂，这里为了内容集中省略了很多）。 75 | 76 | ```rust 77 | fn run() -> Output { 78 | loop { 79 | if let Ready(result) = task.poll() { 80 | return result; 81 | } 82 | } 83 | } 84 | ``` 85 | 86 | 需要强调的是，每个 `.await` 都代表着一个或者多个函数调用 (调用到 `poll()` 或者说是 `poll_future()`)。这就是标题中“隐藏的控制流”，以及 cancellation 发生的地方。 87 | 88 | 我们再看一段简单的程序来探测 runtime 的行为（可以直接在 [playground](https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=40220605392e951e833a0b45719ed1e1) 里面运行这段代码)： 89 | 90 | ```rust 91 | use tokio::time::{sleep, Duration, timeout}; 92 | 93 | #[tokio::main] 94 | async fn main() { 95 | let f = async { 96 | print(1).await; 97 | println!("1 is done"); 98 | print(2).await; 99 | println!("2 is done"); 100 | print(3).await; 101 | println!("3 is done"); 102 | }; 103 | 104 | if let Err(_) = timeout(Duration::from_millis(150), f).await { 105 | println!("timeout"); 106 | } 107 | 108 | sleep(Duration::from_millis(300)).await; 109 | println!("exit") 110 | } 111 | 112 | async fn print(val: u32) { 113 | sleep(Duration::from_millis(100)).await; 114 | println!("val is {}", val); 115 | } 116 | ``` 117 | 118 | 只要花几分钟时间猜测一下上方代码的输出结果，如果和下面的一致，相信你已经知道问题出在哪里。 119 | 120 | ``` 121 | val is 1 122 | 1 is done 123 | timeout 124 | exitprint(2).await 125 | ``` 126 | 127 | `println!("1 is done");` 之后的语句都因为超时而被 runtime 取消执行了。 128 | 129 | 这个问题其中的原理并不复杂，但是（对我来说）能够定位到它并不轻易。在把其他问题都排除掉之后我知道问题就发生在这里，在这个 `.await` 上。也许是太多次成功的异步函数调用麻痹了注意，亦或是我的心智模型中没有把这两点联系起来，联想到这点着实费了一番心思。 130 | 131 | ## Cancellation 132 | 133 | 目前为止的内容是问题复盘的标准流程，接下来，让我们来展开讨论一下 cancellation，它是与 runtime 的行为相关的。 134 | 135 | 虽然 Rust 中的很多 runtime 都有类似的行为，但是这不是一个必须的特性，比如这个自己写的 [runtime](https://github.com/waynexia/texn) 就不支持 cancellation。因为问题发生在 tokio 上，因此这里会以它为例，而其他的 runtime 也是类似的。在 tokio 中，可以使用 [`JoinHandle::abort()`](https://docs.rs/tokio/latest/tokio/task/struct.JoinHandle.html#method.abort) 来取消一个 task。task 结构中有一个“cancel marker bit”来跟踪一个任务是否被取消了。如果它发现一个 task 被取消了，就会停止执行这个 task。 136 | 137 | # Current Solution 138 | 139 | ## Explicit Detach 140 | 141 | 现在是否有手段能防止 task 被取消呢？在 tokio 中我们可以通过 drop `JoinHandle` 来 detach 一个任务到后台。一个 detached task 意味着没有前台的 handle 来控制这个任务，从某种意义上来说也就使得其他人不能在外面套一层 `timeout` 或 `select`，从而间接地使它不会被取消执行。并且开头提到的问题就是通过这种方式解决的。 142 | 143 | > JoinHandle **detaches** the associated task when it is dropped, which means that there is no longer any handle to the task, and no way to `join` on it. 144 | 145 | 不过虽然有办法能够实现这个功能，是否像 [`glommio`](https://docs.rs/glommio/0.7.0/glommio/struct.Task.html#method.detach) 一样有一个显式的 `detach` 方法，类似一个不返回 `JoinHandle` 的 `spawn` 方法会更好。但这些都是琐碎的事情，一个 runtime 通常不会完全没有理由就取消一个 task，并且在大多数情况下都是出于用户的要求，只不过有时候可能没有注意到，就像 `select` 中的那些“未选中的分支”或者 `tonic` 中请求处理的逻辑那样。所以如果我们确定一个 task 是不能被取消的话，显式地 `detach` 可能能预防某些悲剧的发生。 146 | 147 | 目前为止所有问题都清晰了，让我们开始修复这个 bug 吧！ 148 | 149 | 首先，为什么我们的 future 会被取消呢？通过函数调用链路很容易就能发现整个处理过程都是在 `tonic` 的请求执行逻辑中就地执行的，而对于一个网络请求来说有一个超时行为是很常见的。解决方案也很简单，就是将服务器处理逻辑提交到另一个 runtime 中执行，从而防止它被取消。只需要[几行代码](https://github.com/GreptimeTeam/greptimedb/pull/376/files#diff-9756dcef86f5ba1d60e01e41bf73c65f72039f9aaa057ffd03f3fc2f7dadfbd0R46-R54)就能完成。 150 | 151 | ```diff 152 | impl BatchHandler { 153 | - for db_req in batch_req.databases { 154 | - for obj_expr in db_req.exprs { 155 | - let object_resp = self.query_handler.do_query(obj_expr).await?; 156 | - db_resp.results.push(object_resp); 157 | + let (tx, rx) = oneshot::channel(); 158 | + let query_handler = self.query_handler.clone(); 159 | + let _ = self.runtime.spawn(async move { 160 | + // execute the request in another runtime to prevent its execution from being cancelled unexpectedly by tonic runtime. 161 | + let mut result = vec![]; 162 | + for db_req in batch_req.databases { 163 | + for obj_expr in db_req.exprs { 164 | + let object_resp = query_handler.do_query(obj_expr).await; 165 | + 166 | + result.push(object_resp); 167 | + } 168 | } 169 | ``` 170 | 171 | 这个问题到这里就修复完了，不过并不是从根本上解决 async cancellation 带来的 bug，而是采用间接手段去规避任务由于超时而被提前取消的问题，毕竟我们的这些异步逻辑还是需要被完整执行的。 172 | 173 | 但是这样的处理会放大另外一些问题，比如我们也无法提前取消掉对于已经不用执行或资源消耗特别大的任务，从而导致系统资源的浪费。这些是我们之后需要持续改进的地方。接下来会就这方面继续展开，从 async 生态的方面讨论有哪些可能能提升 async cancellation 的使用体验。 174 | 175 | # Runtime Behavior 176 | 177 | ## Marker Trait 178 | 179 | 首先，我们自然希望 runtime 不要无条件地取消我的 task，而是尝试通过类型系统来变得更友好，比如借助类似 `CancelSafe` 的 marker trait 。对于 cancellation safety 这个词，tokio 在它的[文档](https://docs.rs/tokio/latest/tokio/macro.select.html#cancellation-safety)中有提到： 180 | 181 | > To determine whether your own methods are cancellation safe, look for the location of uses of `.await` . This is because when an asynchronous method is cancelled, that always happens at an `.await`. If your function behaves correctly even if it is restarted while waiting at an `.await`, then it is cancellation safe. 182 | 183 | 简单来说就是用来描述一个 task 是否可以安全地被取消掉，这是 async task 的属性之一。tokio 维护了一个很长的列表，列出了哪些是安全的以及哪些是不安全的。看起来这和 [`UnwindSafe`](https://doc.rust-lang.org/std/panic/trait.UnwindSafe.html) 这个 marker trait 很像。两者都是描述“这种控制流程并不总是被预料到的”，并且“有可能导致一些微妙的 bug” 的这样一种属性。 184 | 185 | 如果有这样一个 `CancelSafe` 的 trait，我们就有途径可以告诉 runtime 我们的异步任务是否可以安全地被取消掉，同时也是一种方式让用户承诺 cancelling 这个控制流程是被仔细处理过的。如果发现没有实现这个 trait，那就意味着我们不希望这个 task 被取消掉，简单而清晰。以 `timeout`为例： 186 | 187 | ```rust 188 | /// The marker trait 189 | trait CancelSafe {} 190 | 191 | /// Only cancellable task can be timeout-ed 192 | pub fn timeout(duration: Duration, future: F) -> Timeout where 193 | F: Future + CancelSafe 194 | {} 195 | ``` 196 | 197 | ## Volunteer Cancel 198 | 199 | 另一个方式是让任务自愿地取消。就像 Kotlin 中的 [cooperative cancellation](https://kotlinlang.org/docs/cancellation-and-timeouts.html#cancellation-is-cooperative) 一样，它有一个 `isActive` 方法来检查一个 task 是否被取消掉。这只是一个检测方法，是否要取消完全取决于 task 本身。下面是 Kotlin 文档中的一个例子, cooperative cancellation 发生在第 5 行。这种方式把“隐藏的控制流程”放在了明面上，让我们能以一种更自然的方式来考虑和处理 cancellation，就像 `Option` 或 `Result` 一样。 200 | 201 | ```Kotlin 202 | val startTime = System.currentTimeMillis() 203 | val job = launch(Dispatchers.Default) { 204 | var nextPrintTime = startTime 205 | var i = 0 206 | while (isActive) { // cancellable computation loop 207 | // print a message twice a second 208 | if (System.currentTimeMillis() >= nextPrintTime) { 209 | println("job: I'm sleeping ${i++} ...") 210 | nextPrintTime += 500L 211 | } 212 | } 213 | } 214 | delay(1300L) // delay a bit 215 | println("main: I'm tired of waiting!") 216 | job.cancelAndJoin() // cancels the job and waits for its completion 217 | println("main: Now I can quit.") 218 | ``` 219 | 220 | 并且我认为这也不难实现，Tokio 现在已经有了 `Cancelled bit` 和 `CancellationToken`，只是看起来和期望的还有点不一样。最后还是需要 runtime 把 cancellation 的权利交给 task，否则情况可能没有什么大的不同。 221 | -------------------------------------------------------------------------------- /content/issue-1/intro.md: -------------------------------------------------------------------------------- 1 | 2 | # Preface 3 | 4 | Dear Rustaceans, 5 | 6 | We are thrilled to announce the first issue of Rust Magazine, a publication dedicated to exploring and promoting the Rust programming language. This issue is packed with a range of exciting articles from the community. 7 | 8 | In this issue, you'll find: 9 | 10 | - An overview of the new `VecDeque::resize()` optimization `@folyd` 11 | - A step-by-step tutorial on building a LSM storage engine in a week using Rust `@skyzh` 12 | - An in-depth interview with `@weihanglo`, a prominent contributor to the cargo and ecosystem `@folyd` 13 | - A personal account of one developer's journey contributing to the Rust compiler `@yukang` 14 | - A brief introduction to using Rust for Linux systems `@shijicheng` 15 | - A review of the adoption of Rust in the business world in 2022 `@HandongZhang` 16 | - Some Insights on an Async Cancellation Problem in Rust `@waynexia` 17 | 18 | We hope that this issue will inspire you to dive deeper into the world of Rust and provide valuable insights for both experienced Rust developers and those just getting started. Enjoy! 19 | 20 | Best regards, 21 | 22 | The Rust Magazine Team 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /content/issue-1/minilsm.md: -------------------------------------------------------------------------------- 1 | MiniLSM is a working-in-progress tutorial series for building a storage engine. Based on the experience of building 2 | [AgateDB](https://github.com/tikv/agatedb) and [RisingWave](https://github.com/risingwavelabs/risingwave), MiniLSM 3 | introduces you the basics of an LSM-tree storage engine with a step-by-step tutorial. 4 | 5 | The starter code is accessible on [GitHub](https://github.com/skyzh/mini-lsm), and the tutorial is available on 6 | [GitHub Pages](https://skyzh.github.io/mini-lsm/). 7 | 8 | ```urlpreview 9 | https://github.com/skyzh/mini-lsm 10 | ``` 11 | 12 | # What is LSM, and Why LSM? 13 | 14 | Log-structured merge tree is a data structure to maintain key-value pairs. This data structure is widely used in 15 | distributed database systems like [TiDB](https://www.pingcap.com) and [CockroachDB](https://www.cockroachlabs.com) as 16 | their underlying storage engine. [RocksDB](http://rocksdb.org), based on [LevelDB](https://github.com/google/leveldb), 17 | is an implementation of LSM-tree storage engine. It provides a wide range of key-value access functionalities and is 18 | used in a lot of production systems. 19 | 20 | LSM-tree storage engine usually views keys and values as byte array encoded by users. For example, TiDB encodes a row 21 | in the table [in the following way](https://docs.pingcap.com/tidb/dev/tidb-computing#mapping-table-data-to-key-value): 22 | 23 | ``` 24 | | row_id | col1 | col2 | col3 | 25 | | 0 | 1 | 2 | 3 | 26 | 27 | key: table1_record00000000 28 | value: encoded([0, 1, 2, 3]) 29 | ``` 30 | 31 | where the row is encoded into a byte string, so that LSM engine can easily process it. 32 | 33 | Generally speaking, LSM Tree is an append-friendly data structure. It is more intuitive to compare LSM to other 34 | key-value data structure like RB-Tree and B-Tree. For RB-Tree and B-Tree, all data operations are in-place. That is to 35 | say, when you update the value corresponding to the key, the value will be overwritten at its original memory or disk 36 | space. But writes in an LSM Tree, i.e., insertions, updates, deletions, are performed in an append-only way. We will 37 | never change data already written to disk. LSM-tree engine always writes changes to new files (which are called 38 | sorted-string tables, aka. SSTs) and then merge existing files (via compaction). 39 | 40 | This architectural design makes LSM tree easy to work with. 41 | 42 | 1. Data are immutable on persistent storage, which means that it is easier to offload the background tasks (compaction) 43 | to remote servers. It is also feasible to directly store and serve data from cloud-native storage systems like S3. 44 | 2. An LSM tree can balance between read, write and space amplification by changing the compaction algorithm. The data 45 | structure itself is super versatile and can be optimized for different workloads. 46 | 47 | In this tutorial, we will learn how to build an LSM-tree-based storage engine in the Rust programming language. 48 | 49 | # Overview of LSM 50 | 51 | An LSM storage engine generally contains 3 parts: 52 | 53 | 1. Write-ahead log to persist temporary data for recovery. 54 | 2. SSTs on the disk for maintaining a tree structure. 55 | 3. Mem-tables in memory for batching small writes. 56 | 57 | The storage engine generally provides the following interfaces: 58 | 59 | * `Put(key, value)`: store a key-value pair in the LSM tree. 60 | * `Delete(key)`: remove a key and its corresponding value. 61 | * `Get(key)`: get the value corresponding to a key. 62 | * `Scan(range)`: get a range of key-value pairs. 63 | 64 | To ensure persistence, 65 | 66 | * `Sync()`: ensure all the operations before `sync` are persisted to the disk. 67 | 68 | Some engines choose to combine `Put` and `Delete` into a single operation called `WriteBatch`, which accepts a batch 69 | of key value pairs. 70 | 71 | In this tutorial, we assume the LSM tree is using leveled compaction algorithm, which is commonly used in real-world 72 | systems. 73 | 74 | # Write Flow 75 | 76 | ![](https://skyzh.github.io/mini-lsm/figures/lsm-tutorial/00-lsm-write-flow.svg) 77 | 78 | The write flow of LSM contains 4 steps: 79 | 80 | 1. Write the key-value pair to write-ahead log, so that it can be recovered after the storage engine crashes. 81 | 2. Write the key-value pair to memtable. After (1) and (2) completes, we can notify the user that the write operation 82 | is completed. 83 | 3. When a memtable is full, we will flush it to the disk as an SST file in the background. 84 | 4. We will compact some files in some level into lower levels to maintain a good shape for the LSM tree, so that read 85 | amplification is low. 86 | 87 | # Read Flow 88 | 89 | ![](https://skyzh.github.io/mini-lsm/figures/lsm-tutorial/00-lsm-read-flow.svg) 90 | 91 | When we want to read a key, 92 | 93 | 1. We will first probe all the memtables from latest to oldest. 94 | 2. If the key is not found, we will then search the entire LSM tree containing SSTs to find the data. 95 | 96 | # Tutorial Overview 97 | 98 | ![](https://skyzh.github.io/mini-lsm/figures/lsm-tutorial/00-lsm-tutorial-overview.svg) 99 | 100 | ## Day 1 - Block Encoding 101 | 102 | Block is the minimum read unit in LSM. It is of 4KB size in general, similar database pages. In each block, we will 103 | store a sequence of sorted key value pairs. In day 1, we will explore how to use the `bytes` library to do manual data 104 | encoding. Also, you will learn how to write RocksDB-style iterator in Rust. 105 | 106 | **What you will implement:** 107 | 108 | ```rust 109 | impl BlockBuilder { 110 | /// Adds a key-value pair to the block. Returns false when the block is full. 111 | #[must_use] 112 | pub fn add(&mut self, key: &[u8], value: &[u8]) -> bool { 113 | unimplemented!() 114 | } 115 | 116 | /// Finalize the block. 117 | pub fn build(self) -> Block {} 118 | } 119 | 120 | impl BlockIterator { 121 | /// Returns the key of the current entry. 122 | pub fn key(&self) -> &[u8] {} 123 | 124 | /// Returns the value of the current entry. 125 | pub fn value(&self) -> &[u8] {} 126 | 127 | /// Move to the next key in the block. 128 | pub fn next(&mut self) {} 129 | 130 | /// Seek to the first key that >= `key`. 131 | pub fn seek_to_key(&mut self, key: &[u8]) {} 132 | } 133 | ``` 134 | 135 | ## Day 2 - Table Encoding 136 | 137 | SST (sorted string table) is composed of data blocks and index blocks stored on the disk. Usually, data blocks are 138 | lazily loaded -- they will not be loaded into the memory until a user requests it. Index blocks can also be loaded 139 | on-demand, but in this tutorial, we make simple assumptions that all SST index blocks (meta blocks) can fit in memory. 140 | Generally, an SST file is of 256MB size. In day 2, we will implement SST encoding and learn how to compose RocksDB-style 141 | iterators together in Rust. 142 | 143 | **What you will implement:** 144 | 145 | ```rust 146 | impl SsTableBuilder { 147 | /// Adds a key-value pair to SSTable 148 | pub fn add(&mut self, key: &[u8], value: &[u8]) {} 149 | 150 | /// Builds the SSTable and writes it to the given path. No need to actually write to disk until 151 | /// chapter 4 block cache. 152 | pub fn build(self, ...) -> Result {} 153 | } 154 | 155 | impl StorageIterator for SsTableIterator { 156 | fn value(&self) -> &[u8] {} 157 | 158 | fn key(&self) -> &[u8] {} 159 | 160 | fn is_valid(&self) -> bool {} 161 | 162 | fn next(&mut self) -> Result<()> {} 163 | } 164 | ``` 165 | 166 | ## Day 3 - Mem Table and Merge Iterators 167 | 168 | In this tutorial, we use [crossbeam-skiplist](https://docs.rs/crossbeam-skiplist) as the implementation of memtable. You 169 | will learn how to use self-referential struct in Rust to erase lifetime from a Rust-style iterator object, and therefore 170 | being easily composed with other RocksDB-style iterators in the system. 171 | 172 | Also, we will see how to implement high-performance merge iterators with correct error handling with Rust's `BinaryHeap`. 173 | We intentionally avoid using dynamic dispatch (i.e. `Box`) to achieve higher performance. You will need to 174 | implement two different merge iterators: a two-merge iterator to merge different types of iterators, and a multi-way 175 | merge iterator to merge multiple iterators of the same type. You will have a better understanding of lifetime and trait 176 | system in Rust. 177 | 178 | **What you will implement:** 179 | 180 | ```rust 181 | /// Merge multiple iterators of the same type. If the same key occurs multiple times in some 182 | /// iterators, prefer the one with smaller index. 183 | pub struct MergeIterator { 184 | iters: BinaryHeap>, 185 | current: HeapWrapper, 186 | } 187 | 188 | /// Merges two iterators of different types into one. If the two iterators have the same key, only 189 | /// produce the key once and prefer the entry from A. 190 | pub struct TwoMergeIterator { 191 | a: A, 192 | b: B, 193 | } 194 | 195 | /// A basic mem-table based on crossbeam-skiplist 196 | pub struct MemTable { 197 | map: crossbeam_skiplist::SkipMap, 198 | } 199 | ``` 200 | 201 | ## Day 4 - Storage Engine and Block Cache 202 | 203 | In this part, you will need to compose everything you have implemented in previous parts to build a fully functional 204 | storage engine. You will explore how surprisingly easy for us to maintain object references and ownerships in a 205 | multi-thread program with the help of borrow checker. 206 | 207 | 208 | **What you will implement:** 209 | 210 | ```rust 211 | impl LsmStorage { 212 | /// Get a key from the storage. In day 7, this can be further optimized by using a bloom filter. 213 | pub fn get(&self, key: &[u8]) -> Result> {} 214 | 215 | /// Put a key-value pair into the storage by writing into the current memtable. 216 | pub fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {} 217 | 218 | /// Remove a key from the storage by writing an empty value. 219 | pub fn delete(&self, _key: &[u8]) -> Result<()> {} 220 | 221 | /// Persist data to disk. 222 | /// 223 | /// In day 3: flush the current memtable to disk as L0 SST. 224 | /// In day 6: call `fsync` on WAL. 225 | pub fn sync(&self) -> Result<()> {} 226 | 227 | /// Create an iterator over a range of keys. 228 | pub fn scan( 229 | &self, 230 | _lower: Bound<&[u8]>, 231 | _upper: Bound<&[u8]>, 232 | ) -> Result> {} 233 | } 234 | ``` 235 | 236 | ## Day 5 - Compaction 237 | 238 | We now have an LSM structure, and we can merge some files in the background so as to clean garbage data and improve 239 | read efficiency. We will use RocksDB's leveled compaction algorithm to do the compaction. 240 | 241 | ## Day 6 - Recovery 242 | 243 | Storage engine should have durability, which means that data won't be lost after the engine crashes. In LSM-tree, we 244 | achieve durability by using a WAL (write-ahead log). In day 6, you will modify the write path to persist in-memory 245 | data to disk, and implement recovery process on startup. 246 | 247 | ## Day 7 - Optimizations 248 | 249 | Two key optimizations are widely adapted in today's LSM-tree engines -- prefix key compression and bloom filter. We 250 | will implement the new key encoding and the bloom filter in the last day. 251 | -------------------------------------------------------------------------------- /content/issue-1/rust-for-linux-brief-introduction.md: -------------------------------------------------------------------------------- 1 | Rust is popular among programmers for a quite long time. It won the favorite programming language on Stackoverflow in the last several years. Rust language community is active and more and more tools are built with Rust, such as block chains and databases. Even some OSes are built with Rust, though most of them are prototypes for education or demos. Of course besides these toy OSes, Rust is used to build real world OS, Linux. 2 | 3 | ```urlpreview 4 | https://github.com/Rust-for-Linux/linux 5 | ``` 6 | 7 | [Rust for Linux], short for **r4l** later, is announced in 2020, which try to reduce the number of bugs in Linux with the help of Rust. After two years of development, **r4l** is accepted by the Linux community, got [merged in Linux 6.1](https://lkml.org/lkml/2022/12/11/206). 8 | 9 | 10 | # Rust for Linux Definition 11 | 12 | Before diving into **r4l**, we must answer the question "What's is Rust for Linux". It's not about rewriting Linux with Rust or translating all the drivers in Linux from C to Rust. **r4l** is just a tool, helping kernel developers use Rust in Linux. Though **r4l** contains some drivers, they are demos for teaching Rust driver development. 13 | 14 | Here we list the main components of the **r4l** project: 15 | 16 | - Bindings of kernel functions and data structures, which are automatic generated. 17 | - Memory management module. 18 | - High level abstraction of data structures and interfaces. 19 | - Macro definition. 20 | - Sample code. 21 | - Modified Makefile 22 | 23 | # Get Hands Dirty 24 | 25 | It's time to get our hands dirty. Let's write a Rust kernel module and analyze it. 26 | 27 | ## Compile 28 | 29 | You can download the Linux kernel code from [torvalds/linux](https://github.com/torvalds/linux), usually it's a `git clone`. Rust compiler is a little different from what we use daily. The project depends on some nightly features of the compiler so that a specific version is required. Let's follow the instruction in the file [Documentation/rust/quick-start.rst](https://github.com/torvalds/linux/blob/master/Documentation/rust/quick-start.rst), in which the way to turn on the Rust functions is also covered. Then we can compile the kernel as usual. 30 | 31 | ## Linux kernel module in C 32 | 33 | Let's take a look of a C kernel module, which is shown below: 34 | 35 | ```c 36 | static struct file_operations fops = { 37 | .read = device_read, 38 | .write = device_write, 39 | .open = device_open, 40 | .release = device_release 41 | }; 42 | 43 | /* 44 | * This function is called when the module is loaded 45 | */ 46 | int init_module(void) 47 | { 48 | Major = register_chrdev(0, DEVICE_NAME, &fops); 49 | 50 | if (Major <0) { 51 | printk (KERN_ALERT "Registering char device failed with%d\n", Major); 52 | return Major; 53 | } 54 | 55 | printk(KERN_INFO "I was assigned major number %d. To taalk to\n", Major); 56 | printk(KERN_INFO "the driver, create a dev file with\n"); 57 | printk(KERN_INFO " mknod/dev/%s c%d 0'. N", DEVICE NAME, Major); 58 | printk(KERN_INFO "Try various minor numbers. Try to cat and echo to\n"); 59 | printk(KERN INFO "the device file. \n"); 60 | printk(KERN_INFO "Remove the device file and module when done. \n"); 61 | 62 | return SUCCESS; 63 | } 64 | 65 | /* 66 | * This function is called when the module is unloadled 67 | */ 68 | void cleanup_module(void) 69 | { 70 | /* 71 | * Unregister the device 72 | */ 73 | int ret = unregister_chrdev(Major, DEVICE_NAME); 74 | if (ret < 0) 75 | printk(KERN_ALERT "Error in unregister_chrdev: %d\n", ret); 76 | } 77 | ``` 78 | 79 | The whole module consists of three parts, call back functions definition, module initialization and module destruction. 80 | 81 | ## Linux kernel module in Rust 82 | 83 | Now let's take a look of a Rust kernel module: 84 | 85 | ```rust 86 | module! { 87 | type: RustChrdev, 88 | name: b"rust_chrdev", 89 | author: b"Rust for Linux Contributors", 90 | description: b"Rust character device sample", 91 | license: b"GPL", 92 | } 93 | 94 | struct RustFile; 95 | 96 | #[vtable] 97 | impl file::Operations for RustFile { 98 | fn open(_shared: &(), _file: &file::File) -> Result { 99 | Ok(()) 100 | } 101 | } 102 | 103 | struct RustChrdev { 104 | _dev: Pin>>, 105 | } 106 | 107 | impl kernel::Module for RustChrdev { 108 | fn init(name: &'static CStr, module: &'static ThisModule) -> Result { 109 | pr_info!("Rust character device sample (init)\n"); 110 | 111 | let mut chrdev_reg = chrdev::Registration::new_pinned(name, 0, module)?; 112 | 113 | // Register the same kind of device twice, we'ree just demonstrating 114 | // that you can use multiple minors. There anre two minors in this case 115 | // because its type is `chrdev::Registration<2>` 116 | chrdev_reg.as_mut().register::()?; 117 | chrdev_reg.as_mut().register::()?; 118 | 119 | Ok(RustChrdev { _dev: chrdev_reg }) 120 | } 121 | } 122 | 123 | impl Drop for RustChrdev { 124 | fn drop(&mut self) { 125 | pr_info!("Rust character device sample (exit)\n"); 126 | } 127 | } 128 | ``` 129 | 130 | It's easy to tell that these two versions are very similar, so migrating a driver from C to Rust should not be too hard. Let's dive into the code behind the kernel, we'll find out how they help simplify the module build. 131 | 132 | # Rust for Linux Deep Dive 133 | 134 | First let's talk about the macro used in the first few lines `module`. That macro seems simple and only declare some property of the module. If we jump to the definition of the macro, we'll find some interesting work there, such as C language interface and some utilities to initialize and destroy the module from C code, which is shown below. 135 | 136 | ```rust 137 | #[cfg(not(MODULE))] 138 | #[doc(hidden)] 139 | #[no_mangle] 140 | pub extern "C" fn __{name}_init() -> core::ffi::c_int {{ 141 | __init() 142 | }} 143 | 144 | #[cfg(not(MODULE))] 145 | #[doc(hidden)] 146 | #[no_mangle] 147 | pub extern "C" fn __{name}_exit() { 148 | __exit() 149 | }} 150 | 151 | fn __init() -> core::ffi::c_int {{ 152 | match <{type_} as kernel::Module>::init(kernel::c_str!("{name}"), &THIS_MODULE) {{ 153 | Ok(m) => {{ 154 | unsafe {{ 155 | __MOD = Some(m); 156 | }} 157 | return 0; 158 | }} 159 | Err(e) => {{ 160 | return e.to_kernel_errno(); 161 | }} 162 | }} 163 | }} 164 | 165 | fn __exit() {{ 166 | unsafe {{ 167 | // Invokes `drop()` on `__MOD`, which should be used for cleanup. 168 | __MOD = None; 169 | }} 170 | }} 171 | ``` 172 | 173 | Then there's another macro named `vtable`. How does it work? It add a bool constant for each function like `HAS_XXX`. If function `A` is defined, `HAS_A` is true, otherwise it's false. The caller can tell if the function is defined by checking these constants. Besides that `vtable` also generate some FFI part, which is called by the C code. 174 | 175 | In this example, it's not hard to find out that **r4l** is kind of "glue", building a bridge between Rust and C. It make the module developers focus on the functionality instead of the language related dirty work. 176 | 177 | # Summary 178 | 179 | Rust for Linux is a utility project to help write Linux kernel module in Rust. By using it, the developers write the pure Rust code dealing with the module's function instead of handling the gap between C and Rust. Though the project is in the early stage, many important components are still missing, we believe they'll be ready in the near future with the help of the whole community. 180 | 181 | [rust for linux]: https://github.com/Rust-for-Linux/linux 182 | -------------------------------------------------------------------------------- /content/issue-1/zine.toml: -------------------------------------------------------------------------------- 1 | slug = "issue-1" 2 | number = 1 3 | title = "Issue 1" 4 | cover = "/static/ferris/1.jpeg" 5 | pub_date = "2023-02-06" 6 | publish = true 7 | 8 | [[article]] 9 | path = "/announcing" 10 | file = "announcing.md" 11 | title = "Announcing Rust Magazine" 12 | author = "rust-magazine" 13 | topic = ["announcement"] 14 | pub_date = "2022-12-10" 15 | publish = true 16 | featured = true 17 | i18n.zh = { path = "/announcing-zh", file = "announcing-zh.md", title = "宣布 Rust 杂志" } 18 | 19 | [[article]] 20 | file = "vecdeque-resize-optimization.md" 21 | title = "VecDeque::resize() optimization" 22 | author = "folyd" 23 | topic = ["pr-demystifying", "optimization"] 24 | pub_date = "2022-12-24" 25 | publish = true 26 | featured = true 27 | 28 | [[article]] 29 | file = "minilsm.md" 30 | title = "MiniLSM: A Tutorial of Building Storage Engine in a Week using Rust" 31 | author = "skyzh" 32 | topic = ["tutorial"] 33 | pub_date = "2022-12-30" 34 | publish = true 35 | featured = true 36 | 37 | 38 | [[article]] 39 | file = "hidden-control-flow.md" 40 | title = "The Hidden Control Flow — Some Insights on an Async Cancellation Problem in Rust" 41 | author = ["waynexia", "greptime-team"] 42 | topic = ["async", "database"] 43 | pub_date = "2023-01-12" 44 | publish = true 45 | featured = true 46 | canonical = "https://greptime.com/blogs/2023-01-12-hidden-control-flow.html" 47 | i18n.zh = { file = "hidden-control-flow-zh.md", title = "看不见的控制流 — Rust 异步取消问题的几点思考"} 48 | 49 | [[article]] 50 | file = "contribute-to-rustc.md" 51 | title = "How I contribute to Rust Compiler" 52 | author = "yukang" 53 | topic = ["tutorial"] 54 | pub_date = "2023-01-29" 55 | publish = true 56 | featured = true 57 | 58 | [[article]] 59 | file = "weihanglo.md" 60 | title = "Interview with Weihang Lo" 61 | author = ["folyd", "weihanglo"] 62 | topic = ["interview"] 63 | pub_date = "2023-01-30" 64 | publish = true 65 | featured = true 66 | 67 | [[article]] 68 | file = "rust-for-linux-brief-introduction.md" 69 | title = "Rust for Linux Brief Introduction" 70 | author = "shijicheng" 71 | topic = ["tutorial"] 72 | pub_date = "2023-01-30" 73 | publish = true 74 | featured = true 75 | 76 | [[article]] 77 | file = "2022-review-the-adoption-of-rust-in-business.md" 78 | title = "2022 Review | The adoption of Rust in Business" 79 | author = "HandongZhang" 80 | topic = ["business"] 81 | pub_date = "2023-01-31" 82 | publish = true 83 | featured = true 84 | i18n.zh = {file = "2022-review-the-adoption-of-rust-in-business-zh.md", title = "Rust 2022 全球商业化应用盘点", author = ["HandongZhang"] } 85 | -------------------------------------------------------------------------------- /content/issue-2/how-i-designed-the-api-for-backon-a-user-friendly-retry-crate.md: -------------------------------------------------------------------------------- 1 | [backon](https://github.com/Xuanwo/backon) is a Rust error retry library, and this article aims to share some techniques I used in implementing it. 2 | 3 | ```urlpreview 4 | https://github.com/Xuanwo/backon 5 | ``` 6 | 7 | ## Origin 8 | 9 | When implementing the RetryLayer for [OpenDAL](https://github.com/datafuselabs/opendal), I needed to provide a backoff mechanism to implement features such as exponential backoff and jitter. 10 | 11 | > Backoff is a technique used in computer science and networking to handle errors or congestion in a more efficient way. The basic idea is to progressively increase the time between retries when encountering an error or congestion, so that the system has time to recover and handle the issue without overwhelming it with requests. 12 | 13 | Although I found [`backoff`](https://github.com/ihrwein/backoff) through a simple search, I wasn't quite satisfied. First, I noticed that the maintenance status of the library didn't seem to be good, with 4 unmerged PRs, and the main branch was last updated in 2021. Secondly, I didn't like the API it provided: 14 | 15 | ```rust 16 | async fn fetch_url(url: &str) -> Result { 17 | retry(ExponentialBackoff::default(), || async { fetch().await }).await 18 | } 19 | ``` 20 | 21 | Backoff implementation isn't complicated, so why not make one that feels comfortable to use? 22 | 23 | ## Design 24 | 25 | My first idea was to use an `Iterator` to represent the backoff. Any iterator that can return a `Duration` type can be used as backoff. Using an iterator to represent backoff has a very direct and clear meaning, and users can easily understand and implement it without reading every function's comments. Secondly, I wanted to provide a usage experience for backoff similar to Rust's native functions: 26 | 27 | ```rust 28 | async fn fetch_url(url: &str) -> Result { 29 | fetch.retry(ExponentialBackoff::default()).await 30 | } 31 | ``` 32 | 33 | It looks great: simple and direct, doesn't disrupt the user's reading order, and allows the user to locate the business logic position at a glance. Let's get started implementing it! 34 | 35 | ## Implementation 36 | 37 | First of all, what we need to understand is that async functions in Rust are essentially generators. These generators capture variables from the current environment and generate an anonymous Future. To retry an async function, we need to call the generator again to generate a brand new Future to execute. 38 | 39 | I once went down the wrong path with a failed demo for retry: [we can't retry a future directly](https://github.com/Xuanwo/backon/pull/1). At that time, I naively tried to retry a TryFuture directly: 40 | 41 | ```rust 42 | pub trait Retryable bool>: TryFuture + Sized { 43 | fn retry(self, backoff: B, handle: F) -> Retry; 44 | } 45 | ``` 46 | 47 | Now I understand that this approach is incorrect. Once a `Future` enters the `Poll::Ready` state, we should not poll it again, as documented: 48 | 49 | > Once a future has completed (returned `Ready` from `poll`), calling its `poll` method again may panic, block forever, or cause other kinds of problems 50 | 51 | Next, I needed to adjust my thinking and focus on implementing for `|| -> impl Future>`. First, I defined a `Retryable` trait and implemented it for all `FnMut() -> Fut` types: 52 | 53 | ```rust 54 | pub trait Retryable< 55 | B: BackoffBuilder, 56 | T, 57 | E, 58 | Fut: Future>, 59 | FutureFn: FnMut() -> Fut, 60 | > 61 | { 62 | /// Generate a new retry 63 | fn retry(self, builder: &B) -> Retry; 64 | } 65 | 66 | impl Retryable for FutureFn 67 | where 68 | B: BackoffBuilder, 69 | Fut: Future>, 70 | FutureFn: FnMut() -> Fut, 71 | { 72 | fn retry(self, builder: &B) -> Retry { 73 | Retry::new(self, builder.build()) 74 | } 75 | } 76 | ``` 77 | 78 | This trait involves the following type parameters: 79 | 80 | - `B: BackoffBuilder`: the backoff builder passed in by the user, which specifies different backoff parameters 81 | - `FutureFn: FnMut() -> Fut`: indicates that its type is a function that returns a `Fut` 82 | - `Fut: Future>`: represents a Future that returns a `Result` 83 | 84 | The returned Retry struct wraps all of the above types: 85 | 86 | ```rust 87 | pub struct Retry>, FutureFn: FnMut() -> Fut> { 88 | backoff: B, 89 | retryable: fn(&E) -> bool, 90 | notify: fn(&E, Duration), 91 | future_fn: FutureFn, 92 | 93 | #[pin] 94 | state: State, 95 | } 96 | ``` 97 | 98 | Apart from `backoff` and `future_fn`, we introduced `retryable` and `notify` to implement retryable error checking and notification functions. Once the type system is clear, the next step is to implement the correct `Future` trait for `Retry`, and the details will not be elaborated: 99 | 100 | ```rust 101 | impl Future for Retry 102 | where 103 | B: Backoff, 104 | Fut: Future>, 105 | FutureFn: FnMut() -> Fut, 106 | { 107 | type Output = Result; 108 | 109 | fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { 110 | ... 111 | } 112 | } 113 | ``` 114 | 115 | In addition, there are some transactional tasks that need to be completed: we need to let users define which errors can be retried, and provide custom notification for retrying. 116 | 117 | Finally, the combined effect is as follows: 118 | 119 | ```rust 120 | #[tokio::main] 121 | async fn main() -> Result<()> { 122 | let content = fetch 123 | .retry(&ExponentialBuilder::default()) 124 | .when(|e| e.to_string() == "EOF") 125 | .notify(|err, dur| { 126 | println!("retrying error {:?} with sleeping {:?}", err, dur); 127 | }) 128 | .await?; 129 | 130 | Ok(()) 131 | } 132 | ``` 133 | 134 | Looks perfect! 135 | 136 | ## One More Thing 137 | 138 | Oh, wait a minute, backon doesn't support synchronous functions yet! No problem, we just need to apply the same approach: 139 | 140 | ```rust 141 | pub trait BlockingRetryable Result> { 142 | /// Generate a new retry 143 | fn retry(self, builder: &B) -> BlockingRetry; 144 | } 145 | 146 | impl BlockingRetryable for F 147 | where 148 | B: BackoffBuilder, 149 | F: FnMut() -> Result, 150 | { 151 | fn retry(self, builder: &B) -> BlockingRetry { 152 | BlockingRetry::new(self, builder.build()) 153 | } 154 | } 155 | ``` 156 | 157 | Due to the unavailability of the [`fn_traits`](https://github.com/rust-lang/rust/issues/29625) feature, which is still unstable, I chose to add a new function to `BlockingRetry` instead: 158 | 159 | ```rust 160 | impl BlockingRetry 161 | where 162 | B: Backoff, 163 | F: FnMut() -> Result, 164 | { 165 | pub fn call(mut self) -> Result { 166 | ... 167 | } 168 | } 169 | ``` 170 | 171 | It feels great to perform retry operations in the call function, and it also has a similar beauty to the Async version. 172 | 173 | ```rust 174 | fn main() -> Result<()> { 175 | let content = fetch 176 | .retry(&ExponentialBuilder::default()) 177 | .when(|e| e.to_string() == "EOF") 178 | .notify(|err, dur| { 179 | println!("retrying error {:?} with sleeping {:?}", err, dur); 180 | }) 181 | .call()?; 182 | 183 | Ok(()) 184 | } 185 | ``` 186 | 187 | In this article, I have shared the design and implementation of `backon`. Throughout the process, I have mainly used Rust's generics mechanisms, namely `FnMut() -> Fut` and `FnMut() -> Result`, to create custom traits and add new functionality. I hope this implementation can inspire you to design more user-friendly library APIs. 188 | 189 | Thank you for reading! 190 | -------------------------------------------------------------------------------- /content/issue-2/intro.md: -------------------------------------------------------------------------------- 1 | Dear Rustaceans, 2 | 3 | Welcome to the second issue of Rust Magazine! We are excited to bring you another round of engaging articles and tutorials from the Rust community. 4 | 5 | In this issue, we explore topics ranging from optimizing Rust binary search to designing an API for a user-friendly retry crate. We also have an interview with `@sunli`, the creator of popular crates such as [async-graphql] and [poem], as well as a conversation with ChatGPT on the topic of Rust lifetimes. Additionally, we feature articles on [Egg](https://github.com/egraphs-good/egg), a popular Rust framework to help build a SQL optimizer, and an applicative command-line parser with Category Theory. 6 | 7 | Thank you to our authors for their contributions and dedication to the Rust community. We couldn't have done it without you. 8 | 9 | As we close the chapter on Issue 2, we are excited to announce that we are looking for articles for Issue 3, and we would love to hear from you. Do you have a Rust project you want to share with the community? Have you learned something new about Rust that you want to pass on? Or do you have an idea for a Rust-related article that you would like to explore? Please don't hesitate to submit your proposal via our website. Let's make Rust Magazine a platform for everyone to share their Rust experience and knowledge! 10 | 11 | Best regards, 12 | 13 | The Rust Magazine Team. 14 | 15 | [async-graphql]: https://github.com/async-graphql/async-graphql 16 | [poem]: https://github.com/poem-web/poem 17 | -------------------------------------------------------------------------------- /content/issue-2/optimize-rust-comptime-zh.md: -------------------------------------------------------------------------------- 1 | 虽然经常有逸闻抱怨 Rust 编译速度臭名昭著地慢，但我们的项目 [RisingWave](https://github.com/risingwavelabs/risingwave) 在经过前人比如（[skyzh](https://github.com/skyzh)，[BugenZhao](https://github.com/bugenzhao)）的一些努力后，编译速度并不算慢，特别是我自从用上 M1 的 Macbook Pro 后，编译速度根本不是问题，全量 debug 编译也就两三分钟。 2 | 3 | 然而随着时间推移，CI 里加了越来越多的东西，越来越臃肿了。现在 main workflow 需要大概 40min，PR workflow 需要大概 25min30s。虽然并不算特别慢，但是可以体感到比以前变慢了不少。 4 | 5 | 于是我前两天心血来潮，决定花点时间研究一下能不能再优化一点编译速度。 6 | 7 | 令我非常震惊的是，没想到存在着一些非常简单的方法，动动小手就产生了惊人的成效。感觉完全可以用 low-hanging fruits、silver bullet 甚至是 free lunch 来形容 🤯。 8 | 9 | --- 10 | 11 | P.S. 很推荐 [matklad](https://github.com/matklad)（IntelliJ Rust 和 rust-analyzer 的原作者）的 blog： 12 | 13 | - [Fast Rust Builds](https://matklad.github.io/2021/09/04/fast-rust-builds.html) 14 | - [Delete Cargo Integration Tests](https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html) 15 | 16 | 我用到的大部分方法这里面都有讲到，而且他讲的清晰明了。如果没有另外说明，那么文中的 quote 都来自这里。 17 | 18 | 本文算是我的实践记录，或者大概可以也当成一个 tl; dr 版。每一个优化点都带上了相应的 PR，可以结合 [commit history](https://github.com/risingwavelabs/risingwave/commits/main?after=d8198fa138003e1f1431053f4f5f09e4a5fa8fd8+69&branch=main&qualified_name=refs%2Fheads%2Fmain) 点开每个优化点前后的页面对比效果。 19 | 20 | --- 21 | 22 | P.P.S. 优化完的结果：main 最快 27min，PR 最快 16min，大多在 17-19min 左右。 23 | 24 | # 可供参考的数据、图表 25 | 26 | > Build times are a fairly easy optimization problem: it’s trivial to get direct feedback (just time the build), there are a bunch of tools for profiling, and you don’t even need to come up with a representative benchmark. 27 | 28 | 前两天在研究 [profiling](https://xxchan.github.io/cs/2023/02/08/profiling-101.html)，那现在提到要优化，当然应该看看有没有什么数据、图表看看，找到瓶颈在哪里再来优化。 29 | 30 | ## CI waterfall & dag graph 31 | 32 | 我们的 CI 用的是 Buildkite，正常点开一个页面（例如 [Build #17099](https://buildkite.com/risingwavelabs/pull-request/builds/17099)）长这样： 33 | 34 | ![buildkite-1.png](https://xxchan.github.io/assets/img/comptime/buildkite-1.png) 35 | 36 | Buildkite 有两个非常好用的隐藏页面，分别是在 `/waterfall` 和 `/dag` 里，可以看到： 37 | 38 | ![buildkite-waterfall.png](https://xxchan.github.io/assets/img/comptime/buildkite-waterfall.png) 39 | 40 | ![buildkite-dag.png](https://xxchan.github.io/assets/img/comptime/buildkite-dag.png) 41 | 42 | 从图上我们可以清晰地看出，最大的瓶颈是 simulation build -> recovery test 43 | 44 | ## `cargo build --timings` 45 | 46 | Cargo 自带 profiling 编译时间的支持（貌似是去年 stablize 的），通过 [cargo build --timings](https://doc.rust-lang.org/cargo/reference/timings.html) 启用，它长这样： 47 | 48 | ![timings.png](https://xxchan.github.io/assets/img/comptime/timings.png) 49 | 50 | 可以发现 `zstd-sys` ， `protobuf-src` 等几个依赖的编译时间非常长，应该想办法看看能不能优化掉。 51 | 52 | # Step 1: Compilation cache 53 | 54 | [ci: try sccache #7799](https://github.com/risingwavelabs/risingwave/pull/7799) 55 | 56 | > If you think about it, it’s pretty obvious how a good caching strategy for CI should work. 57 | > 58 | > Unfortunately, almost nobody does this. 59 | 60 | [2023-04: 为什么你该试试 Sccache？](https://xuanwo.io/reports/2023-04/) 在 xuanwo 的大力鼓吹下，我非常心动，也想尝试一下 sccache。这也算是我这次搞优化的一大 trigger。 61 | 62 | 不用多说，非常简单好用。只需加两个环境变量就一键启动了： 63 | 64 | ```dockerfile 65 | ENV RUSTC_WRAPPER=sccache 66 | ENV SCCACHE_BUCKET=ci-sccache-bucket 67 | ``` 68 | 69 | （在这背后其实需要研究一下 Buildkite 和 AWS 的配置——实际上也非常傻瓜。Buildkite 可以通过 IAM Role 来获得权限，加一个 S3 bucket 的 policy 就 work 了，完全不用配置 secret key 之类的东西。我之前还在思考能不能在 CI 里把 key echo 出来，看来是完全不用担心这种事 😄） 70 | 71 | 效果立竿见影，simulation build 减少了 2.5min，非瓶颈的 debug build 减少了 4min。虽然并没有质变，但是免费的量变何乐而不为呢？ 72 | 73 | # Step 2: Remove unused dependencies 74 | 75 | [build: remove unused deps #7816](https://github.com/risingwavelabs/risingwave/pull/7816) 76 | 77 | 在 `Cargo.toml` 中声明的依赖不管实际上有没有用到，都会被编译。更甚它可能会引入不必要的 syncronization point，影响编译的并行度。 78 | 79 | 有个老工具 [cargo-udeps](https://github.com/est31/cargo-udeps) 就是干这个的，但是首先它并不支持自动修复，而且它很慢。另外印象中有一个毛病是它不能和 `workspace-hack` 一起用。这导致 RisingWave 中长期没有清理过 unused dependencies。典型的破窗效应 🥲！ 80 | 81 | 在 `cargo-udeps` 里关于自动 fix 的 issue 下面看到有人提了 [ `cargo-machete` ](https://github.com/bnjbvr/cargo-machete)（这个名字是大砍刀的意思 🤣），觉得是骡子是马拉出来遛遛，发现它跑的飞快，也没有几个 false postive。虽然有几个小问题（参考上面 PR 的 commit history），但是都能容易地修掉。 82 | 83 | 大砍刀的作者有一篇 [blog](https://blog.benj.me/2022/04/27/cargo-machete/) 介绍了 unused dependencies 的危害以及 `cargo-machete` 的解法。具体说来，`cargo-udeps` 是用 `cargo check` 先编译了一遍再分析的，而 `cargo-machete` 是简单粗暴的 `ripgrep`。 84 | 85 | 这个 PR 一下子删掉了大几十个 udeps，也是让我大吃一惊 🤯。可惜的是，CI 的时间并没有进一步缩短，感觉这侧面说明了 cache 效果很好……我本地粗略地测了一下，大概快了十几二十秒。蚊子腿也是肉嘛，anyway it's free! 86 | 87 | --- 88 | 89 | P.S. 其实 `cargo-udeps` 配一下也是能和 `workspace-hack` 用的：[feat(risedev): add `check-udeps` #7836](https://github.com/risingwavelabs/risingwave/pull/7836) 90 | 91 | # Step 3: Disable incremental compilation 92 | 93 | [build: disable incremental build in CI #7838](https://github.com/risingwavelabs/risingwave/pull/7838) 94 | 95 | 干完上面两个小工作之后本来已经想收工了，但有点心痒痒，觉得 simulation build 还是有点慢。于是我决定 profiling 一下看看。然后就看到了一开始贴的 `--timings` 的图中的几个庞然大物，我觉得这很不 make sense。 96 | 97 | 我搜了搜 sccache non-cacheable 的原因，发现 incremental compilation 是个很大的 caveat，立马尝试了一下，然后我再次震惊了，效果 _stupidly_ 好： 98 | 99 | ![timings-2](https://xxchan.github.io/assets/img/comptime/timings-2.png) 100 | 101 | 这让 simulation build 的时间瞬间下降了 4 分钟…… 102 | 103 | 实际上我们的 debug build 是很早之前就关掉了 incremental compilation： 104 | 105 | ```toml 106 | [profile.ci-dev] 107 | incremental = false 108 | ``` 109 | 110 | 但是后来加上新的 build profile 的时候没有考虑到这个问题。仔细想一想，incremental compilation 虽好，但它在 CI 里不太 make sense 啊！ 111 | 112 | > CI builds often are closer to from-scratch builds, as changes are typically much bigger than from a local edit-compile cycle. For from-scratch builds, incremental adds an extra dependency-tracking overhead. It also significantly increases the amount of IO and the size of `./target`, which make caching less effective. 113 | 114 | 于是我干脆在 CI 里加了个全局的 env var 来把它关掉，一劳永逸。 115 | 116 | # Step 4: Single binary integration test 117 | 118 | [build: single-binary integration test #7842](https://github.com/risingwavelabs/risingwave/pull/7842) 119 | 120 | 又是一个 _stupidly effective_ 的优化。tl;dr: 121 | 122 | Don’t do this: 123 | 124 | ``` 125 | tests/ 126 | foo.rs 127 | bar.rs 128 | ``` 129 | 130 | Do this instead: 131 | 132 | ``` 133 | tests/ 134 | integration/ 135 | main.rs 136 | foo.rs 137 | bar.rs 138 | ``` 139 | 140 | 因为 `tests/` 下面的每个文件都会编译成一个单独的 binary（意味着会每个都 link 一下依赖）。除了编译慢，这甚至还可能导致测试跑的慢（`cargo test` 的缺陷）。 141 | 142 | 这个优化没有减少我们的测试时间（可能是因为 `cargo nextest` 的优越性），但它一下子又减少了快 2 分钟的编译时间……另外说来有点可笑的是，它还减少了 2 分钟的 artifacts 上传下载、压缩解压的时间……（虽然后者在瓶颈上并没有影响） 143 | 144 | # 其他一些先前就存在的优化 145 | 146 | 以上就是我这次优化的主要过程了，这下终于可以心满意足地收工了。最后想再总结一些前人的努力，以供参考。 147 | 148 | - 使用 [`cargo nextest`](https://github.com/nextest-rs/nextest) 替代 `cargo test`。 149 | - 使用 `workspace-hack` 技术：见 [`cargo hakari`](https://docs.rs/cargo-hakari/latest/cargo_hakari/about/index.html)。 150 | - 给 cargo registry 加 cache，或者使用刚刚 stablize 的 sparse index，可参考 [DCjanus](https://github.com/dcjanus) 的这篇 [blog](https://blog.dcjanus.com/posts/cargo-registry-index-in-http/)。 151 | - 把巨大的 crate 拆分成多个小 create。 152 | - link time 的优化：link 很花时间，而且是单线程的，很可能成为瓶颈 153 | - 使用更快的 linker：`mold` for Linux, `zld` for macOS. `lld` is the most mature option for production use. 154 | - 在 debug build 上关掉 Link Time Optimization (LTO)。 155 | - Trade-off between compile time and performance：CI 的总时间是编译+测试，那么编译优化（包括上面的 LTO）开不开，开多少实际上就是在前后者之间 trade-off，可以调整测试来达到一个整体最优的选择。例如 bugen gg 在我们的 build profile 上的骚操作： 156 | 157 | ```toml 158 | # The profile used for CI in pull requests. 159 | # External dependencies are built with optimization enabled, while crates in this workspace are built 160 | # with `dev` profile and full debug info. This is a trade-off between build time and e2e test time. 161 | [profile.ci-dev] 162 | inherits = "dev" 163 | incremental = false 164 | [profile.ci-dev.package."*"] # external dependencies 165 | opt-level = 1 166 | [profile.ci-dev.package."tokio"] 167 | opt-level = 3 168 | [profile.ci-dev.package."async_stack_trace"] 169 | opt-level = 3 170 | [profile.ci-dev.package."indextree"] 171 | opt-level = 3 172 | [profile.ci-dev.package."task_stats_alloc"] 173 | opt-level = 3 174 | 175 | # The profile used for deterministic simulation tests in CI. 176 | # The simulator can only run single-threaded, so optimization is required to make the running time 177 | # reasonable. The optimization level is customized to speed up the build. 178 | [profile.ci-sim] 179 | inherits = "dev" 180 | opt-level = 2 181 | incremental = false 182 | ``` 183 | 184 | 除此以外的更多优化也有很多人都总结过，我就不多说（不懂）了，例如这篇 blog：[Tips for Faster Rust Compile Times](https://endler.dev/2020/rust-compile-times/) 185 | 186 | # 总结 187 | 188 | CI、开发者体验这种东西很容易就会在无人照料的情况下变得杂草丛生，但如果定期打理一下，可能会有意想不到的收获，一点点微小的努力就带来巨大的提升。 189 | 190 | 最后再摘两段 matklad [blog](https://matklad.github.io/2021/09/04/fast-rust-builds.html) 里的话作结： 191 | 192 | > Compilation time is a _multiplier_ for basically everything. Whether you want to ship more features, to make code faster, to adapt to a change of requirements, or to attract new contributors, build time is a factor in that. 193 | > 194 | > It also is a non-linear factor. Just waiting for the compiler is the smaller problem. The big one is losing the state of the flow or (worse) mental context switch to do something else while the code is compiling. One minute of work for the compiler wastes more than one minute of work for the human. 195 | -------------------------------------------------------------------------------- /content/issue-2/sunli.md: -------------------------------------------------------------------------------- 1 | I am pleased to introduce our second interviewee, [Sunli], the creator of two widely-used crates: [async-graphql] and [poem]. It is an honor to have the chance to speak with him and gain insights into his experiences and expertise. 2 | 3 | # Introduction 4 | 5 | **Introduce yourself and share a bit about your background with Rust. When did you start learning Rust, and what inspired you to do so?** 6 | 7 | ```quote 8 | avatar="/static/avatar/sunli.jpeg" 9 | author="Sunli" 10 | content=""" 11 | Hello everyone, I am **Sunli** (Github: [@sunli829](https://github.com/sunli829)), an older programmer who loves coding. Previously, C++ was my main programming language, and even though I have been using it for many years, I still feel like I haven't fully mastered it. 12 | 13 | Three years ago, I started learning Rust. Rust's ownership and lifetime mechanisms force developers to follow conventions and naturally consider exceptional cases when writing code. This is one of the truly distinctive features that sets Rust apart from other programming languages. Therefore, I began applying it to my work. I also enjoy the Rust community, there are many helpful people. 14 | """ 15 | ``` 16 | 17 | # Async-graphql 18 | 19 | ```urlpreview 20 | https://github.com/async-graphql/async-graphql 21 | ``` 22 | 23 | **Can you explain what async-graphql is and how it differs from other GraphQL libraries?** 24 | 25 | ```quote 26 | avatar="/static/avatar/sunli.jpeg" 27 | author="Sunli" 28 | content=""" 29 | [GraphQL](https://graphql.org/) is a query language for APIs that allows clients to request specific data and enables servers to respond with the corresponding data based on the request content. The flexibility and powerful querying capabilities of GraphQL have made it increasingly popular in developing modern applications. 30 | 31 | **async-graphql** is a server-side library used to implement the GraphQL specification, leveraging Rust features such as procedural macros to enhance development experience and provide full asynchronous support. 32 | 33 | Another library similar to **async-graphql** is [juniper](https://github.com/graphql-rust/juniper), which I have used before and is also excellent. 34 | """ 35 | ``` 36 | 37 | **What inspired you to create async-graphql, and what has the response been like from the Rust community?** 38 | 39 | ```quote 40 | avatar="/static/avatar/sunli.jpeg" 41 | author="Sunli" 42 | content=""" 43 | As a Rust learner, I wanted to contribute something to the community. As I wasn't familiar with GraphQL at the time, I did some research and decided to develop the **async-graphql** project, also to learn GraphQL. 44 | 45 | During development, I referenced the more complete GraphQL library at the time, **Juniper**, but it didn't support Rust's asynchronous features. So my goal became to implement all the features provided by **Juniper** and add asynchronous support to provide a better user experience. 46 | 47 | About a month later, I released the first version of **async-graphql** and have been maintaining and improving it ever since. As it is the only library that supports asynchronous, many people have migrated from **Juniper** to **async-graphql**, which I believe is a very important point. 48 | 49 | Although the first version of the code was not perfect, it still worked properly. Therefore, I received a lot of feedback, which helped me better understand Rust and GraphQL and provided many suggestions for improving **async-graphql**. Some even provided PRs to rewrite parts of the code, which greatly improved my skills and made me more aware of how to contribute to the open-source community. 50 | """ 51 | ``` 52 | 53 | # Poem 54 | 55 | ```urlpreview 56 | https://github.com/poem-web/poem 57 | ``` 58 | 59 | **Can you tell us about Poem and its design philosophy?** 60 | 61 | ```quote 62 | avatar="/static/avatar/sunli.jpeg" 63 | author="Sunli" 64 | content=""" 65 | I wanted a web framework that may not have the best performance, but would be relatively simple to use, which is why I made it. 66 | 67 | **Poem** was named by `@HandongZhang`, a friend in the Rust community. It is definitely a good name. He transferred this crate to me for free, hoping that using this library can really be like writing poem. 68 | """ 69 | ``` 70 | 71 | **What inspired you to develop another web framework, and how does Poem differ from other Rust web frameworks?** 72 | 73 | ```quote 74 | avatar="/static/avatar/sunli.jpeg" 75 | author="Sunli" 76 | content=""" 77 | Rust has many high-quality web frameworks, some of which are better than **Poem**. However, for me personally, **Poem** is very important because it's mine and I can freely modify it, which makes me more free during the development process. 78 | 79 | Over time, I wanted to add more features and capabilities to **Poem**, so I started learning the OpenAPI specification and created a sub-project called [poem-openapi](https://github.com/poem-web/poem/tree/master/poem-openapi). Using Rust's procedural macros to automatically generate API documentation that conforms to the OpenAPI specification, similar to the popular [FastAPI](https://github.com/tiangolo/fastapi) library in Python. This greatly improved the efficiency of the development process. 80 | """ 81 | ``` 82 | 83 | **Can you share any exciting upcoming features or plans for Poem?** 84 | 85 | ```quote 86 | avatar="/static/avatar/sunli.jpeg" 87 | author="Sunli" 88 | content="There is no plan currently as the functionalities provided by **Poem** are already sufficient for my needs." 89 | ``` 90 | 91 | # Open-source 92 | 93 | **What are the most challenging aspects of maintaining two popular open-source projects like async-graphql and Poem?** 94 | 95 | ```quote 96 | avatar="/static/avatar/sunli.jpeg" 97 | author="Sunli" 98 | content=""" 99 | I didn't expect these two libraries to have so many users. In the early days, I would receive around 10 issue reports from users around the world every day, which required me to work for more than 12 hours a day. Although it was very hard, as time went by, I saw that the number of users of these libraries was increasing, and I felt that it was all worth it. In fact, this was my first attempt at an open-source project, so I learned a lot during this process, such as how to collaborate with other developers using Github and how to better manage and solve problems. 100 | """ 101 | ``` 102 | 103 | **How do you balance maintaining these projects with other commitments?** 104 | 105 | ```quote 106 | avatar="/static/avatar/sunli.jpeg" 107 | author="Sunli" 108 | content="Balancing maintaining projects with other commitments can be difficult and definitely requires more time." 109 | ``` 110 | 111 | # End question 112 | 113 | **Are there any other hobbies or interests that you have, and how do you balance them with your work on Rust projects?** 114 | 115 | ```quote 116 | avatar="/static/avatar/sunli.jpeg" 117 | author="Sunli" 118 | content=""" 119 | I only have two hobbies, one is programming and the other is playing games. When I feel tired, I will spend some time playing games to relax my mind and body. This is also the secret to my long-term work efficiency. In this way, I can get rest and entertainment, so that I can better focus on my programming projects. 120 | 121 | **I am fortunate to have the support of my family, especially my wife, who always encourages me to do things that make me happy. This support gives me more confidence to continue doing what I love, and I feel very lucky and grateful for it.** 122 | """ 123 | ``` 124 | 125 | [sunli]: https://github.com/sunli829 126 | [async-graphql]: https://github.com/async-graphql/async-graphql 127 | [poem]: https://github.com/poem-web/poem 128 | -------------------------------------------------------------------------------- /content/issue-2/zine.toml: -------------------------------------------------------------------------------- 1 | 2 | slug = "issue-2" 3 | number = 2 4 | title = "Issue 2" 5 | cover = "/static/ferris/2.jpeg" 6 | pub_date = "2023-03-08" 7 | publish = true 8 | 9 | [[article]] 10 | file = "optimize-binary-search.md" 11 | title = "Optimize Binary Search of Rust" 12 | author = "folyd" 13 | pub_date = "2023-02-11" 14 | topic = ["pr-demystifying", "optimization"] 15 | publish = true 16 | featured = true 17 | 18 | [[article]] 19 | file = "sunli.md" 20 | title = "Interview with Sunli" 21 | author = ["folyd", "sunli"] 22 | topic = ["interview"] 23 | pub_date = "2023-02-15" 24 | publish = true 25 | featured = true 26 | 27 | [[article]] 28 | file = "optimize-rust-comptime.md" 29 | title = "Stupidly effective ways to optimize Rust compile time" 30 | author = "xxchan" 31 | topic = ["optimization"] 32 | pub_date = "2023-02-18" 33 | publish = true 34 | featured = true 35 | canonical = "https://xxchan.github.io/cs/2023/02/17/optimize-rust-comptime-en.html" 36 | i18n.zh = { file = "optimize-rust-comptime-zh.md", title = "我如何动动小手就让 CI 时间减少了 10 分钟" } 37 | 38 | [[article]] 39 | file = "applicative-parsing.md" 40 | title = "Using Category Theory to parse command line options" 41 | author = ["manpacket"] 42 | topic = ["tutorial"] 43 | publish = true 44 | featured = true 45 | pub_date = "2023-02-23" 46 | 47 | [[article]] 48 | file = "how-i-designed-the-api-for-backon-a-user-friendly-retry-crate.md" 49 | title = "How I Designed the API for Backon, a User-Friendly Retry Crate" 50 | author = "xuanwo" 51 | topic = ["tutorial", "async"] 52 | pub_date = "2023-03-02" 53 | publish = true 54 | featured = true 55 | canonical = "https://xuanwo.io/reports/2023-09/" 56 | 57 | [[article]] 58 | file = "demystifying-rust-lifetime-with-chatgpt.md" 59 | title = "Demystifying Rust Lifetimes: A Conversation with ChatGPT" 60 | author = ["handongzhang"] 61 | pub_date = "2023-03-05" 62 | publish = true 63 | featured = true 64 | 65 | [[article]] 66 | file = "write-a-sql-optimizer-using-egg.md" 67 | title = "Write a SQL Optimizer using Egg" 68 | author = "wangrunji" 69 | topic = ["tutorial", "database", "optimizer"] 70 | pub_date = "2023-03-07" 71 | publish = true 72 | featured = true 73 | i18n.zh = { file = "write-a-sql-optimizer-using-egg-zh.md", title = "用 Egg 实现 SQL 优化器" } 74 | -------------------------------------------------------------------------------- /content/issue-3/bridging-async-and-sync-in-rust-zh.md: -------------------------------------------------------------------------------- 1 | > 在同步的 Rust 方法中调用异步代码经常会导致一些问题，特别是对于不熟悉异步 Rust runtime 底层原理的初学者。在本文中，我们将讨论我们遇到的一个特殊问题，并分享我们采取的解决方法的经验。 2 | 3 | ## 背景和问题 4 | 5 | 最近在做我们的 [GreptimeDB](https://greptime.com/) 项目的时候遇到一个关于在同步 Rust 方法中调用异步代码的问题。经过一系列故障排查后，我们弄清了问题的原委，这大大加深了对异步 Rust 的理解，因此在这篇文章中分享给大家，希望能给被相似问题困扰的 Rust 开发者一些启发。 6 | 7 | 我们的整个项目是基于 [Tokio](https://tokio.rs/) 这个异步 Rust runtime 的，它将协作式的任务运行和调度方便地封装在 `.await` 调用中，非常简洁优雅。但是这样也让不熟悉 Tokio 底层原理的用户一不小心就掉入到坑里。 8 | 9 | 我们遇到的问题是，需要在一个第三方库的 trait 实现中执行一些异步代码，而这个 trait 是同步的 `:sweat_smile:`，我们无法修改这个 trait 的定义。 10 | 11 | ```rust 12 | trait Sequencer { 13 | fn generate(&self) -> Vec; 14 | } 15 | ``` 16 | 17 | 我们用一个`PlainSequencer` 来实现这个 trait ，而在实现 `generate` 方法的时候依赖一些异步的调用（比如这里的 `PlainSequencer::generate_async`）： 18 | 19 | ```rust 20 | impl PlainSequencer { 21 | async fn generate_async(&self)->Vec{ 22 | let mut res = vec![]; 23 | for i in 0..self.bound { 24 | res.push(i); 25 | tokio::time::sleep(Duration::from_millis(100)).await; 26 | } 27 | res 28 | } 29 | } 30 | 31 | impl Sequencer for PlainSequencer { 32 | fn generate(&self) -> Vec { 33 | self.generate_async().await 34 | } 35 | } 36 | ``` 37 | 38 | 这样就会出现问题，因为 `generate` 是一个同步方法，里面是不能直接 await 的。 39 | 40 | ```latex 41 | error[E0728]: `await` is only allowed inside `async` functions and blocks 42 | --> src/common/tt.rs:32:30 43 | | 44 | 31 | / fn generate(&self) -> Vec { 45 | 32 | | self.generate_async().await 46 | | | ^^^^^^ only allowed inside `async` functions and blocks 47 | 33 | | } 48 | | |_____- this is not `async` 49 | ``` 50 | 51 | 我们首先想到的是，Tokio 的 runtime 有一个 `Runtime::block_on` 方法，可以同步地等待一个 future 完成。 52 | 53 | ```rust 54 | impl Sequencer for PlainSequencer { 55 | fn generate(&self) -> Vec { 56 | RUNTIME.block_on(async{ 57 | self.generate_async().await 58 | }) 59 | } 60 | } 61 | 62 | #[cfg(test)] 63 | mod tests{ 64 | #[tokio::test] 65 | async fn test_sync_method() { 66 | let sequencer = PlainSequencer { 67 | bound: 3 68 | }; 69 | let vec = sequencer.generate(); 70 | println!("vec: {:?}", vec); 71 | } 72 | } 73 | ``` 74 | 75 | 编译可以通过，但是运行时直接报错： 76 | 77 | ```latex 78 | thread 'tests::test_sync_method' panicked at 'Cannot start a runtime 79 | from within a runtime. This happens because a function (like `block_on`) 80 | attempted to block the current thread while the thread is being used 81 | to drive asynchronous tasks.' 82 | ``` 83 | 84 | 提示不能从一个执行中一个 runtime 直接启动另一个异步 runtime。看来 Tokio 为了避免这种情况特地在 `Runtime::block_on` 入口做了检查。 85 | 既然不行那我们就再看看其他的异步库是否有类似的异步转同步的方法。果然找到一个 `futures::executor::block_on`。 86 | 87 | ```rust 88 | impl Sequencer for PlainSequencer { 89 | fn generate(&self) -> Vec { 90 | futures::executor::block_on(async { 91 | self.generate_async().await 92 | }) 93 | } 94 | } 95 | ``` 96 | 97 | 编译同样没问题，但是运行时代码直接直接 hang 住不返回了。 98 | 99 | ```latex 100 | cargo test --color=always --package tokio-demo --bin tt tests::test_sync_method --no-fail-fast -- --format=json --exact -Z unstable-options --show-output 101 | Compiling tokio-demo v0.1.0 (/Users/lei/Workspace/Rust/learning/tokio-demo) 102 | Finished test [unoptimized + debuginfo] target(s) in 0.39s 103 | Running unittests src/common/tt.rs (target/debug/deps/tt-adb10abca6625c07) 104 | { "type": "suite", "event": "started", "test_count": 1 } 105 | { "type": "test", "event": "started", "name": "tests::test_sync_method" } 106 | 107 | # the execution just hangs here :( 108 | ``` 109 | 110 | 明明 `generate_async` 方法里面只有一个简单的 `sleep()` 调用，但是为什么 future 一直没完成呢？ 111 | 112 | 并且吊诡的是，同样的代码，在 `tokio::test` 里面会 hang 住，但是在 `tokio::main` 中则可以正常执行完毕： 113 | 114 | ```rust 115 | #[tokio::main] 116 | pub async fn main() { 117 | let sequencer = PlainSequencer { 118 | bound: 3 119 | }; 120 | let vec = sequencer.generate(); 121 | println!("vec: {:?}", vec); 122 | } 123 | ``` 124 | 125 | 执行结果： 126 | 127 | ```latex 128 | cargo run --color=always --package tokio-demo --bin tt 129 | Finished dev [unoptimized + debuginfo] target(s) in 0.05s 130 | Running `target/debug/tt` 131 | vec: [0, 1, 2] 132 | ``` 133 | 134 | > 其实当初真正遇到这个问题的时候定位到具体在哪里 hang 住并没有那么容易。真实代码中 async 执行的是一个远程的 gRPC 调用，当初怀疑过是否是 gRPC server 的问题，动用了网络抓包等等手段最终发现是 client 侧的问题。这也提醒了我们在**出现 bug 的时候，抽象出问题代码的执行模式并且做出一个最小可复现的样例（Minimal Reproducible Example）是非常重要的**。 135 | 136 | ## Catchup 137 | 138 | 在 Rust 中，一个异步的代码块会被 [`make_async_expr`](https://github.com/rust-lang/rust/blob/7e966bcd03f6d0fae41f58cf80bcb10566ab971a/compiler/rustc_ast_lowering/src/expr.rs#L585) 编译为一个实现了 `std::future::Future` 的 generator。 139 | 140 | ```Rust 141 | #[tokio::test] 142 | async fn test_future() { 143 | let future = async { 144 | println!("hello"); 145 | }; 146 | 147 | // the above async block won't get executed until we await it. 148 | future.await; 149 | } 150 | ``` 151 | 152 | 而 `.await` 本质上是一个语法糖，则会被 [lower_expr_await](https://github.com/rust-lang/rust/blob/7e966bcd03f6d0fae41f58cf80bcb10566ab971a/compiler/rustc_ast_lowering/src/expr.rs#L717) 编译成类似于下面的一个语法结构： 153 | 154 | ```Rust 155 | // pseudo-rust code 156 | match ::std::future::IntoFuture::into_future() { 157 | mut __awaitee => loop { 158 | match unsafe { ::std::future::Future::poll( 159 | <::std::pin::Pin>::new_unchecked(&mut __awaitee), 160 | ::std::future::get_context(task_context), 161 | ) } { 162 | ::std::task::Poll::Ready(result) => break result, 163 | ::std::task::Poll::Pending => {} 164 | } 165 | task_context = yield (); 166 | } 167 | } 168 | ``` 169 | 170 | 在上面这个去掉了语法糖的伪代码中，可以看到有一个循环不停地检查 generator 的状态是否为已完成（`std::future::Future::poll`）。 171 | 172 | 自然地，必然存在一个组件来做这件事。这里就是 Tokio 和 [async-std](https://async.rs/) 这类异步运行时发挥作用的地方了。Rust 在设计之初就特意将异步的语法（async/await）和异步运行时的实现分开，在上述的示例代码中，poll 的操作是由 Tokio 的 executor 执行的。 173 | 174 | ## 问题分析 175 | 176 | 回顾完背景知识，我们再看一眼方法的实现： 177 | 178 | ```rust 179 | fn generate(&self) -> Vec { 180 | futures::executor::block_on(async { 181 | self.generate_async().await 182 | }) 183 | } 184 | ``` 185 | 186 | 调用 `generate` 方法的肯定是 Tokio 的 executor，那么 block_on 里面的 `self.generate_async().await` 这个 future 又是谁在 poll 呢？ 187 | 188 | 一开始我以为，`futures::executor::block_on` 会有一个内部的 runtime 去负责 `generate_async` 的 poll。于是点进去[代码](https://docs.rs/futures-executor/0.3.6/src/futures_executor/local_pool.rs.html#77-104)（主要是`futures_executor::local_pool::run_executor`这个方法）： 189 | 190 | ```rust 191 | fn run_executor) -> Poll>(mut f: F) -> T { 192 | let _enter = enter().expect( 193 | "cannot execute `LocalPool` executor from within \ 194 | another executor", 195 | ); 196 | 197 | CURRENT_THREAD_NOTIFY.with(|thread_notify| { 198 | let waker = waker_ref(thread_notify); 199 | let mut cx = Context::from_waker(&waker); 200 | loop { 201 | if let Poll::Ready(t) = f(&mut cx) { 202 | return t; 203 | } 204 | let unparked = thread_notify.unparked.swap(false, Ordering::Acquire); 205 | if !unparked { 206 | thread::park(); 207 | thread_notify.unparked.store(false, Ordering::Release); 208 | } 209 | } 210 | }) 211 | } 212 | ``` 213 | 214 | 立刻嗅到了一丝不对的味道，虽然这个方法名为 `run_executor`，但是整个方法里面貌似没有任何 spawn 的操作，只是在当前线程不停的循环判断用户提交的 future 的状态是否为 ready 啊！ 215 | 216 | 这意味着，当 Tokio 的 runtime 线程执行到这里的时候，会立刻进入一个循环，在循环中不停地判断用户的的 future 是否 ready，如果还是 pending 状态，则将当前线程 park 住。 217 | 218 | 假设，用户 future 的异步任务也是交给了当前线程去执行，`futures::executor::block_on` 等待用户的 future ready，而用户 future 等待 `futures::executor::block_on` 释放当前的线程资源，那么不就死锁了？ 219 | 220 | 这个推论听起来很有道理，让我们来验证一下。既然不能在当前 runtime 线程 block，那就重新开一个 runtime block： 221 | 222 | ```rust 223 | impl Sequencer for PlainSequencer { 224 | fn generate(&self) -> Vec { 225 | let bound = self.bound; 226 | futures::executor::block_on(async move { 227 | RUNTIME.spawn(async move { 228 | let mut res = vec![]; 229 | for i in 0..bound { 230 | res.push(i); 231 | tokio::time::sleep(Duration::from_millis(100)).await; 232 | } 233 | res 234 | }).await.unwrap() 235 | }) 236 | } 237 | } 238 | ``` 239 | 240 | 果然可以了。 241 | 242 | ``` 243 | cargo test --color=always --package tokio-demo \ 244 | --bin tt tests::test_sync_method \ 245 | --no-fail-fast -- --format=json \ 246 | --exact -Z unstable-options --show-output 247 | Finished test [unoptimized + debuginfo] target(s) in 0.04s 248 | Running unittests src/common/tt.rs (target/debug/deps/tt-adb10abca6625c07) 249 | vec: [0, 1, 2] 250 | ``` 251 | 252 | 值得注意的是，在 `futures::executor::block_on` 里面，额外使用了一个 `RUNTIME` 来 spawn 我们的异步代码。其原因还是刚刚所说，这个异步任务需要一个 runtime 来驱动状态的变化。 253 | 254 | 如果我们删除 `RUNTIME`，而为 `futures::executor::block_on` 生成一个新的线程，虽然死锁问题得到了解决，但 `tokio::time::sleep` 方法的调用会报错"no reactor is running"，这是因为 Tokio 的功能运作需要一个 runtime： 255 | 256 | ``` 257 | called `Result::unwrap()` on an `Err` value: Any { .. } 258 | thread '' panicked at 'there is no reactor running, must be called from the context of a Tokio 1.x runtime', 259 | ... 260 | ``` 261 | 262 | ### `tokio::main` 和 `tokio::test` 263 | 264 | 在分析完上面的原因之后，“为什么 `tokio::main` 中不会 hang 住而 `tokio::test` 会 hang 住”这个问题也很清楚了，他们两者所使用的的 runtime 并不一样。`tokio::main` 使用的是多线程的 runtime，而 `tokio::test` 使用的是单线程的 runtime，而在单线程的 runtime 下，当前线程被 `futures::executor::block_on` 卡死，那么用户提交的异步代码是一定没机会执行的，从而必然形成上面所说的死锁。 265 | 266 | ## Best practice 267 | 268 | 经过上面的分析，结合 Rust 基于 generator 的协作式异步特性，我们可以总结出 Rust 下桥接异步代码和同步代码的一些注意事项： 269 | 270 | - 将异步代码与同步代码结合使用可能会导致阻塞，因此不是一个明智的选择。 271 | 272 | - 在同步的上下文码调用异步代码时，请使用 `futures::executor::block_on` 并将异步代码 spawn 到另一个专用的 runtime 中执行 ，因为前者会阻塞当前线程。 273 | 274 | - 如果必须从异步的上下文中调用有可能阻塞的同步代码（比如文件 IO 等），则建议使用 `tokio::task::spawn_blocking` 在专门处理阻塞操作的 executor 上执行相应的代码。 275 | 276 | ## 参考 277 | 278 | - [Async: What is blocking?](https://ryhl.io/blog/async-what-is-blocking/) 279 | - [Generators and async/await](https://cfsamson.github.io/books-futures-explained/4_generators_async_await.html) 280 | - [Async and Await in Rust: a full proposal](https://news.ycombinator.com/item?id=17536441) 281 | - [calling futures::executor::block_on in block_in_place may hang](https://github.com/tokio-rs/tokio/issues/2603) 282 | - [tokio@0.2.14 + futures::executor::block_on causes hang](https://github.com/tokio-rs/tokio/issues/2376) 283 | 284 | --- 285 | -------------------------------------------------------------------------------- /content/issue-3/intro.md: -------------------------------------------------------------------------------- 1 | Dear Rustaceans, 2 | 3 | We are excited to announce the release of Issue 3 of Rust Magazine! This issue features a diverse range of articles that explore various aspects of Rust programming, from performance optimization to memory statistics and everything in between. 4 | 5 | In this edition, we have the pleasure of speaking with robjtede, the core maintainer of Actix Web and [deps.rs](https://deps.rs), who shares his insights on the development of Actix Web and the challenges of managing a large open-source project. 6 | 7 | We also have articles on how Rust Search Extension indexes the top 20k crates to provide a instant search experience, and the pursuit of performance in building a JavaScript compiler. 8 | 9 | Additionally, we delve into the comprehensive understanding of unsafe Rust and compare it to Zig's safety features in a thought-provoking article. 10 | 11 | **Another important news is that we opened an [OpenCollective](https://opencollective.com/rustmagazine) account for Rust Magazine to receive donations from the community so that we can pay our contributors for incentivizing them to write more high-quality articles. We are also looking for sponsors to help us pay for the domain and email newsletter costs. If you are interested in helping us, please contact us on [Discord](https://discord.gg/DdwgBuReJe) or email us at [opencollective@rustmagazine.org](mailto://opencollective@rustmagazine.org).** 12 | 13 | We hope that this issue of Rust Magazine provides valuable insights and inspiration to Rustaceans of all levels. Thank you for your continued support, and we look forward to bringing you more exciting content in future issues! 14 | 15 | Best regards, 16 | 17 | The Rust Magazine Team. -------------------------------------------------------------------------------- /content/issue-3/optimizing-compilation-for-databend-zh.md: -------------------------------------------------------------------------------- 1 | ![](/static/issue-3/optimizing-compilation-for-databend/2.png) 2 | 3 | # 背景 4 | 5 | 时至今日，Databend 已经成长为一个大型、复杂、完备的数据库系统。团队维护着数十万行代码，每次发布十几个编译产物，并且还提供基于 Docker 的一些构建工具以改善开发者 / CI 的体验。 6 | 7 | 之前的文章介绍过 [使用 PGO 优化 Databend 二进制构建](https://www.databend.cn/blog/2023/02/24/build-databend-with-pgo) ，用户可以根据自己的工作负载去调优 Databend 的编译。再早些时候，还有一些介绍 Databend [开发环境](https://www.databend.cn/blog/setup-databend-dev-env) 和 [构建](https://www.databend.cn/blog/build-and-run-databend) 的文章。 8 | 9 | 对于 Databend 这样的中大型 Rust 程序而言，编译实在算不上是一件轻松的事情： 10 | 11 | ![](/static/issue-3/optimizing-compilation-for-databend/3.png) 12 | 13 | - 一方面，在复杂的项目依赖和样板代码堆积之下，Rust 的编译时间显得不那么理想，前两年 [Brian Anderson 的文章](https://cn.pingcap.com/blog/rust-compilation-model-calamity) 中也提到“Rust 糟糕的编译时间”这样的描述。 14 | - 另一方面，为了维护构建结果，不得不引入一些技巧来维护编译流水线的稳定，这并不是一件“一劳永逸”的事情，随着 Workflow 复杂性的提高，就不得不陷入循环之中。 15 | 16 | 为了优化编译体验，Databend 陆陆续续做过很多优化工作，今天的文章将会和大家一同回顾 Databend 中改善编译时间的一些优化。 17 | 18 | # 可观测性 19 | 20 | 可观测性并不是直接作用于编译优化的手段，但可以帮助我们识别当前编译的瓶颈在什么地方，从而对症下药。 21 | 22 | ## cargo build --timings 23 | 24 | 这一命令有助于可视化程序的编译过程。 25 | 26 | 在 1.59 或更早版本时可以使用 `cargo +nightly build -Ztimings` 。 27 | 28 | 在浏览器中打开结果 HTML 可以看到一个甘特图，其中展示了程序中各个 crate 之间的依赖关系，以及程序的编译并行程度和代码生成量级。 29 | 通过观察图表，我们可以决定是否要提高某一模块的代码生成单元数目，或者要不要进一步拆解以优化整个编译流程。 30 | 31 | ![](/static/issue-3/optimizing-compilation-for-databend/4.png) 32 | 33 | ## cargo-depgraph 34 | 35 | 这个工具其实不太常用，但可以拿来分析依赖关系。有助于找到一些潜在的优化点，特别是需要替换某些同类依赖或者优化 crates 组织层次的时候。 36 | 37 | ![](/static/issue-3/optimizing-compilation-for-databend/5.png) 38 | 39 | # 无痛优化，从调整配置开始 40 | 41 | 改善编译体验的第一步其实并不是直接对代码动手术，很多时候，只需要变更少数几项配置，就能够获得很大程度上的改善。 42 | 43 | ## Bump, Bump, Booooooooom 44 | 45 | 前面提到过 Rust 团队的成员们也很早就意识到，编译时间目前还并不理想。所以编译器团队同样会有计划去不断进行针对性的优化。经常可以看到在版本更新说明中有列出对编译的一些改进工作。 46 | 47 | ```toml 48 | [toolchain] 49 | channel = "nightly-2023-03-10" 50 | components = ["rustfmt", "clippy", "rust-src", "miri"] 51 | ``` 52 | 53 | 另外，上游项目同样可能会随着时间的推移去改善过去不合理的设计，很多时候这些改进也最终会反映在对编译的影响上。 54 | 55 | ![](/static/issue-3/optimizing-compilation-for-databend/6.png) 56 | 57 | 一个改善编译时间的最简单的优化方式就是始终跟进上游的变更，并且秉着“上游优先”的理念去参与到生态建设之中。Databend 团队从一开始就是 Rust nightly 的忠实簇拥，并且为更新工具链和依赖关系提供了简明的指导。 58 | 59 | ## 缓存，转角遇到 sccache 60 | 61 | 缓存是一种常见的编译优化手段，思路也很简单，只需要把预先构建好的产物存储起来，在下次构建的时候继续拿过来用。 62 | 63 | 早期 Databend 使用 `rust-cache` 这个 action 在 CI 中加速缓存，获得了不错的效果。但是很遗憾，我们不得不经常手动更新 key 来清理缓存，以避免构建时的误判。而且，Rust 早期对增量构建的支持也很差劲，有那么一段时间可能会考虑如何配置流水线来进行一些权衡。 64 | 65 | 随着时间的推移，一切变得不同了起来。 66 | 67 | ```urlpreview 68 | https://github.com/mozilla/sccache 69 | ``` 70 | 71 | 首先是 Sccache 恢复了活力，而 OpenDAL 也成功打入其内部，成为支撑 Rust 编译缓存生态的重要组件，尽管在本地构建时使用它常常无法展现出真正的威力，但是放在 CI 中，还是能够带来很大惊喜的。 72 | 73 | 另一个重要的改变是，Rust 社区意识到增量编译对于 CI 来讲并不能很好 Work 。 74 | 75 | > CI builds often are closer to from-scratch builds, as changes are typically much bigger than from a local edit-compile cycle. For from-scratch builds, incremental adds an extra dependency-tracking overhead. It also significantly increases the amount of IO and the size of ./target, which make caching less effective. 76 | 77 | # 轻装上阵，将冷气传递给每一个依赖 78 | 79 | Rust 生态里面有一个很有意思的项目是 [mTvare6/hello-world.rs](https://github.com/mTvare6/hello-world.rs) ，它尽可能给你展现了如何让一个 Rust 项目变得尽可能糟糕。 80 | 81 | ![](/static/issue-3/optimizing-compilation-for-databend/8.png) 82 | 83 | 特别是： 84 | 85 | > in a few lines of code with few(1092) dependencies 86 | 87 | Rust 自身是不太能很好自动处理这一点的，它需要把所有依赖一股脑下载下来编译一通。所以避免无用依赖的引入就成为一件必要的事情了。 88 | 89 | 最开始的时候，Databend 引入 cargo-udeps 来检查无用的依赖，大多数时候都工作很良好，但最大的缺点在于，每次使用它检查依赖就相当于要编译一遍，在 CI 中无疑是不划算的。 90 | 91 | [sundy-li](https://github.com/sundy-li) 发现了另外一个快速好用的工具，叫做 cargo-machete 。 92 | 93 | ![](/static/issue-3/optimizing-compilation-for-databend/9.png) 94 | 95 | 一个显著的优点是它很快，因为一切只需要简单的正则表达式来处理。而且也支持了自动修复，这意味着我们不再需要挨个检索文件再去编辑。 96 | 97 | 不过 machete 并不是完美的工具，由于只是进行简单的正则处理，有一些情况无法准确识别，不过 ignore 就好了，总体上性价比还是很高的。 98 | 99 | ## 稀疏索引 100 | 101 | 为了确定 crates.io 上存在哪些 crates，Cargo 需要下载并读取 crates.io-index ，该索引位于托管在 GitHub 上的 git 存储库中，其中列出了所有 crates 的所有版本。 102 | 103 | 然而，随着时间推移，由于索引已经大幅增长，初始获取和更新变得很慢。RFC 2789 引入了稀疏索引来改进 Cargo 访问索引的方式，并使用 https://index.crates.io/ 进行托管。 104 | 105 | ```toml 106 | [registries.crates-io] 107 | protocol = "sparse" 108 | ``` 109 | 110 | ## linker 111 | 112 | 如果项目比较大，而且依赖繁多，那么可能在链接时间上会比较浪费。特别是在你只改了几行代码，但编译却花了很久的时候。 113 | 114 | 最简单的办法就是选择比默认链接器更快的链接器。 115 | 116 | ![](/static/issue-3/optimizing-compilation-for-databend/10.png) 117 | 118 | lld 或者 mold 都可以改善链接时间，Databend 最后选择使用 mold 。其实在 Databend 这个量级的程序上，两个链接器的差距并不明显，但是，使用 mold 的一个潜在好处是能够节约一部分编译时候消耗的内存。 119 | 120 | ```toml 121 | [target.x86_64-unknown-linux-gnu] 122 | linker = "clang" 123 | rustflags = ["-C", "link-arg=-fuse-ld=/path/to/mold"] 124 | ``` 125 | 126 | ## 编译相关配置 127 | 128 | 先看一个常见的 split-debuginfo，在 MacOS 上，rustc 会运行一个名为 dsymutil 的工具，该工具会分析二进制文件，然后构建调试信息目录。配置 split-debuginfo，可以跳过 dsymutil ，从而加快构建速度。 129 | 130 | ```toml 131 | split-debuginfo = "unpacked" 132 | ``` 133 | 134 | 另外的一个例子是 codegen-units，Databend 在编译时使用 codegen-units = 1 来增强优化，并且克制二进制体积大小。但是考虑到部分依赖在编译时会有特别长的代码生成时间（因为重度依赖宏），所以需要针对性放开一些限制。 135 | 136 | ```toml 137 | [profile.release.package] 138 | arrow2 = { codegen-units = 4 } 139 | common-functions = { codegen-units = 16 } 140 | databend-query = { codegen-units = 4 } 141 | databend-binaries = { codegen-units = 4 } 142 | ``` 143 | 144 | # 重新思考，更合理的代码组织 145 | 146 | 前面是一些配置上的调整，接下来将会探讨重构对代码编译时间的一些影响。 147 | 148 | ## 拆分到更合理的 crates 规模 149 | 150 | 对于一个大型的 All in One 式的 Crate 而言，拆分 crates 算是一个比较有收益的重构。一方面可以显著改善并行度。另一方面，通过解耦交叉依赖/循环依赖，可以帮助 Rust 更快地处理代码编译。 151 | 152 | ```urlpreview 153 | https://github.com/datafuselabs/databend/issues/6180 154 | ``` 155 | 156 | 同时，还有一个潜在的好处，就是拆分以后，由于代码的边界更为清晰，维护起来也会省力一些。 157 | 158 | ## 单元式测试与集成式测试的界限 159 | 160 | 单元测试的常见组织形式包括在 src 中维护 tests mod ，和在 tests 目录下维护对应的测试代码。 161 | 162 | 根据 Delete Cargo Integration Tests 的建议，Databend 很早就从代码中剥离了所有的单元测试，并组织成类似这样的形式 163 | 164 | ``` 165 | tests/ 166 | it/ 167 | main.rs 168 | foo.rs 169 | bar.rs 170 | ``` 171 | 172 | 这种形式避免将 `tests/` 下面的每个文件都编译成一个单独的二进制文件，从而减轻对编译时间的影响。 173 | 174 | 另外，Rust 编译时处理 tests mod 和 docs tests 也需要花费大量时间，特别是 docs tests 还需要另外构建目标，在采用上面的组织形式之后，就可以在配置中关掉。 175 | 176 | 但是，这种形式并不十分优雅，不得不为所有需要测试的内容设置成 public ，容易破坏代码之间的模块化组织，在使用前建议进行深入评估。 177 | 178 | ## 更优雅的测试方法 179 | 180 | 对应到编译时间上，可以简单认为，单元测试里需要编译的代码越多，编译时间自然就会越慢。 181 | 182 | 另外，对于 Databend 而言，有相当一部分测试都是对输入输出的端到端测试，如果硬编码在单元测试中需要增加更多额外的格式相关的工作，维护也会比较费力。 183 | 184 | ![](/static/issue-3/optimizing-compilation-for-databend/12.png) 185 | 186 | Databend 巧妙运用 golden files 测试和 SQL logic 测试，替换了大量内嵌在单元测试中的 SQL 查询测试和输出结果检查，从而进一步改善了编译时间。 187 | 188 | # 遗珠之憾 189 | 190 | ## cargo nextest 191 | 192 | cargo nextest 让测试也可以快如闪电，并且提供更精细的统计和优雅的视图。Rust 社区中有不少项目通过引入 cargo nextest 大幅改善测试流水线的时间。 193 | 194 | ![](/static/issue-3/optimizing-compilation-for-databend/13.png) 195 | 196 | 但 Databend 目前还无法迁移到这个工具上。一方面，配置相关的测试暂时还不被支持，如果再针对去单独跑 cargo test 还要重新编译。另一方面，有一部分与超时相关的测试设定了执行时间，必须等待执行完成。 197 | 198 | ## cargo hakari 199 | 200 | 改善依赖项的编译，典型的例子其实是 workspace-hack ，将重要的公共依赖放在一个目录下，这样这些依赖就不需要反复编译了。Rust 社区中的 cargo-hakari，可以用来自动化管理 workspace-hack 。 201 | 202 | ![](/static/issue-3/optimizing-compilation-for-databend/14.png) 203 | 204 | Databend 这边则是由于有大量的 common 组件，主要二进制程序都建立在 common 组件上，暗中符合这一优化思路。另外，随着 workspace 支持依赖继承之后，维护压力也得到减轻。 205 | 206 | # 总结 207 | 208 | 这篇文章介绍了 Databend 团队在改善 Rust 项目编译时间上做的一些探索和努力，从配置优化和代码重构这两个角度，提供了一些能够优化编译时间的一些建议。 209 | 210 | # 参考资料 211 | 212 | - [Fast Rust Builds](https://matklad.github.io/2021/09/04/fast-rust-builds.html) 213 | - [Delete Cargo Integration Tests](https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html) 214 | - [Better support of Docker layer caching in Cargo](https://hackmd.io/@kobzol/S17NS71bh) 215 | - [2023-04: 为什么你该试试 Sccache？](https://xuanwo.io/reports/2023-04/) 216 | - [The Rust Performance Book - Compile Times](https://nnethercote.github.io/perf-book/compile-times.html) 217 | - [Cargo Registry 稀疏索引的一些介绍](https://blog.dcjanus.com/posts/cargo-registry-index-in-http/) 218 | -------------------------------------------------------------------------------- /content/issue-3/zine.toml: -------------------------------------------------------------------------------- 1 | 2 | slug = "issue-3" 3 | number = 3 4 | title = "Issue 3" 5 | cover = "/static/ferris/3.jpg" 6 | pub_date = "2023-05-04" 7 | publish = true 8 | 9 | [[article]] 10 | file = "how-rse-index-top-20k-crates.md" 11 | title = "How Rust Search Extension Indexes Top 20k Crates" 12 | author = "folyd" 13 | pub_date = "2023-03-11" 14 | topic = ["engineering"] 15 | publish = true 16 | featured = true 17 | 18 | [[article]] 19 | file = "robjtede.md" 20 | title = "Interview with @robjtede" 21 | author = ["folyd", "robjtede"] 22 | topic = ["interview"] 23 | pub_date = "2023-03-20" 24 | publish = true 25 | featured = true 26 | 27 | [[article]] 28 | file = "javascript-compiler.md" 29 | title = "Pursuit of Performance on Building a JavaScript Compiler" 30 | author = "boshen" 31 | topic = [] 32 | pub_date = "2023-03-22" 33 | publish = true 34 | featured = true 35 | 36 | [[article]] 37 | file = "understand-unsafe-rust.md" 38 | title = "Comprehensive Understanding of Unsafe Rust" 39 | author = "handongzhang" 40 | topic = ["unsafe-rust", "tutorial"] 41 | pub_date = "2023-04-05" 42 | publish = true 43 | featured = true 44 | 45 | [[article]] 46 | file = "is-zig-safer-than-unsafe-rust.md" 47 | title = "Is Zig safer than Unsafe Rust?" 48 | author = "handongzhang" 49 | topic = ["unsafe-rust"] 50 | pub_date = "2023-04-05" 51 | publish = true 52 | featured = true 53 | 54 | [[article]] 55 | file = "optimizing-compilation-for-databend.md" 56 | title = "Optimizing Compilation for Databend" 57 | author = ["PsiACE", "databend"] 58 | topic = ["database", "optimization"] 59 | pub_date = "2023-04-20" 60 | publish = true 61 | featured = true 62 | canonical = "https://databend.rs/blog/2023/04/20/optimizing-compilation-for-databend" 63 | i18n.zh = { file = "optimizing-compilation-for-databend-zh.md", title = "Databend 中的 Rust 编译时间优化小技巧"} 64 | 65 | [[article]] 66 | file = "task-stats-alloc.md" 67 | title = "TaskStatsAlloc: Fine-grained memory statistics in Rust" 68 | author = "tennyzhuang" 69 | topic = ["memory", "allocator"] 70 | pub_date = "2023-04-22" 71 | featured = true 72 | publish = true 73 | 74 | [[article]] 75 | file = "bridging-async-and-sync-in-rust.md" 76 | title = "Bridging Async and Sync Rust Code - A lesson learned while working with Tokio" 77 | author = ["lei", "greptime-team"] 78 | topic = ["async", "database", "tokio"] 79 | pub_date = "2023-03-13" 80 | publish = true 81 | featured = true 82 | canonical = "https://greptime.com/blogs/2023-03-09-bridging-async-and-sync-rust" 83 | i18n.zh = { file = "bridging-async-and-sync-in-rust-zh.md", title = "如何在同步的 Rust 方法中调用异步代码 - Tokio 使用中的几点教训" } 84 | -------------------------------------------------------------------------------- /content/issue-4/a-brief-discussion-on-invariants-in-rust-zh.md: -------------------------------------------------------------------------------- 1 | 每次我们聊到 unsafe 的时候，我们其实总离不了 Invariant 一词。Invariant 直接翻译过来称为“不变式”，在 Rust 的语境下，一般指那些需要保持的性质。比如说 2 | 3 | - 给定一个`x: bool`，这就有一个 invariant：`x`只会是`true`或者`false`； 4 | - 给定一个`p: Pin>`，这里其中一个 invariant：当`T: !Unpin`时`p`所指向的内存不会移动； 5 | - `unsafe fn read(src: *const T) -> T`，这里其中一个 invariant：`src`指向一个已经完全初始化的值。 6 | 7 | Rust 中有各种各样的 invariant，大部分由类型检查来保证，而有些需要人为验证来保证。 8 | 9 | # Invariant 的分类 10 | 11 | 我们可以大致将 Rust 中的 invariant 分为两类，一类是语言层面的 invariant，一类是库层面的 invariant。 12 | 13 | **语言层面的 invariant 又叫 validity**。语言层面的 invariant 对于编译器来说，编译器用这些 invariant 生成正确的代码，也会用这些 invariant 来进行优化。利用 invariant 进行优化的一个很典型的例子就是 niche optimization，比如将`Option<&T>`的大小优化为一个指针大小，其利用的一个 invariant 是`&T`非空，这时就可以利用空的情况去表示`None`，进而压缩了类型的大小。值得注意的是这里还可以做其他优化，在`T`不包含`UnsafeCell`的情况下，`&T`有一个 invariant 是其指向的值是不可变的，所以我们还可以告诉 LLVM，`&T`这个指针是 readonly 的，然后 LLVM 就可以根据这个信息去进行优化。 14 | 15 | 而一旦违反语言层面的 invariant，后果将是**致命**的，这便是所谓的 UB(Undefined Behavior)。编译器不再保证程序的任何行为（产物甚至还不保证是可执行文件）。比如上文中提到的 invariant，当我们硬是把除了`true`和`false`的值，比如`2`强转为了`bool`，会导致未定义行为；当我们`read`一个未初始化的值时，也是一个未定义行为。[这里](https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html#behavior-considered-undefined) 是一些已经明确了的 UB，违反语言的 invariant，就会导致这些 UB（但不仅限于这个列表）。这种 invariant 是**必须**要遵守的 16 | 17 | 不过编译器也可能因为失误，违反了这些 invariant，导致依赖该 invariant 的业务逻辑失效，这属于是编译器的 bug。 18 | 19 | **库/接口层面的 invariant 又称为 safety**。这一般由库的作者所给定。比如指定一个 invariant，`struct Even(u64)`的值必须是偶数，那么使用`Even`这个类型的地方就可以直接引入这个性质去做业务逻辑。 20 | 21 | 对于`Even`的作者，它可以提供这样的接口（假设这个库仅仅提供这几个接口）： 22 | 23 | ```rust 24 | impl Even { 25 | /// 当n为偶数时返回Some 26 | /// 当n不是偶数时返回None 27 | pub fn new(n: u64) -> Option { 28 | if n % 2 == 0 { 29 | Some(Self(n)) 30 | } else { 31 | None 32 | } 33 | } 34 | 35 | /// n必须是偶数 36 | pub fn unchecked_new(n: u64) -> Self { 37 | Self(n) 38 | } 39 | 40 | /// 返回值为偶数 41 | pub fn as_u64(&self) -> &u64 { 42 | &self.0 43 | } 44 | } 45 | ``` 46 | 47 | 对于接口`Even::new`，invariant 由`Even`的作者保证；对于`Even::unchecked_new`，invariant 由调用者保证。与语言层面的 invariant 相比，这个 invariant 就“温和”许多——破坏了这个 invariant，并不会*在这个库中*造成 UB（但同样也会造成程序出现预期以外的行为）。 48 | 49 | `Pin`的 invariant 也是一个十分典型的库层面的 invariant。一般来说这个“不可移动”的 invariant 由`Pin

`的作者来保证，比如`Pin>`提供的所有接口都无法移动其指向的值，而使用者无需担心自己的什么错误使用操作破坏了这一 invariant（前提是在 safe rust 下，由类型系统来保证）。而当我们破坏了`Pin`的 invariant 后，也可能不会立刻 UB，而是在后续的使用中产生 UB（比如自引用结构移动后，仍访问其引用所指向的内存）。 50 | 51 | Rust 中绝大部分的 invariant 都是库层面的 invariant，比如“str 一定是 utf-8 的编码”，“Send 和 Sync”，以及后续引入的一些 IO-safety 等等，都可以划入这类 invariant 中。 52 | 53 | # 有类型证明的 Invariant 54 | 55 | 人总会是要犯错的，我们不能靠人来确保这些 invariant 不会被破坏，那我们是否有自动化检查 invariant 是否被破坏的方案呢？有，Rust 提供了表达力比较强大的类型系统，靠其类型规则就可以各种各样的 invariant。 56 | 57 | 比如根据类型系统的借用规则，引用的借用范围一定在原值的作用域内，可以保证`&T`一定是有效的： 58 | 59 | ```rust 60 | let p: &String; 61 | { 62 | let s = String::from("123"); 63 | p = &s; 64 | } 65 | // 编译错误，因为`p`借用的范围超出了`s`作用域范围 66 | println!("{p:?}"); 67 | ``` 68 | 69 | 每次编译的时候，都会进行类型检查，当代码不满足类型规则时，编译器就将其视为不合法的程序，不允许其编译。通过类型规则来保证程序各种各样的程序中 invariant。我们将有类型系统证明的那一部分 rust 称之为 safe rust。 70 | 71 | Rust 有个很重要的原则，**在 safe rust 下，一个库所有公开的接口中的 invariant 不会被破坏。**这就是所谓的 soundness。比如说刚刚`Even`其实并不 sound，因为提供了`Even::unchecked_new`这个接口，可以在 type check 的情况下破坏掉 Even 的 invariant。而如果不提供这个接口，这个库就 sound 了，因为在类型系统的加持下，你无法构造出一个非偶数的`Even`，从而保持了 invariant。 72 | 73 | 当然，有些库则是“几乎”严格地遵守了这个原则，比如说标准库。我们在只使用 std 的情况下，我们可以更近一步说，**在 safe rust 下，不会出现 UB**。有类型系统的语言很多，但并不是所有的语言都有这么强的保障，比如说 cpp，稍不注意，写个死循环就 UB 了。 74 | 75 | ![cpp loop ub](/static/issue-4/cpp-loop-ub.jpg) 76 | 77 | 另外 Invariant 不仅仅要由程序员来保证，而是所有的参与方都要努力保证的一个事实，谁违反了就是谁的 bug。这个锅谁来背很重要。Rust 的模块系统（指的是 crate），也在这方面也起到了至关重要的作用，**使得我们无法从外部破坏在库内部所保证的 invariant**。 78 | 79 | 这条规则就是所谓的 coherence rules，这条规则**不允许为第三方的类型实现第三方的 trait**。举个例子，一个库实现了一个指针`Ptr`，大概只提供这些方法： 80 | 81 | ```rust 82 | impl Deref for Ptr { 83 | type Target = T; 84 | // ... 85 | } 86 | 87 | impl Ptr { 88 | // 因为没有实现`DerefMut`，所以`Pin>`没有任何方法可以移动`T` 89 | pub fn pin(t: T) -> Pin> { ... } 90 | 91 | pub fn new(t: T) -> Ptr { ... } 92 | 93 | // 被`Pin`前可以访问`&mut T` 94 | pub fn borrow_mut(&mut self) -> &mut T { ... } 95 | } 96 | ``` 97 | 98 | 如果没有 coherence rules 的话，我们可以为`Ptr`实现`DerefMut`，从而破坏`Pin`的 invariant（这个 invariant 原本在库里已经是被保证了的）： 99 | 100 | ```rust 101 | impl DerefMut for Ptr { 102 | fn deref_mut(&mut self) -> &mut Unmovable { 103 | let mut tmp = Box::new(Unmovale); 104 | // 将Unmovable移动了出来 105 | swap(self.borrow_mut(), &mut tmp); 106 | Box::leak(tmp) 107 | } 108 | } 109 | 110 | let unmovable = Unmovable::new(); 111 | let mut ptr: Pin> = Ptr::pin(unmovable); 112 | // Pin::as_mut() 调用了 Ptr::deref_mut() 使得unmovable移动了 113 | // 破坏了`Pin`的invariant，unsoundnesss! 114 | // 我们可以根据这个漏洞来构造出UB。 115 | ptr.as_mut(); 116 | ``` 117 | 118 | 事实上，`Pin`曾经也在标准库中发生过同样的问题。。。（`&T`, `&mut T`, `Box`, `Pin

`可以打破 coherence rules，所以能直接构造出来这样的漏洞，~~但后续修复了~~ [并没有](https://github.com/rust-lang/rust/issues/85099)） 119 | 120 | 而现在因为 coherence rule 你无法这么做——只要你的 invariant 在本地已经被保证了的，就不能被第三方破坏。所以，在 Rust 中可以严格地划分责任，究竟是谁破坏了 invariant：如果使用者正常使用的情况出了 bug，那么是库作者的 bug，因为正常使用是无法破坏库内部的 invariant 的。 121 | 122 | （但我很好奇，haskell，swift 这些可以随意为第三方库实现第三方的 typeclass(protocol)的语言是如何保证自己的库不被下游或者其它第三方库所影响的） 123 | 124 | # Invariant 与 unsafe 的关系 125 | 126 | 不过 Rust 的类型系统并不是万能的，有些 invariant 是无法靠类型系统来证明的，其中就包括一些 validity（语言级 invariant），这些 invariant 就需要程序员自己去保证了。其它 invariant 破坏了，可能影响比较小，但 validity 不行，太重要了，一碰就炸，所以 rust 给了一个`unsafe`的关键字来表示 validity 相关的问题。 127 | 128 | - `unsafe fn`：表示一个接口有一些 invariant，如果调用者不保证就有可能破坏掉一些 validity，从而发生 UB。这些 invariant 则可以当做**公理**直接在接口的实现中使用。 129 | - `unsafe {}`：则表示确保已经遵守了内部的一些 invariant。rust 会**完全**信任程序员给的承诺。 130 | - `unsafe trait`/`unsafe impl`类似。 131 | 132 | 于是 rust 被 unsafe 一分为二，safe rust 是有 rust 类型系统证明的一部分（出问题责任在编译器），unsafe rust 则是需要程序员自己证明安全的一部分（出问题责任在程序员）。 133 | 134 | 什么应该接口(`fn`和`trait`)标记为`unsafe`，在 rust 中很克制，并不是所有类型系统无法证明的 invariant 都应该标记。**只有那些和 validity 相关的 invariant，以及 FFI 才应该标记为 unsafe**，而且是能不标就不标。比如说`UnwindSafe`就没有标为 unsafe，因为同在标准库内，没有东西会因为不 unwind 会产生 UB，而使用标准库且不使用任何 unsafe 的情况下，也不会产生 UB，所以就没有标。（但我更愿意将这种无法在 safe 下确切证明的性质，称为 hint，而非 invariant，因为没有人会为其负责；就像一开始定义的`Even`一样） 135 | 136 | FFI 是一个比较特别的情况，它与 validity 不一样，它的正确性不由 rust 的编译器保证，因为 Rust 完全不知道 FFI 另一边的信息。但 FFI 的另一侧可以做任何事情，所以理论上执行 FFI 是永远都不安全的。所以这时候就要求程序员知道 FFI 干了啥， `unsafe { call_ffi() }`的含义则变成，“我已知悉调用 FFI 所带来的后果，并愿意接受其带来的所有影响”。 137 | 138 | 除了什么才应该标`unsafe`以外，我们也要求对`unsafe`的内容进行严格的审查。 139 | 140 | 首先是对接口上的`unsafe`对应的 invariant 的检查。比如说 invariant 是否充分（满足 invariant 就安全了吗）？比如 invariant 间是否矛盾（`x: u64`却要求`x < 0`，这就没法实现）？ 141 | 142 | 然后是严格检查`unsafe {}`/`unsafe impl`里的条件是否满足。有些东西是不能依赖的，那些没有被证明且没有被标记为 unsafe 的 invariant，比如 143 | 144 | - 前文的`Even`，声称的为偶数 145 | - 对于一个未知的`T: UnwindSafe`所“声称”的 unwindsafe 146 | - 对于一个未知的`T: Ord`，所“声称”的全序 147 | 148 | 因为这些都能在 safe 的情况下违反掉这些接口声称的 invariant，但在 safe 的情况下我们不能”追责“。（again，我觉得这种就应该叫 hint）一般来说可以依赖的是： 149 | 150 | - **具体类型**的一些性质。比如说`u64: Ord`满足全序，这一点你是可以确保的。这时候具体类型就相当于一个白盒，你可以知道这个类型的所有性质。 151 | - 通过 unsafe 声明的 invariant。 152 | 153 | 人是不可靠的。那么我们应该如何检查我们是否违反了 validity 呢？有工具，但不多。目前我们可以通过 MIRI（一个 rust 的解释器，目前可以理解为这是 rust 的标准行为）去运行你的 rust 程序（仅限于纯 rust 代码）。MIRI 只维护 rust 所有正确行为的状态，当你的程序触发 UB 的时候，MIRI 就会报错。但这是有限制的，MIRI 只能告诉你 UB 了，但无法理解你是违反了那一个 invariant 而 UB 了；然后 MIRI 也不可能跑完所有情况；就算跑完所有情况发现没有 UB，也不能证明你提供的接口是 sound 的。（就像测试一样） 154 | 155 | 还有一些功能有限的形式化证明工具，比如[flux](https://flux-rs.github.io/flux/index.html)，这里就不再展开。 156 | 157 | unsafe 算是 Rust 中一大特色了。如果没有 unsafe，但又要完全安全，就会有这些情况： 158 | 159 | 1. 所有 validity 都可以用类型证明——要求类型系统足够强大（dt 起步，可以证明下标合法），可以表达复杂条件。代价就是类型系统空前复杂，对于使用者心智负担很重，对于实现者也很难证明类型系统的可靠性，另外基本都会碰到 undecidable 的理论天花板。 160 | 2. 所有 validity 都有运行时的动态检查，或者说以运行时的代价消除 UB——这就会在各种地方引入不可避免的开销，性能难以做到极致。甚至限制用户做一些底层的操作（每次都要喷一遍，hs 没法自己通过语言自身提供的语法定义数组，只能靠运行时的开洞或 FFI） 161 | 162 | # 再补充一点点 163 | 164 | 1. rust 的类型系统还没被证明是可靠的，也就是说一些规则可能有矛盾（[不一致](https://github.com/rust-lang/rust/issues/25860)），所以现阶段对 invariant 的证明也不一定可靠。 165 | 166 | 2. rust 的标准库也还没被证明是 soundness 的，也就是说有些接口的 invariant 有可能会被破坏。 167 | 168 | 3. rust 的绝大数第三方库没有验证是否 soundness，尤其是内部用到 unsafe 的库。 169 | 170 | 4. rust 的编译器也有可能破坏 invariant，进行错误的优化，让我们在 safe rust 下构造出[段错误](https://play.rust-lang.org/?version=stable&mode=release&edition=2021&gist=2179a4f4567edd276818c7869aac7e60)。 171 | 172 | 5. 运行 rust 程序的平台也有可能破坏 invariant，比如说`proc/self/mem`可以破坏内存独占的 invariant，修改内存。但从实践意义来说，rust 接受这种 cornercase。 173 | 174 | 以后可能 1 和 2 会得到解决，但是 345 看起来是没法避免的，也就是说 rust 的安全也是有限制的，有些关键的地方还是得靠人来决定，但人永远是不可靠的。让我想起了 linus 关于安全所说的一句话： 175 | 176 | > So? 177 | > 178 | > You had a bug. Shit happens. 179 | 180 | 不过，尽管如此，rust 的安全性仍然是有统计学意义论支撑，而且出了问题责任也分明。这同样十分有意义。 181 | 182 | --- 183 | 184 | 参考： 185 | 186 | - 187 | - 188 | - 189 | -------------------------------------------------------------------------------- /content/issue-4/how-tokio-schedule-tasks-zh.md: -------------------------------------------------------------------------------- 1 | Future 是 Rust 中实现异步的基础，代表一个异步执行的计算任务，与其他语言不同的是，这个计算并不会自动在后台执行，需要主动去调用其 poll 方法。Tokio 是社区内使用最为广泛的异步运行时，它内部采用各种措施来保证 Future 被公平、及时的调度执行。但是由于 Future 的执行是协作式，因此在一些场景中会不可避免的出现 Future 被饿死的情况。 2 | 3 | 下文就将结合笔者在开发 CeresDB 时遇到的一个问题，来分析 Tokio 调度时可能产生的问题，作者水平有限，不足之处请读者指出。 4 | 5 | ## 问题背景 6 | 7 | CeresDB 是一个面向云原生打造的高性能时序数据库，存储引擎采用的是类 LSM 架构，数据先写在 memtable 中，达到一定阈值后 flush 到底层（例如：S3），为了防止小文件过多，后台还会有专门的线程来做合并。 8 | 9 | 在生产环境中，笔者发现一个比较诡异的问题，每次当表的合并请求加剧时，表的 flush 耗时就会飙升，flush 与合并之间并没有什么关系，而且他们都运行在不同的线程池中，为什么会造成这种影响呢？ 10 | 11 | ## 原理分析 12 | 13 | 为了调查清楚出现问题的原因，我们需要了解 Tokio 任务调度的机制，Tokio 本身是一个基于事件驱动的运行时，用户通过 `spawn` 来提交任务，之后 Tokio 的调度器来决定怎么执行，最常用的是[多线程版本的调度器](https://docs.rs/tokio/latest/tokio/runtime/index.html#multi-thread-scheduler)，它会在固定的线程池中分派任务，每个线程都有一个 local run queue，简单来说，每个 worker 线程启动时会进入一个 loop，来依次执行 run queue 中的任务。如果没有一定的策略，这种调度方式很容易出现不均衡的情况，Tokio 使用 work steal 来解决，当某个 worker 线程的 run queue 没有任务时，它会尝试从其他 worker 线程的 local queue 中“偷”任务来执行。 14 | 15 | 在上面的描述中，任务时最小的调度单元，对应代码中就是 `await` 点，Tokio 只有在运行到 `await` 点时才能够被重新调度，这是由于 future 的执行其实是个状态机的执行，例如： 16 | 17 | ```rs 18 | async move { 19 | fut_one.await; 20 | fut_two.await; 21 | } 22 | ``` 23 | 24 | 上面的 async 代码块在执行时会被转化成类似如下形式： 25 | 26 | ```rs 27 | // The `Future` type generated by our `async { ... }` block 28 | struct AsyncFuture { 29 | fut_one: FutOne, 30 | fut_two: FutTwo, 31 | state: State, 32 | } 33 | 34 | // List of states our `async` block can be in 35 | enum State { 36 | AwaitingFutOne, 37 | AwaitingFutTwo, 38 | Done, 39 | } 40 | 41 | impl Future for AsyncFuture { 42 | type Output = (); 43 | 44 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { 45 | loop { 46 | match self.state { 47 | State::AwaitingFutOne => match self.fut_one.poll(..) { 48 | Poll::Ready(()) => self.state = State::AwaitingFutTwo, 49 | Poll::Pending => return Poll::Pending, 50 | } 51 | State::AwaitingFutTwo => match self.fut_two.poll(..) { 52 | Poll::Ready(()) => self.state = State::Done, 53 | Poll::Pending => return Poll::Pending, 54 | } 55 | State::Done => return Poll::Ready(()), 56 | } 57 | } 58 | } 59 | } 60 | ``` 61 | 62 | 在我们通过 `AsyncFuture.await` 调用时，相当于执行了 `AsyncFuture::pool` 方法，可以看到，只有状态切换（返回 `Pending` 或 `Ready`） 时，执行的控制权才会重新交给 worker 线程，如果 `fut_one.poll()` 中包括堵塞性的 API，那么 worker 线程就会一直卡在这个任务中。此时这个 worker 对应的 run queue 上的任务很可能得不到及时调度，尽管有 work steal 的存在，但应用整体可能有较大的长尾请求。 63 | 64 | ![图 1](/static/issue-4/ceresdb-task1.png) 65 | ![图 2](/static/issue-4/ceresdb-task2.png) 66 | 67 | 在上图中，有四个任务，分别是： 68 | 69 | - Task0、Task1 是混合型的，里面既有 IO 型任务，又有 CPU 型任务 70 | - Task2、Task3 是单纯的 CPU 型任务 71 | 72 | 执行方式的不同会导致任务的耗时不同， 73 | 74 | - 图一方式，把 CPU 型与 IO 型任务混合在一个线程执行，那么最差情况下 Task0、Task1 的耗时都是 35ms 75 | - 图二方式，把 CPU 型与 IO 型任务区分开，分两个 runtime 去执行，在这种情况下，Task0、Task1 的耗时都是 20ms 76 | 77 | 因此一般推荐通过 `spawn_blocking` 来执行可能需要长时间执行的任务，这样来保证 worker 线程能够尽快的获取控制权。 78 | 79 | 有了上面的知识，再来尝试分析本文一开始提出的问题，flush 与合并操作的具体内容可以用如下伪代码表示： 80 | 81 | ```rs 82 | async fn flush() { 83 | let input = memtable.scan(); 84 | let processed = expensive_cpu_task(); 85 | write_to_s3(processed).await; 86 | } 87 | 88 | async fn compact() { 89 | let input = read_from_s3().await; 90 | let processed = expensive_cpu_task(input); 91 | write_to_s3(processed).await; 92 | } 93 | 94 | runtime1.block_on(flush); 95 | runtime2.block_on(compact); 96 | ``` 97 | 98 | 可以看到，flush 与 compact 均存在上面说的问题， `expensive_cpu_task` 可能会卡主 worker 线程，进而影响读写 s3 的耗时， s3 的客户端用的是 [object_store](https://docs.rs/object_store/latest/object_store/)，它内部使用 [reqwest](https://docs.rs/reqwest/latest/reqwest/) 来进行 HTTP 通信。 99 | 100 | 如果 flush 和 compact 运行在一个 runtime 内，基本上就不需要额外解释了，但是这两个运行在不同的 runtime 中，是怎么导致相互影响的呢？笔者专门写了个模拟程序来复现问题，代码地址： 101 | 102 | - https://github.com/jiacai2050/tokio-debug 103 | 104 | 模拟程序内有两个 runtime，一个来模拟 IO 场景，一个来模拟 CPU 场景，所有请求按说都只需要 50ms 即可返回，由于 CPU 场景有堵塞操作，所以实际的耗时会更久，IO 场景中没有堵塞操作，按说都应该在 50ms 左右返回，但多次运行中，均会有一两个任务耗时在 1s 上下，而且主要集中在 io-5、io-6 这两个请求上。 105 | 106 | ```bash 107 | [2023-08-06T02:58:49.679Z INFO foo] io-5 begin 108 | [2023-08-06T02:58:49.871Z TRACE reqwest::connect::verbose] 93ec0822 write (vectored): b"GET /io-5 HTTP/1.1\r\naccept: */*\r\nhost: 127.0.0.1:8080\r\n\r\n" 109 | [2023-08-06T02:58:50.694Z TRACE reqwest::connect::verbose] 93ec0822 read: b"HTTP/1.1 200 OK\r\nDate: Sun, 06 Aug 2023 02:58:49 GMT\r\nContent-Length: 14\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nHello, \"/io-5\"" 110 | [2023-08-06T02:58:50.694Z INFO foo] io-5 cost:1.015695346s 111 | ``` 112 | 113 | 上面截取了一次运行日志，可以看到 `io-5` 这个请求从开始到真正发起 HTTP 请求，已经消耗了 192ms（871-679），从发起 HTTP 请求到得到响应，经过了 823ms，正常来说只需要 50ms 的请求，怎么会耗时将近 1s 呢？ 114 | 115 | 给人的感觉像是 reqwest 实现的连接池出了问题，导致 IO 线程里面的请求在等待 cpu 线程里面的连接，进而导致了 IO 任务耗时的增加。通过在构造 reqwest 的 Client 时设置 `pool_max_idle_per_host` 为 0 来关闭连接复用后，IO 线程的任务耗时恢复正常。 116 | 117 | 笔者[在这里](https://github.com/seanmonstar/reqwest/discussions/1935)向社区提交了这个 issue，但还没有得到任何答复，所以根本原因还不清楚。不过，通过这个按理，笔者对 Tokio 如何调度任务有了更深入的了解，这有点像 Node.js，绝不能阻塞调度线程。而且在 CeresDB 中，我们是通过添加一个专用运行时来隔离 CPU 和 IO 任务，而不是禁用链接池来解决这个问题，感兴趣的读者可以参考 [PR #907](https://github.com/CeresDB/ceresdb/pull/907/files)。 118 | 119 | ## 总结 120 | 121 | 上面通过一个 CeresDB 中的生产问题，用通俗易懂的语言来介绍了 Tokio 的调度原理，真实的情况当然要更加复杂，Tokio 为了实现最大可能的低延时做了非常多细致的优化，感兴趣的读者可以参考下面的文章来了解更多内容： 122 | 123 | - [Making the Tokio scheduler 10x faster](https://tokio.rs/blog/2019-10-scheduler) 124 | - [Task scheduler 源码解读](https://tony612.github.io/tokio-internals/03_task_scheduler.html) 125 | - [走进 Tokio 的异步世界](https://xie.infoq.cn/article/5694ce615d1095cf6e1a5d0ae) 126 | 127 | 最后，希望读者能够通过本文的案例，意识到 Tokio 使用时可能存在的潜在问题，尽量把 CPU 等会堵塞 worker 线程的任务隔离出去，减少对 IO 型任务的影响。 128 | 129 | ## 扩展阅读 130 | 131 | - [Making the Tokio scheduler 10x faster](https://tokio.rs/blog/2019-10-scheduler) 132 | - [One bad task can halt all executor progress forever #4730](https://github.com/tokio-rs/tokio/issues/4730) 133 | - [2023 Rust China Conf -- CeresDB Rust 生产实践](https://github.com/CeresDB/community/blob/main/slides/20230617-Rust-China-Conf.pptx) 134 | -------------------------------------------------------------------------------- /content/issue-4/how-tokio-schedule-tasks.md: -------------------------------------------------------------------------------- 1 | `Future`s, representing an asynchronous computation task, are the basis for implementing asynchronous tasks in Rust. Unlike other languages, the computation does not automatically execute in the background, it needs to actively call its `poll` method. Tokio is the most widely used asynchronous runtime in the community. It uses various tips internally to ensure that `Future`s are scheduled and executed fairly and timely. However, since the execution of a `Future` is cooperative, starvation can inevitably occur in some scenarios. 2 | 3 | This article will analyze a problem I encountered while developing CeresDB and discuss issues that may arise with Tokio scheduling. Please point out any inadequacies as my knowledge is limited. 4 | 5 | # Background 6 | 7 | [CeresDB](https://github.com/CeresDB/ceresdb) is a high-performance time series database designed for cloud-native. The storage engine uses an LSM-like architecture where data is first written to memtable, and when some threshold is reached, it is flushed to the underlying storage(e.g. S3). To prevent too many small files, there is also a background thread that does compaction. 8 | 9 | In production, I found a strange problem. Whenever compaction requests increased for a table, the flush time would spike even though flush and compaction run in different thread pools and have no direct relationship. Why did they affect each other? 10 | 11 | # Analysis 12 | 13 | To investigate the root cause, we need to understand Tokio's task scheduling mechanism. Tokio is an event-driven, non-blocking I/O platform for writing asynchronous applications, users submit tasks via `spawn`, then Tokio's scheduler decides how to execute them, most of time using a multi-threaded scheduler. 14 | 15 | Multi-threaded scheduler dispatches tasks to a fixed thread pool, each worker thread has a local run queue to save pending tasks. When starts, each worker thread will enter a loop to sequentially fetch and execute tasks in its run queue. Without a strategy, this scheduling can easily become imbalanced. Tokio uses work stealing to address this - when a worker's run queue is empty, it tries to "steal" tasks from other workers' queues to execute. 16 | 17 | In the above, a task is the minimum scheduling unit, corresponding to `await` in our code. Tokio can only reschedule on reaching an await, since future execution is a state machine like below: 18 | 19 | ```rs 20 | async move { 21 | fut_one.await; 22 | fut_two.await; 23 | } 24 | ``` 25 | 26 | This async block is transformed to something below when get executed: 27 | 28 | ```rs 29 | // The `Future` type generated by our `async { ... }` block 30 | struct AsyncFuture { 31 | fut_one: FutOne, 32 | fut_two: FutTwo, 33 | state: State, 34 | } 35 | 36 | // List of states our `async` block can be in 37 | enum State { 38 | AwaitingFutOne, 39 | AwaitingFutTwo, 40 | Done, 41 | } 42 | 43 | impl Future for AsyncFuture { 44 | type Output = (); 45 | 46 | fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { 47 | loop { 48 | match self.state { 49 | State::AwaitingFutOne => match self.fut_one.poll(..) { 50 | Poll::Ready(()) => self.state = State::AwaitingFutTwo, 51 | Poll::Pending => return Poll::Pending, 52 | } 53 | State::AwaitingFutTwo => match self.fut_two.poll(..) { 54 | Poll::Ready(()) => self.state = State::Done, 55 | Poll::Pending => return Poll::Pending, 56 | } 57 | State::Done => return Poll::Ready(()), 58 | } 59 | } 60 | } 61 | } 62 | ``` 63 | 64 | When we call `AsyncFuture.await`, `AsyncFuture::poll` get executed. We can see that control flow returns to the worker thread only on state transitions (Pending or Ready). If `fut_one.poll()` contains blocking API, the worker thread will be stuck on that task. Tasks on that worker's run queue are likely to not be scheduled timely despite work stealing. Let me explain this in more details: 65 | 66 | ![Figure 1](/static/issue-4/ceresdb-task1.png) 67 | ![Figure 2](/static/issue-4/ceresdb-task2.png) 68 | 69 | In the above figure, there are four tasks: 70 | 71 | - Task0, Task1 are hybrid, which contain both IO and CPU work 72 | - Task2 and Task3 are purely CPU-bound tasks 73 | 74 | The different execution strategy will lead to different time consumptions of the tasks. 75 | 76 | - In Figure 1, CPU and IO tasks are mixed in one thread, Task0 and Task1 will take 35ms in the worst case. 77 | - In Figure 2, CPU and IO tasks are separated and executed in two runtimes, in this case, Task0 and Task1 both take 20ms. 78 | 79 | Therefore, it is generally recommended to use `spawn_blocking` to execute tasks that may take a long time to execute in Tokio, so that the worker thread can gain control as quickly as possible. 80 | 81 | With the above knowledge in mind, let's try to analyze the question posed at the beginning of this article. The specifics of the flush and merge operations can be expressed in the following pseudo-code: 82 | 83 | ```rs 84 | async fn flush() { 85 | let input = memtable.scan(); 86 | let processed = expensive_cpu_task(); 87 | write_to_s3(processed).await; 88 | } 89 | 90 | async fn compact() { 91 | let input = read_from_s3().await; 92 | let processed = expensive_cpu_task(input); 93 | write_to_s3(processed).await; 94 | } 95 | 96 | runtime1.block_on(flush); 97 | runtime2.block_on(compact); 98 | ``` 99 | 100 | As we can see, both flush and compact have the above issue - `expensive_cpu_task` can block the worker thread, affecting s3 read/write times. The s3 client uses [object_store](https://docs.rs/object_store/latest/object_store/) which uses [reqwest](https://docs.rs/reqwest/latest/reqwest/) for HTTP. 101 | 102 | If flush and compact run in the same runtime, there is no further explanation needed. But how do they affect each other when running in different runtimes? I wrote a [minimal program](https://github.com/jiacai2050/tokio-debug) to reproduce this. 103 | 104 | This program has two runtimes, one for IO and one for CPU scenarios. All requests should take only 50ms but actual time is longer due to blocking API used in the CPU scenario. IO has no blocking so should cost around 50ms, but some tasks, especially `io-5` and `io-6`, their cost are roughly 1s: 105 | 106 | ```bash 107 | [2023-08-06T02:58:49.679Z INFO foo] io-5 begin 108 | [2023-08-06T02:58:49.871Z TRACE reqwest::connect::verbose] 93ec0822 write (vectored): b"GET /io-5 HTTP/1.1\r\naccept: */*\r\nhost: 127.0.0.1:8080\r\n\r\n" 109 | [2023-08-06T02:58:50.694Z TRACE reqwest::connect::verbose] 93ec0822 read: b"HTTP/1.1 200 OK\r\nDate: Sun, 06 Aug 2023 02:58:49 GMT\r\nContent-Length: 14\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nHello, \"/io-5\"" 110 | [2023-08-06T02:58:50.694Z INFO foo] io-5 cost:1.015695346s 111 | ``` 112 | 113 | The above log shows `io-5` already took 192ms before the HTTP request, and cost 823ms from request to response, which should only be 50ms. It seems like the connection pool in `reqwest` have some weird issues, that's the IO thread is waiting for connections held by the CPU thread, increasing cost of IO task. Setting `pool_max_idle_per_host` to 0 to disable connection reuse solve the problem. 114 | 115 | I filed this issue [here](https://github.com/seanmonstar/reqwest/discussions/1935), but haven't get any answers yet, so the root cause here is still unclear. However I gain better understand how Tokio schedule tasks, it's kind of like Node.js, we should never block schedule thread. In CeresDB we fix it by adding a dedicated runtime to isolate CPU and IO tasks instead of disabling the pool, see [here](https://github.com/CeresDB/ceresdb/pull/907/files) if curious. 116 | 117 | # Summary 118 | 119 | Through a CeresDB production issue, this post explains Tokio scheduling in simple terms. Real situations are of course more complex, users need to analyze carefully how async code gets scheduled before finding a solution. Also, Tokio makes many detailed optimizations for lowest possible latency, interested readers can learn more from links below: 120 | 121 | - [Making the Tokio scheduler 10x faster](https://tokio.rs/blog/2019-10-scheduler) 122 | - [One bad task can halt all executor progress forever #4730](https://github.com/tokio-rs/tokio/issues/4730) 123 | - [Using Rustlang's Async Tokio Runtime for CPU-Bound Tasks](https://www.influxdata.com/blog/using-rustlangs-async-tokio-runtime-for-cpu-bound-tasks/) 124 | 125 | Finally, I hope readers can realize potential issues when using Tokio. As a rule of thumb, try to isolate blocking tasks to minimize impact on IO tasks. 126 | -------------------------------------------------------------------------------- /content/issue-4/jhpratt.md: -------------------------------------------------------------------------------- 1 | Today, we are glad to announce our latest interview with **Jacob Pratt**, the core maintainer of the [time](https://crates.io/crates/time) crate and a contributor to the Rust compiler and standard library. We are grateful for Jacob's time and expertise, and we look forward to sharing his insights with you and hope you enjoy our interview with **Jacob Pratt**. 2 | 3 | # Introduction 4 | 5 | **Introduce yourself and share a bit about your background with Rust. When did you start learning Rust, and what inspired you to do so?** 6 | 7 | ```quote 8 | author="Jacob" 9 | avatar="/static/avatar/jhpratt.png" 10 | content=""" 11 | Hello! For those that don't know me, I am **Jacob Pratt** (GitHub [@jhpratt](https://github.com/jhpratt)). I've been maintaining the `time` crate and contributing to `rustc` and `std` for a few years now. To the best of my recollection, I've been using Rust since October 2016, though I had played around with it somewhat before then. At that time, I was working on a backend for a website in JavaScript, but could not convince myself that a non-admin user would not be able to access the admin endpoints. This is because JavaScript is weakly typed, so any effort to verify this would have taken significant work. I had heard of this relatively new language "Rust", and decided to check it out. As Rust is strongly typed, my concerns were immediately alleviated, as I could simply require an instance of an admin user as a parameter. As a nice bonus, the backend got significantly faster in the rewrite! 12 | 13 | In the nearly seven years since I started using Rust, I have written an immense amount of code in it. A large part of this is maintenance of `time` and contributing to the Rust compiler and standard library. 14 | """ 15 | ``` 16 | 17 | # `time` crate 18 | 19 | **How did you get involved with the `time` crate, and what has your experience been like as a core maintainer?** 20 | 21 | ```urlpreview 22 | https://github.com/time-rs/time 23 | ``` 24 | 25 | ```quote 26 | author="Jacob" 27 | avatar="/static/avatar/jhpratt.png" 28 | content=""" 29 | Odd as it may seem for such a popular crate, I just asked. `time` was officially deprecated when I took it over, so the people with publishing permission and GitHub access were willing to hand it over once I had written sufficient code (so as to avoid malicious intent). 30 | 31 | My experience as a maintainer has been largely positive. Of course it's impossible to please everyone, but I believe that on the whole I have written a solid crate that is widely used without much issue. 32 | """ 33 | ``` 34 | 35 | **What are the differences between `chrono` and `time` crate? Can you offer any advice to help users choose which crate to use for their specific needs?** 36 | 37 | ```quote 38 | author="Jacob" 39 | avatar="/static/avatar/jhpratt.png" 40 | content=""" 41 | Overall, [`time`](https://crates.io/crates/time) and [`chrono`](https://crates.io/crates/chrono) provide a generally similar API. For the best comparison, I honestly suggest that people just try them to see which one they like more. However, I'll try to give a relatively simple overview. 42 | 43 | `time` has three core types: [`Date`](https://docs.rs/time/latest/time/struct.Date.html), [`Time`](https://docs.rs/time/latest/time/struct.Time.html), and [`UtcOffset`](https://docs.rs/time/latest/time/struct.UtcOffset.html). These can be combined to form a [`PrimitiveDateTime`](https://docs.rs/time/latest/time/struct.PrimitiveDateTime.html) (consisting of a `Date` and `Time`) or an [`OffsetDateTime`](https://docs.rs/time/latest/time/struct.OffsetDateTime.html) (consisting of all three). `chrono` is similar in having [`NaiveDate`](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDate.html), [`NaiveTime`](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveTime.html), [`FixedOffset`](https://docs.rs/chrono/latest/chrono/offset/struct.FixedOffset.html), [`NaiveDateTime`](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDateTime.html), and [`DateTime`](https://docs.rs/chrono/latest/chrono/struct.DateTime.html), respectively. A quick check shows that the sizes of all types are the same, with the exception of `UtcOffset`, which is three bytes to four for `FixedOffset`. With that said, `time` supports niche value optimization! So if you have `Option`, your types won't be any larger if at all possible. `chrono` does not do this. 44 | 45 | Perhaps the largest difference between `time` and `chrono` is the choice of formatting. `time` prefers more powerful, readable descriptions (e.g. `[weekday], [month repr:long] [day padding:none]`), while `chrono` prefers a more limited description that is compatible with C (e.g. `%A, %B %-d, %Y`). I made this change out of frustration with the existing description, as it's not clear what is actually being written. Even those familiar with the format must admit that it's difficult to remember that `%A` corresponds to "Sunday" and `%B` "July", for example. An unexpected advantage of the change is that I was able to implement many modifiers, such as case sensitivity. Format descriptions in `time` can also be constructed via a macro, which statically verifies the format and eliminates all runtime overhead of parsing it, leading to it being faster the last time I benchmarked it. 46 | """ 47 | ``` 48 | 49 | **Has the time crate fixed the `localtime` soundness issue? If not, why? Have you considered using `tz-rs` to address the issue?** 50 | 51 | > **Editor:** 52 | > 53 | > - `localtime` soundness issue was reported by [@quininer](https://github.com/quininer) in 2020, both `time` and `chrono` have been affected. See [time#294](https://github.com/time-rs/time/issues/293) and [chrono#499](https://github.com/chronotope/chrono/issues/499) respectively. 54 | > - [`tz-rs`](https://crates.io/crates/tz-rs) is an emerging crate reimplementation of `libc` functions `localtime`, `gmtime` and `mktime`. 55 | 56 | ```quote 57 | author="Jacob" 58 | avatar="/static/avatar/jhpratt.png" 59 | content=""" 60 | "Fixed" is a relative term, unfortunately. It is fixed in the sense that there is no unsoundness (unless explicitly opted into via an `unsafe` method call), but the issue still remains in that Unix-based operating systems without thread-safe environments are unable to soundly get the local UTC offset. For that reason any method calls that require doing that return the error value. 61 | 62 | With regard to `tz-rs`, I considered it when it was first released. Using that crate would be extremely costly, as it would require parsing the entire time zone database on every invocation to avoid returning incorrect values. For this reason I decided not to use it. 63 | 64 | One solution that I hope will eventually be picked back up is `#[deprecated_safe]`, which should be applied to `std::env::set_var`. _Ultimately_ I would love for every operating system to have thread-safe environments, but that's an issue dating back to the very introduction of environment variables in 1979. 65 | """ 66 | ``` 67 | 68 | # Contribution to Rust 69 | 70 | **In addition to maintaining the `time` crate, what other contributions have you made to the Rust community? For example, I know you are the author of RFC and the implementation of [`derive_default_enum`](https://rust-lang.github.io/rfcs/3107-derive-default-enum.html).** 71 | 72 | ```quote 73 | author="Jacob" 74 | avatar="/static/avatar/jhpratt.png" 75 | content=""" 76 | `time` is definitely the largest. Aside from that, I maintain [deranged](https://github.com/jhpratt/deranged) — a proof of concept for ranged integers — and [num_threads](https://github.com/jhpratt/num_threads) — to get the number of threads the process is currently using. I previously maintained [standback](https://github.com/jhpratt/standback), which backports new features from the standard library to older compilers, but stopped maintaining it because no one used it. 77 | 78 | As you mentioned, I also (somewhat) wrote the RFC for `#[derive(Default)]` on `enum`s in addition to implementing and stabilizing it. More recently, I wrote the now-accepted RFC for restrictions, which will permit sealed traits at a compiler-level, rather than (ab)using visibility and accessibility. There is a pull request open to implement this, so it'll hit nightly relatively soon. By the time this is live, it may already be on nightly! 79 | 80 | Beyond the two RFCs, I've stabilized a large number of standard library items and language features, largely allowing new behavior in `const` contexts. Aside from `const` things, I implemented proper stability checks for `impl const Trait`, improved stability checks within `std`, and reduced the frequency of merge conflicts for PRs to the Rust repository. Something I've been looking into more recently is reducing the special-casing of the standard library, which will have significant benefits to the Rust ecosystem as a whole. **I'll actually be talking about this at [RustConf](https://rustconf.com/schedule/the-standard-library-is-special-let-s-change-that-) in September!** 81 | """ 82 | ``` 83 | 84 | **What are the most rewarding part and biggest challenges that you have faced as a maintainer of `time` crate and contributor of rust?** 85 | 86 | ```quote 87 | author="Jacob" 88 | avatar="/static/avatar/jhpratt.png" 89 | content=""" 90 | Probably the most rewarding part is knowing that millions of people use code that I write on a daily basis. When I get a feature request, I know it's because someone thinks the crate is useful and wants to see it improved, even if in a small way. The biggest challenge is unfortunately finding time to work on things. Funding is also far from ideal, but that's the case for everyone. As far as technical challenges, I tend to push the boundaries of what's possible with the compiler, leading me to occasionally run into barriers. 91 | """ 92 | ``` 93 | 94 | # Vision 95 | 96 | **What is your vision for the future of the `time` crate and Rust as a whole?** 97 | 98 | ```quote 99 | author="Jacob" 100 | avatar="/static/avatar/jhpratt.png" 101 | content=""" 102 | I have _a lot_ planned, to say the least. 103 | 104 | In `time`, I started work on integrating the time zone database in April. That would mean full support for actual time zones, including daylight saving time adjustments, rather than only fixed UTC offsets. This is a large undertaking that will consume a significant amount of time to complete. Another thing I have been working on is having a generic `DateTime` type that supports a variety of offsets: statically known, dynamic (equivalent to `OffsetDateTime`), a time zone, or no offset at all (equivalent to `PrimitiveDateTime`). To that effect, both `PrimitiveDateTime` and `OffsetDateTime` are actually simple wrappers around this currently-internal type. I hope to eventually make it public, but there are some language features that I want to land on stable before that happens. 105 | 106 | With Rust as a language, there are tons of things that I want to work on. I have an RFC that needs to be rewritten that would permit certain fields of `struct`s/`enum`s to have default values, such that they can be omitted when constructing them, with the values also being used by `#[derive(Default)]`. Other RFCs that I have planned, in some form or another, are `unsafe` fields and significant expansion of `#[derive]` capabilities, plus a couple more that I'm deliberately keeping the details of private. Rust is already a great language, but there is always room to improve. I intend to do my part in moving that forward. 107 | """ 108 | ``` 109 | 110 | # End question 111 | 112 | **When you're not working on Rust projects, what do you like to do in your free time? Do you have any hobbies or interests that you're particularly passionate about?** 113 | 114 | ```quote 115 | author="Jacob" 116 | avatar="/static/avatar/jhpratt.png" 117 | content=""" 118 | While I do not train any more, I trained for nearly ten years in [American Kenpo](https://en.wikipedia.org/wiki/American_Kenpo), specializing in self-defense and achieving the rank of third degree black belt. In that time, I taught others for a number of years. Another thing I do on occasion is solving Rubik's cubes, though like the martial arts I don't do this as much as I used to. Something I do quite often nowadays is watch various sports. 119 | """ 120 | ``` 121 | 122 | ![](/static/issue-4/jacob-in-training.jpg) 123 | 124 | > Jacob is in training for American Kenpo 125 | -------------------------------------------------------------------------------- /content/issue-4/show.md: -------------------------------------------------------------------------------- 1 | We are excited to announce a new topic in Rust Magazine called `#show-magazine`. This is a dedicated space for Rustaceans to showcase their projects and crates to the larger Rust community. Similar to [Show HN](https://news.ycombinator.com/shownew) or [Show /r/rust](https://www.google.com.hk/search?q=site%3Areddit.com+show+%2Fr%2Frust), **#show-magazine** is an opportunity for developers to share their works-in-progress, completed projects, and upcoming Rust-related events with the community. It is the perfect platform to receive feedback, connect with other Rust enthusiasts, and celebrate the diverse range of projects being developed in the Rust ecosystem. 2 | 3 | We encourage everyone to participate in **#show-magazine** by submitting their projects and crates to Rust Magazine. 4 | 5 | # Duo 6 | 7 | > Author: `@folyd` 8 | 9 | ```urlpreview 10 | https://github.com/duo-rs/duo 11 | ``` 12 | 13 | **Duo** is an easy-to-use observability solution that provides both logging and tracing capabilities for Rust applications. While traditional observability solutions are powerful (such as [ELK](https://elastic.co), [jaegertracing](https://jaegertracing.io), etc), it is also complex to deploy and maintain. Duo aimed to provide a less-powerful but complete set of observability features, with extremely simple deployment and maintenance. 14 | 15 | Here is the tracing demo: 16 | 17 | ![](https://github.com/duo-rs/duo/raw/master/duo-ui.png) 18 | 19 | # More to coming 20 | 21 | Feel free to submit yours! 22 | -------------------------------------------------------------------------------- /content/issue-4/zine.toml: -------------------------------------------------------------------------------- 1 | slug = "issue-4" 2 | number = 4 3 | title = "Issue 4" 4 | cover = "/static/ferris/4.jpeg" 5 | pub_date = "2023-06-05" 6 | publish = true 7 | 8 | [[article]] 9 | file = "show.md" 10 | title = "Show Magazine #1" 11 | author = "rust-magazine" 12 | topic = ["show-magazine"] 13 | pub_date = "2023-05-26" 14 | publish = true 15 | featured = true 16 | 17 | [[article]] 18 | file = "jhpratt.md" 19 | title = "Interview with @jhpratt" 20 | author = ["folyd", "jhpratt"] 21 | topic = ["interview"] 22 | pub_date = "2023-05-27" 23 | publish = true 24 | featured = true 25 | 26 | [[article]] 27 | file = "a-brief-discussion-on-invariants-in-rust.md" 28 | title = "A Brief Discussion on Invariants in Rust: Essential Properties to Uphold" 29 | author = ["TOETOE55"] 30 | topic = ["unsafe-rust"] 31 | pub_date = "2023-08-15" 32 | publish = true 33 | featured = true 34 | i18n.zh = { file = "a-brief-discussion-on-invariants-in-rust-zh.md", title = "稍微聊聊 Rust 中的 Invariant —— 那些必须保持的性质" } 35 | 36 | [[article]] 37 | file = "how-tokio-schedule-tasks.md" 38 | title = "How Tokio schedule tasks: A hard Lesson learnt" 39 | author = ["jiacai2050"] 40 | topic = ["tokio", "async"] 41 | pub_date = "2023-08-21" 42 | publish = true 43 | featured = true 44 | i18n.zh = { file = "how-tokio-schedule-tasks-zh.md", title = "Tokio 任务调度原理分析" } -------------------------------------------------------------------------------- /footer.json: -------------------------------------------------------------------------------- 1 | { 2 | "social": [ 3 | { 4 | "name": "GitHub", 5 | "image": "/static/github.svg", 6 | "url": "https://github.com/RustMagazine/rustmagazine" 7 | }, 8 | { 9 | "name": "Twitter", 10 | "image": "/static/twitter.svg", 11 | "url": "https://twitter.com/rustmagazineorg" 12 | }, 13 | { 14 | "name": "Discord", 15 | "image": "/static/discord.svg", 16 | "url": "https://discord.gg/DdwgBuReJe" 17 | } 18 | ], 19 | "column": { 20 | "reviewers": [ 21 | { 22 | "name": "@Macko76", 23 | "url": "https://github.com/macko76" 24 | }, 25 | { 26 | "name": "@Michael Johann", 27 | "url": "https://github.com/malagant" 28 | }, 29 | { 30 | "name": "@Mfrw", 31 | "url": "https://github.com/mfrw" 32 | }, 33 | { 34 | "name": "@Paul Tuck", 35 | "url": "https://github.com/pau1tuck" 36 | } 37 | ], 38 | "magazine": [ 39 | { 40 | "name": "All topics", 41 | "url": "/topics" 42 | }, 43 | { 44 | "name": "All authors", 45 | "url": "/authors" 46 | } 47 | ], 48 | "more": [ 49 | { 50 | "name": "About", 51 | "url": "/about" 52 | }, 53 | { 54 | "name": "FAQ", 55 | "url": "/faq" 56 | }, 57 | { 58 | "name": "Open Collective", 59 | "url": "https://opencollective.com/rustmagazine" 60 | }, 61 | { 62 | "name": "How to contribute", 63 | "url": "/contribution" 64 | } 65 | ] 66 | } 67 | } -------------------------------------------------------------------------------- /pages/about.md: -------------------------------------------------------------------------------- 1 | # About Rust Magazine 2 | 3 | > **Rust Magazine is a publication for the Rust programming language community, with contributions from Rustaceans all over the world.** 4 | 5 | ## The editorial team 6 | 7 | Rust Magazine is maintained by Rustaceans from all around the world and currently consists of the following editorial team: 8 | 9 | - `@folyd` 10 | - `@lcrossman` 11 | - `@HandongZhang` 12 | - `@Xuanwo` 13 | 14 | ## Disclaimer 15 | 16 | The views and opinions expressed in Rust Magazine are those of the individual authors and do not necessarily reflect the views of the editorial team or the Rust community as a whole. 17 | 18 | ## Commitment to diversity and inclusion: 19 | 20 | At Rust Magazine, we are committed to promoting diversity and inclusion within the Rust community. We believe that a diverse and inclusive community is essential for the growth and success of Rust, and we strive to support and amplify underrepresented voices. We encourage submissions from people of all backgrounds and experiences, and we are dedicated to creating a welcoming and inclusive environment for all contributors and readers. 21 | -------------------------------------------------------------------------------- /pages/contribution.md: -------------------------------------------------------------------------------- 1 | # How to contribute? 2 | 3 | ## Install zine 4 | 5 | ``` 6 | cargo install zine 7 | ``` 8 | 9 | ## Clone the magazine 10 | 11 | ``` 12 | $ git clone https://github.com/rustmagazine/rustmagazine 13 | 14 | $ cd rustmagazine 15 | 16 | $ zine serve 17 | listening on http://127.0.0.1:3000 18 | ``` 19 | 20 | ## Add your article 21 | 22 | ``` 23 | $ tree content/issue-1 24 | content/issue-1 25 | ├── announcing-zh.md 26 | ├── announcing.md 27 | └── zine.toml 28 | 29 | 0 directories, 3 files 30 | ``` 31 | 32 | Add your markdown file to the `content/issue-1` directory. 33 | 34 | ## Update zine.toml 35 | 36 | ```toml 37 | [[article]] 38 | path = "/your-article-path" 39 | file = "your-article.md" 40 | title = "Your article title" 41 | author = "Your-name" 42 | topic = ["optional-topic"] 43 | pub_date = "e.g. 2022-12-10" 44 | publish = true 45 | featured = true 46 | ``` 47 | 48 | ## Add author 49 | 50 | If this is your first time contributing to the magazine, you should also add your profile to the `[authors]` section of the root `zine.toml` file. 51 | 52 | ```diff 53 | [authors] 54 | rust-magazine = { name = "The Editorial Team", editor = true, bio = "The magazine editorial team" } 55 | + yourname = {} 56 | ``` 57 | 58 | Finally, run `git commit`, `git push`, and submit your PR! -------------------------------------------------------------------------------- /pages/faq.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ## What is Rust Magazine? 4 | 5 | Rust Magazine is an non-profit online publication dedicated to the Rust programming language and its ecosystem. 6 | 7 | ## Is Rust Magazine an official publication of Rust? 8 | 9 | No, Rust Magazine is an independent publication and is not affiliated with or endorsed by the Rust programming language or its maintainers. 10 | 11 | ## Who runs Rust Magazine? 12 | 13 | Rust Magazine is run by a team of volunteers from the Rust community. If you're interested in joining the team, please reach out to us via email or Github. 14 | 15 | ## Who is Rust Magazine for? 16 | 17 | Rust Magazine is for anyone interested in Rust, from beginners to advanced users, developers, and enthusiasts. 18 | 19 | ## How often is Rust Magazine published? 20 | 21 | Rust Magazine is published on a monthly basis, with each issue consisting of, at most, 10 articles. 22 | 23 | ## How to contribute? 24 | 25 | There are several ways to contribute articles to Rust Magazine: 26 | 27 | - When we find a great article from the community which we think deserves adding to the magazine, we’ll try to contact you. If you agree to publish it in the magazine, then we’ll help you submit your PR and get it merged. 28 | 29 | - We’ll invite some Rustaceans to write an article based on various topics and themes. 30 | 31 | - You can also submit your PR if you think your article is excellent. We appreciate everyone’s efforts to contribute. 32 | 33 | For more information about how to contribute, please check the [how to contribute page](/contribution). 34 | 35 | ## Have you considered asking for different cover images? 36 | 37 | We encourage authors to customize their cover image, however, it is not mandatory. 38 | 39 | ```toml 40 | [[article]] 41 | ... 42 | cover = "/path/to/cover.png" 43 | ``` 44 | 45 | ## Do you have any information about your reader base, such as their size or where they are mainly from? 46 | 47 | We don't have concrete details on the distribution of our readers since we have not added analysis services like Google Analytics to track them. However, we believe that we have a considerable audience worldwide, as we have had [an article](https://news.ycombinator.com/item?id=34689870) trend on Hacker News on February, 2023. We are constantly striving to reach more readers and grow our community. 48 | 49 | ## Does the author get paid? 50 | 51 | Since Rust Magainze is a non-profit publication, we rely on sponsorship to pay our authors. Visit the [Sponsor us](/sponsor) page to learn more. 52 | 53 | ## What is the license of the content in Rust Magazine? 54 | 55 | ![](https://mirrors.creativecommons.org/presskit/buttons/88x31/png/by-nc-sa.png) 56 | 57 | The content of the magazine is licensed under the [CC BY-NC-SA 4.0 LICENSE](https://creativecommons.org/licenses/by-nc-sa/4.0/) by default unless otherwise specified. 58 | 59 | ## Can I write about topics other than Rust programming in Rust Magazine? 60 | 61 | No, Rust Magazine is dedicated solely to topics related to the Rust programming language and its ecosystem. 62 | 63 | ## How can I get involved with Rust Magazine? 64 | 65 | You can get involved with Rust Magazine by contributing articles, editing, reviewing, leaving comment, and providing feedback. 66 | 67 | ## How is Rust Magazine built? 68 | 69 | Rust Magazine is built using [zine](https://github.com/zineland/zine), an intuitive static site generator tailored for magazines. 70 | -------------------------------------------------------------------------------- /pages/sponsor.md: -------------------------------------------------------------------------------- 1 | # Sponsorship 2 | 3 | ## Why does Rust Magazine need sponsorship? 4 | 5 | Rust Magazine is a non-profit, community-driven publication that relies on the contributions of volunteers to bring high-quality content to the Rust community. Rust Magazine is seeking sponsorships to: 6 | 7 | - Cover the running costs of Rust Magazine, including but not limited to domain fees, email fees, etc. 8 | - Pay our authors for their contributions and inspire more high-quality content. 9 | - Pay our editors a small salary for their time and effort in maintaining Rust Magazine. 10 | 11 | ## How to sponsor Rust Magazine? 12 | 13 | You can sponsor Rust Magazine through the [OpenCollective](https://opencollective.com/rustmagazine) platform. Rust Magazine offers several different sponsorship tiers: 14 | 15 | - For individuals, you can make a one-time donation with a customized amount, or sponsor Rust Magazine with $1, $5, or $10 per month. 16 | - For enterprise, you can sponsor us with $25, $50, and $100 per month. Rust Magazine offers different benefits for each tiers. 17 | - For more detail, please visit . 18 | 19 | [![](https://opencollective.com/rustmagazine/tiers/reader.svg?avatarHeight=120)](https://opencollective.com/rustmagazine) 20 | 21 | ## How does Rust Magazine pay the authors? 22 | 23 | ```callout, theme:red 24 | This is a **draft** and is subject to change in the future. 25 | ``` 26 | 27 | We pay each author according to the following formula: **_min($100, word count / 100 \* $1.25)_**. We pay $1.25 per 100 words, with a maximum payout of $100 per article. 28 | 29 | Each author must submit an expense request on the OpenCollective platform. After the admin approves the request, the Fiscal Host will make a payout. 30 | 31 | ## How does Rust Magazine pay the editors? 32 | 33 | ```callout, theme:red 34 | This is a **draft** and is subject to change in the future. 35 | ``` 36 | 37 | Rust Magazine has several editors who take turns being responsible for each Issue. The editor on duty for each Issue will receive a payment of **$50**. If an editor is not on duty for a particular Issue, Rust Magazine will not pay them for that Issue. Worth noting there is only one duty editor for each Issue. 38 | 39 | Each editor must submit an expense request on the OpenCollective platform. After the admin approves the request, the Fiscal Host will make a payout. 40 | 41 | ## What happens if some authors need to be paid but insufficient sponsors have come forwards - Who is responsible? 42 | 43 | We can't pay the author if our balance is insufficient, however, the author can still submit an expense request on OpenCollective, after we have enough sponsors, we'll approve it and make a payout. So no one else needs additional responsibility for this. 44 | 45 | ## What if the editor on duty is not the best editor for a particular article? 46 | 47 | This isn't a problem. Editor on duty isn't mean that the editor is responsible all thing, it means he/she is in charge of this issue. When he/she meets something he/she is not good at it, just seek help from the editor team. 48 | -------------------------------------------------------------------------------- /sponsors.json: -------------------------------------------------------------------------------- 1 | { 2 | "gold": [ 3 | { 4 | "name": "Ferris", 5 | "url": "https://rust-lang.org", 6 | "logo": "/static/ferris/rustacean-flat-happy.svg" 7 | }, 8 | { 9 | "name": "Ferris", 10 | "url": "https://rust-lang.org", 11 | "logo": "/static/ferris/rustacean-flat-happy.svg" 12 | } 13 | ], 14 | "silver": [ 15 | { 16 | "name": "Ferris", 17 | "url": "https://rust-lang.org", 18 | "logo": "/static/ferris/rustacean-flat-happy.svg" 19 | }, 20 | { 21 | "name": "Ferris", 22 | "url": "https://rust-lang.org", 23 | "logo": "/static/ferris/rustacean-flat-happy.svg" 24 | }, 25 | { 26 | "name": "Ferris", 27 | "url": "https://rust-lang.org", 28 | "logo": "/static/ferris/rustacean-flat-happy.svg" 29 | } 30 | ], 31 | "bronze": [ 32 | { 33 | "name": "Ferris", 34 | "url": "https://rust-lang.org", 35 | "logo": "/static/ferris/rustacean-flat-happy.svg" 36 | }, 37 | { 38 | "name": "Ferris", 39 | "url": "https://rust-lang.org", 40 | "logo": "/static/ferris/rustacean-flat-happy.svg" 41 | }, 42 | { 43 | "name": "Ferris", 44 | "url": "https://rust-lang.org", 45 | "logo": "/static/ferris/rustacean-flat-happy.svg" 46 | }, 47 | { 48 | "name": "Ferris", 49 | "url": "https://rust-lang.org", 50 | "logo": "/static/ferris/rustacean-flat-happy.svg" 51 | }, 52 | { 53 | "name": "Ferris", 54 | "url": "https://rust-lang.org", 55 | "logo": "/static/ferris/rustacean-flat-happy.svg" 56 | }, 57 | { 58 | "name": "Ferris", 59 | "url": "https://rust-lang.org", 60 | "logo": "/static/ferris/rustacean-flat-happy.svg" 61 | }, 62 | { 63 | "name": "Ferris", 64 | "url": "https://rust-lang.org", 65 | "logo": "/static/ferris/rustacean-flat-happy.svg" 66 | }, 67 | { 68 | "name": "Ferris", 69 | "url": "https://rust-lang.org", 70 | "logo": "/static/ferris/rustacean-flat-happy.svg" 71 | } 72 | ] 73 | } -------------------------------------------------------------------------------- /static/avatar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar.png -------------------------------------------------------------------------------- /static/avatar/chatgpt.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/avatar/databend.svg: -------------------------------------------------------------------------------- 1 | Databend LOGO -------------------------------------------------------------------------------- /static/avatar/folyd.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/folyd.jpeg -------------------------------------------------------------------------------- /static/avatar/jhpratt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/jhpratt.png -------------------------------------------------------------------------------- /static/avatar/m-ou-se.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/m-ou-se.jpeg -------------------------------------------------------------------------------- /static/avatar/malagant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/malagant.png -------------------------------------------------------------------------------- /static/avatar/psiace.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/psiace.jpg -------------------------------------------------------------------------------- /static/avatar/robjtede.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/robjtede.jpeg -------------------------------------------------------------------------------- /static/avatar/skyzh.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/skyzh.jpeg -------------------------------------------------------------------------------- /static/avatar/sunli.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/sunli.jpeg -------------------------------------------------------------------------------- /static/avatar/wangrunji.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/wangrunji.jpeg -------------------------------------------------------------------------------- /static/avatar/weihanglo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/weihanglo.png -------------------------------------------------------------------------------- /static/avatar/xuanwo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/xuanwo.jpeg -------------------------------------------------------------------------------- /static/avatar/xxchan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/avatar/xxchan.png -------------------------------------------------------------------------------- /static/bg.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/bg.jpeg -------------------------------------------------------------------------------- /static/discord.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon.ico -------------------------------------------------------------------------------- /static/favicon/16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon/16x16.png -------------------------------------------------------------------------------- /static/favicon/32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon/32x32.png -------------------------------------------------------------------------------- /static/favicon/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon/android-chrome-192x192.png -------------------------------------------------------------------------------- /static/favicon/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon/android-chrome-512x512.png -------------------------------------------------------------------------------- /static/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /static/favicon/site.webmanifest: -------------------------------------------------------------------------------- 1 | {"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} -------------------------------------------------------------------------------- /static/ferris/1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/ferris/1.jpeg -------------------------------------------------------------------------------- /static/ferris/2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/ferris/2.jpeg -------------------------------------------------------------------------------- /static/ferris/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/ferris/3.jpg -------------------------------------------------------------------------------- /static/ferris/4.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/ferris/4.jpeg -------------------------------------------------------------------------------- /static/ferris/rustacean-flat-happy.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /static/ferris/sunshine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/ferris/sunshine.png -------------------------------------------------------------------------------- /static/footer.css: -------------------------------------------------------------------------------- 1 | *,::backdrop,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:#3b82f680;--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.container{width:100%}@media (min-width:640px){.container{max-width:640px}}@media (min-width:768px){.container{max-width:768px}}@media (min-width:1024px){.container{max-width:1024px}}@media (min-width:1280px){.container{max-width:1280px}}@media (min-width:1536px){.container{max-width:1536px}}.absolute{position:absolute}.relative{position:relative}.bottom-0{bottom:0}.left-0{left:0}.right-0{right:0}.top-16{top:4rem}.z-10{z-index:10}.m-2{margin:.5rem}.m-auto{margin:auto}.mx-4{margin-left:1rem;margin-right:1rem}.mx-auto{margin-left:auto;margin-right:auto}.my-2{margin-bottom:.5rem;margin-top:.5rem}.my-4{margin-bottom:1rem;margin-top:1rem}.mb-28{margin-bottom:7rem}.mb-4{margin-bottom:1rem}.mt-12{margin-top:3rem}.mt-4{margin-top:1rem}.block{display:block}.flex{display:flex}.h-10{height:2.5rem}.h-14{height:3.5rem}.h-20{height:5rem}.h-5{height:1.25rem}.h-8{height:2rem}.w-12{width:3rem}.w-28{width:7rem}.w-5{width:1.25rem}.w-8{width:2rem}.max-w-sm{max-width:24rem}.grow{flex-grow:1}.translate-y-\[-9rem\]{--tw-translate-y:-9rem;transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.justify-center{justify-content:center}.border-2{border-width:2px}.border-solid{border-style:solid}.bg-\[\#0f1125\]{--tw-bg-opacity:1;background-color:rgb(15 17 37/var(--tw-bg-opacity))}.bg-\[\#f0ece8\]{--tw-bg-opacity:1;background-color:rgb(240 236 232/var(--tw-bg-opacity))}.p-16{padding:4rem}.p-4{padding:1rem}.px-10{padding-left:2.5rem;padding-right:2.5rem}.px-2{padding-left:.5rem;padding-right:.5rem}.py-28{padding-bottom:7rem;padding-top:7rem}.py-3{padding-bottom:.75rem;padding-top:.75rem}.py-6{padding-bottom:1.5rem;padding-top:1.5rem}.py-8{padding-bottom:2rem;padding-top:2rem}.pb-10{padding-bottom:2.5rem}.pt-12{padding-top:3rem}.text-center{text-align:center}.text-3xl{font-size:1.875rem;line-height:2.25rem}.text-4xl{font-size:2.25rem;line-height:2.5rem}.text-lg{font-size:1.125rem;line-height:1.75rem}.text-sm{font-size:.875rem;line-height:1.25rem}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-xs{font-size:.75rem;line-height:1rem}.font-bold{font-weight:700}.uppercase{text-transform:uppercase}.capitalize{text-transform:capitalize}.text-slate-100{--tw-text-opacity:1;color:rgb(241 245 249/var(--tw-text-opacity))}.text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184/var(--tw-text-opacity))}.underline{text-decoration-line:underline}.first\:ml-0:first-child{margin-left:0}.hover\:underline:hover{text-decoration-line:underline}@media (min-width:640px){.sm\:h-12{height:3rem}.sm\:h-24{height:6rem}.sm\:flex-row{flex-direction:row}.sm\:justify-around{justify-content:space-around}.sm\:p-6{padding:1.5rem}.sm\:pt-0{padding-top:0}}@media (min-width:768px){.md\:px-4{padding-left:1rem;padding-right:1rem}.md\:text-2xl{font-size:1.5rem;line-height:2rem}} -------------------------------------------------------------------------------- /static/github.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/issue-2/bs152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/bs152.png -------------------------------------------------------------------------------- /static/issue-2/egg/alphago.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/alphago.png -------------------------------------------------------------------------------- /static/issue-2/egg/constant-analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/constant-analysis.png -------------------------------------------------------------------------------- /static/issue-2/egg/constant-folding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/constant-folding.png -------------------------------------------------------------------------------- /static/issue-2/egg/egg-official.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/egg-official.png -------------------------------------------------------------------------------- /static/issue-2/egg/egraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/egraph.png -------------------------------------------------------------------------------- /static/issue-2/egg/join-reordering.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/join-reordering.png -------------------------------------------------------------------------------- /static/issue-2/egg/join-swap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/join-swap.png -------------------------------------------------------------------------------- /static/issue-2/egg/pushdown-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/pushdown-filter.png -------------------------------------------------------------------------------- /static/issue-2/egg/rewrite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/egg/rewrite.png -------------------------------------------------------------------------------- /static/issue-2/instruction-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/instruction-pipeline.png -------------------------------------------------------------------------------- /static/issue-2/opt1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/opt1.png -------------------------------------------------------------------------------- /static/issue-2/opt2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-2/opt2.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/1.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/10.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/12.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/13.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/14.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/2.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/3.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/4.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/5.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/6.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/8.png -------------------------------------------------------------------------------- /static/issue-3/optimizing-compilation-for-databend/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/optimizing-compilation-for-databend/9.png -------------------------------------------------------------------------------- /static/issue-3/robjtede.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/robjtede.jpeg -------------------------------------------------------------------------------- /static/issue-3/rust-search-extension.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/rust-search-extension.gif -------------------------------------------------------------------------------- /static/issue-3/task-stats-alloc/scope-meta.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/task-stats-alloc/scope-meta.png -------------------------------------------------------------------------------- /static/issue-3/update-index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-3/update-index.png -------------------------------------------------------------------------------- /static/issue-4/ceresdb-task1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-4/ceresdb-task1.png -------------------------------------------------------------------------------- /static/issue-4/ceresdb-task2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-4/ceresdb-task2.png -------------------------------------------------------------------------------- /static/issue-4/cpp-loop-ub.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-4/cpp-loop-ub.jpg -------------------------------------------------------------------------------- /static/issue-4/jacob-in-training.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RustMagazine/rustmagazine/0302cdfd87d793a31724a4fded0c95e57e7a643e/static/issue-4/jacob-in-training.jpg -------------------------------------------------------------------------------- /static/theme.css: -------------------------------------------------------------------------------- 1 | .zine-header { 2 | background-position-x: center; 3 | background-position-y: top; 4 | background-size: cover; 5 | } 6 | 7 | /* Set font family for those text */ 8 | .zine-brand, 9 | .zine-menu, 10 | .zine-diamond, 11 | .zine-button, 12 | .zine-article-title, 13 | .zine-card-title, 14 | h1 { 15 | font-family: "Alfa Slab One", serif !important; 16 | color: var(--main-color) !important; 17 | } 18 | 19 | h1 { 20 | font-size: 2em !important; 21 | margin-top: 1.5em !important; 22 | overflow-wrap: anywhere; 23 | } 24 | 25 | .zine-card-date { 26 | font-family: "Alfa Slab One", serif !important; 27 | } 28 | 29 | main, footer { 30 | font-family: "Fira Sans", Helvetica, Arial, sans-serif; 31 | } 32 | 33 | .zine-brand-bottom { 34 | font-family: "Alfa Slab One", serif; 35 | } 36 | 37 | .prose { 38 | font-size: 1.05rem !important; 39 | } 40 | 41 | .prose, .max-w-prose { 42 | max-width: 80ch !important; 43 | } 44 | 45 | /* ============== sponsorship styles ======================= */ 46 | .hill { 47 | position: absolute; 48 | contain: paint; 49 | height: 100%; 50 | width: 100%; 51 | } 52 | 53 | .hill::before { 54 | z-index: 0; 55 | border-top-left-radius: 100%; 56 | border-top-right-radius: 100%; 57 | position: absolute; 58 | top: 2rem; 59 | } 60 | 61 | .hill::before, :not(.has-ferris)>.hill::before { 62 | background: rgb(240 236 232); 63 | right: -16rem; 64 | left: -16rem; 65 | bottom: -8rem; 66 | content: ""; 67 | } 68 | 69 | .friendship-footer .friendship-tier h3 { 70 | font-size: 22px; 71 | color: rgb(122, 95, 79); 72 | font-weight: 600; 73 | position: relative; 74 | margin-bottom: 1rem; 75 | } 76 | 77 | .friendship-footer .friendship-tier h3::before { 78 | content: ""; 79 | display: block; 80 | height: 1px; 81 | width: 90%; 82 | background-color: rgba(118,110,101,40%); 83 | position: absolute; 84 | top: 50%; 85 | left: 50%; 86 | transform: translateX(-50%); 87 | max-width: 70rem; 88 | } 89 | 90 | .friendship-footer .friendship-tier h3>span { 91 | background-color: rgb(240 236 232); 92 | position: relative; 93 | padding-right: 2rem; 94 | padding-left: 2rem; 95 | } -------------------------------------------------------------------------------- /static/twitter.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | // Install tailwindcss standalone CLI, see https://tailwindcss.com/blog/standalone-cli. 2 | // Or install node version of tailwindcss. 3 | // 4 | // tailwindcss -o static/footer.css --watch --minify 5 | 6 | module.exports = { 7 | content: [ 8 | './templates/**/*.html', 9 | ], 10 | corePlugins: { 11 | // Disable preflight styles. 12 | preflight: false, 13 | }, 14 | plugins: [ 15 | ], 16 | } -------------------------------------------------------------------------------- /templates/article-extend.html: -------------------------------------------------------------------------------- 1 |

2 |
3 |
4 | 21 | -------------------------------------------------------------------------------- /templates/footer.html: -------------------------------------------------------------------------------- 1 |
2 | {% set sponsors = load_json("sponsors.json") %} 3 | {% if sponsors|length > 0 -%} 4 |
5 |
6 | Ferris the Crab 7 |
8 |
9 |

Thanks our sponsors

10 |
11 |
    12 |
  1. 13 |

    14 | Gold 15 |

    16 |
      17 | {% for item in sponsors["gold"] -%} 18 |
    • 19 | 20 | {{ item.name }} 21 | 22 |
      • 23 | {% endfor -%} 24 |
    25 |
    2. 26 | 27 |
  2. 28 |

    29 | Silver 30 |

    31 |
      32 | {% for item in sponsors["silver"] -%} 33 |
    • 34 | 35 | {{ item.name }} 36 | 37 |
      • 38 | {% endfor -%} 39 |
    40 |
    3. 41 | 42 |
  3. 43 |

    44 | Bronze 45 |

    46 |
      47 | {% for item in sponsors["bronze"] -%} 48 |
    • 49 | 50 | {{ item.name }} 51 | 52 |
      • 53 | {% endfor -%} 54 |
    55 |
    4. 56 |
57 | Sponsor us on OpenCollective 58 |
59 |
60 | {% endif -%} 61 |
62 |
63 |
64 | Subscriber: 65 | 66 | Newsletter, 67 | RSS 68 | 69 |
70 |
71 | {% set footer = load_json("footer.json") -%} 72 |
73 |
74 |
75 | {{ site.name }} 76 |
77 | {{ site.description }} 78 |
79 |
80 | {% for social in footer["social"] -%} 81 | 82 | {{ social.name }} 83 | 84 | {% endfor -%} 85 |
86 |
87 |
88 |
89 |
90 |
Editors
91 | {% set authors = get_entity("authors") -%} 92 | {% for author in authors|sort(attribute="id") -%} 93 | {% if author.editor and not author.team -%} 94 | @{{ author.name }} 95 | {% endif -%} 96 | {% endfor -%} 97 |
98 | {% for column in ["reviewers", "magazine", "more"] -%} 99 |
100 |
101 |
{{ column }}
102 | {% for item in footer["column"][column] -%} 103 | {{ item.name }} 104 | {% endfor -%} 105 |
106 | {% endfor -%} 107 |
108 |
109 |
110 |
111 |
112 |
113 | cc-by-nc-sa 115 | The content of the magazine is licensed under the 117 | CC BY-NC-SA 4.0 LICENSE 118 | by default unless otherwise specified. 119 |
120 |
121 | Copyright © 2023 {{ site.name }} 122 |
123 |
-------------------------------------------------------------------------------- /templates/head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /zine-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "urlPreviews": { 3 | "https://crates.io/crates/bpaf": [ 4 | "bpaf - crates.io", 5 | "A simple Command Line Argument Parser with parser combinators", 6 | "https://crates.io/assets/og-image.png" 7 | ], 8 | "https://github.com/Boshen/oxc": [ 9 | "GitHub - Boshen/oxc: The JavaScript Oxidation Compiler -> Linter", 10 | "The JavaScript Oxidation Compiler -> Linter. Contribute to Boshen/oxc development by creating an account on GitHub.", 11 | "https://opengraph.githubassets.com/2c5181ef0e49cd8cf380b484ddc24f0ee8a3ec6b4ad2c7b6132be6295a01437b/Boshen/oxc" 12 | ], 13 | "https://github.com/Rust-for-Linux/linux": [ 14 | "Rust-for-Linux/linux", 15 | "Adding support for the Rust language to the Linux kernel. - GitHub", 16 | "https://opengraph.githubassets.com/da51017fd78fd030005cefcdf108cc1e30c5dd9bff63a47e5970e644baf997b5/Rust-for-Linux/linux" 17 | ], 18 | "https://github.com/Xuanwo/backon": [ 19 | "GitHub - Xuanwo/backon: Retry with backoff without effort.", 20 | "Retry with backoff without effort. Contribute to Xuanwo/backon development by creating an account on GitHub.", 21 | "https://opengraph.githubassets.com/da3c5354a6e443e8d98948564e19e7f4d261d9ebe2727f3dae9881a65b05fb2f/Xuanwo/backon" 22 | ], 23 | "https://github.com/async-graphql/async-graphql": [ 24 | "GitHub - async-graphql/async-graphql: A GraphQL server library implemented in Rust", 25 | "A GraphQL server library implemented in Rust. Contribute to async-graphql/async-graphql development by creating an account on GitHub.", 26 | "https://opengraph.githubassets.com/a3613d90408005e2e1b9323a30ba15151184685a902bfa501fa7d1bb67d04e53/async-graphql/async-graphql" 27 | ], 28 | "https://github.com/datafuselabs/databend/issues/6180": [ 29 | "split databend-query to multiple crates · Issue #6180 · datafuselabs/databend · GitHub", 30 | "Summary Now Databend-Query is a rather large crate and we had to waste a lot of time on compiling and linking. It's time to split it up into a few relatively small crates. Related #4399", 31 | "https://opengraph.githubassets.com/855cd62b49753184fd23c301b3da40402c443696013761c9ec887426aea4e548/datafuselabs/databend/issues/6180" 32 | ], 33 | "https://github.com/duo-rs/duo": [ 34 | "GitHub - duo-rs/duo: Observability duet: Logging and Tracing", 35 | "Observability duet: Logging and Tracing. Contribute to duo-rs/duo development by creating an account on GitHub.", 36 | "https://opengraph.githubassets.com/ec2f377627caf39b7257e864c34fdd9442c582b6406287ca42a97f9d39dddb79/duo-rs/duo" 37 | ], 38 | "https://github.com/egraphs-good/egg": [ 39 | "GitHub - egraphs-good/egg: egg is a flexible, high-performance e-graph library", 40 | "egg is a flexible, high-performance e-graph library - GitHub - egraphs-good/egg: egg is a flexible, high-performance e-graph library", 41 | "https://opengraph.githubassets.com/f583a5edadcea6c19dfc54b6e93d2235480606c8ca18ac9d9b1cbaaef0484f06/egraphs-good/egg" 42 | ], 43 | "https://github.com/mozilla/sccache": [ 44 | "GitHub - mozilla/sccache: sccache is ccache with cloud storage", 45 | "sccache is ccache with cloud storage. Contribute to mozilla/sccache development by creating an account on GitHub.", 46 | "https://opengraph.githubassets.com/b01339bd81ff2a776c3e5f704c0e86d73f5ae7f17c5c9fb8ae8f7e51a95d8959/mozilla/sccache" 47 | ], 48 | "https://github.com/poem-web/poem": [ 49 | "GitHub - poem-web/poem: A full-featured and easy-to-use web framework with the Rust programming language.", 50 | "A full-featured and easy-to-use web framework with the Rust programming language. - GitHub - poem-web/poem: A full-featured and easy-to-use web framework with the Rust programming language.", 51 | "https://opengraph.githubassets.com/73cca9a4b04114708a5be9cf2ba876942219bb37b2358cd01d17d44b1e6f45c3/poem-web/poem" 52 | ], 53 | "https://github.com/risinglightdb/sql-optimizer-labs": [ 54 | "GitHub - risinglightdb/sql-optimizer-labs: 🚧 Build a SQL optimizer in 1000 lines of Rust using egg.", 55 | "🚧 Build a SQL optimizer in 1000 lines of Rust using egg. - GitHub - risinglightdb/sql-optimizer-labs: 🚧 Build a SQL optimizer in 1000 lines of Rust using egg.", 56 | "https://opengraph.githubassets.com/4d62c05c142ab0f3e53557b5a7239ccc2470ad57c036fb866eca879a14458703/risinglightdb/sql-optimizer-labs" 57 | ], 58 | "https://github.com/rust-lang/rust/pull/104435": [ 59 | "`VecDeque::resize` should re-use the buffer in the passed-in element by scottmcm · Pull Request #104435 · rust-lang/rust · GitHub", 60 | "", 61 | "https://opengraph.githubassets.com/bff46e9c46a3ba4c1353931657eaef937f0bbee83cbab257cbce435f5781687d/rust-lang/rust/pull/104435" 62 | ], 63 | "https://github.com/rust-lang/rust/pull/74024": [ 64 | "Improve slice.binary_search_by()'s best-case performance to O(1) by Folyd · Pull Request #74024 · rust-lang/rust · GitHub", 65 | "This PR aimed to improve the slice.binary_search_by()'s best-case performance to O(1).\nNoticed\nI don't know why the docs of binary_search_by said \"If there are multiple matches, then any one of the ma", 66 | "https://opengraph.githubassets.com/d65ab523be84a75f559441723a6b9725d967c3cb76c9c28c6f33b1c9619363bc/rust-lang/rust/pull/74024" 67 | ], 68 | "https://github.com/servo/string-cache/pull/268": [ 69 | "feat: use bucket mutex instead of global mutex for dynamic set by Boshen · Pull Request #268 · servo/string-cache · GitHub", 70 | "Background:\nI'm working a concurrent program using swc to parse files in parallel. swc uses string-cache.\nI have found the global mutex in the dynamic set being the major performance bottleneck for ou", 71 | "https://opengraph.githubassets.com/a83189ee3a2852575aee40e0671d5375b45eedb9744be6ebd0844754e55f4b8d/servo/string-cache/pull/268" 72 | ], 73 | "https://github.com/skyzh/mini-lsm": [ 74 | "skyzh/mini-lsm", 75 | "A tutorial of building an LSM-Tree storage engine in a week!- GitHub", 76 | "https://opengraph.githubassets.com/83c82d82a7ab5be32dc4bf96da23f2d066f84fc56aadcfc8e3e866e1472171a9/skyzh/mini-lsm" 77 | ], 78 | "https://github.com/time-rs/time": [ 79 | "GitHub - time-rs/time: The most used Rust library for date and time handling.", 80 | "The most used Rust library for date and time handling. - GitHub - time-rs/time: The most used Rust library for date and time handling.", 81 | "https://opengraph.githubassets.com/05a44fc8533b2916cb80bd6d41cad5ed9159ca347d22b2d9a3bcbd882574c351/time-rs/time" 82 | ], 83 | "https://polkadot.network/blog/a-polkadot-postmortem-24-05-2021": [ 84 | "A Polkadot Postmortem - 24.05.2021", 85 | "On 24 May 2021, Polkadot nodes failed with an out of memory (OOM) error on block 5,202,216. This block contained an on-chain solution to the validator election, which is normally computed off-chain an", 86 | "https://cms.polkadot.network/content/images/2021/05/image1.png" 87 | ], 88 | "https://rust.extension.sh": [ 89 | "Rust Search Extension - The ultimate search extension for Rust", 90 | "Search docs, crates, builtin attributes, official books, and error codes, etc in your address bar instantly.", 91 | "https://rust.extension.sh/banner.png" 92 | ], 93 | "https://ziglang.org/": [ 94 | "⚡\nZig Programming Language", 95 | "Zig is a general-purpose programming language and toolchain for maintaining robust, optimal and reusable software.", 96 | "https://ziglang.org/zig-logo-dark.svg" 97 | ] 98 | } 99 | } -------------------------------------------------------------------------------- /zine.toml: -------------------------------------------------------------------------------- 1 | 2 | [site] 3 | url = "https://rustmagazine.org" 4 | name = "Rust Magazine" 5 | description = "A publication dedicated to the Rust programming language." 6 | edit_url = "https://github.com/rustmagazine/rustmagazine/edit/main" 7 | menu = [ 8 | { name = "About", url = "/about" }, 9 | { name = "FAQ", url = "/faq" }, 10 | { name = "Sponsor us", url = "/sponsor" }, 11 | ] 12 | social_image = "/static/bg.jpeg" 13 | 14 | [theme] 15 | primary_color = "#f8e9b8" 16 | secondary_color = "#faf8f3" 17 | main_color = "#222c4b" 18 | link_color = "#36a5ea" 19 | background_image = "/static/bg.jpeg" 20 | head_template = "templates/head.html" 21 | footer_template = "templates/footer.html" 22 | article_extend_template = "templates/article-extend.html" 23 | default_cover = "/static/bg.jpeg" 24 | default_avatar = "/static/avatar.png" 25 | 26 | [markdown] 27 | highlight_theme = "ayu-light" 28 | 29 | [topics] 30 | announcement = {} 31 | interview = {} 32 | show-magazine = { name = "Show Magazine", description = "This is a dedicated space for Rustaceans to showcase their projects and crates to the larger Rust community. Similar to [Show HN](https://news.ycombinator.com/shownew) or [Show /r/rust](https://www.google.com.hk/search?q=site%3Areddit.com+show+%2Fr%2Frust), **#show-magazine** is an opportunity for developers to share their works-in-progress, completed projects, and upcoming Rust-related events with the community." } 33 | game = { name = "Game development", description = "Topic of rust game development." } 34 | pr-demystifying = { name = "PR Demystifying", description = "Topic of PR demystifying in Rust community." } 35 | optimization = {} 36 | async = {} 37 | database = {} 38 | optimizer = {} 39 | tokio = { description = "A runtime for writing reliable, asynchronous, and slim applications with the Rust programming language" } 40 | business = { description = "Rust in Business" } 41 | tutorial = {} 42 | memory = {} 43 | allocator = {} 44 | unsafe-rust = { name = "Unsafe Rust", description = "Topic of unsafe rust." } 45 | engineering = {} 46 | 47 | # Authors declare below 48 | [authors] 49 | rust-magazine = { name = "The Editorial Team", editor = true, team = true, bio = "The magazine editorial team" } 50 | lcrossman = { name = "LCrossman", editor = true } 51 | HandongZhang = { name = "Handong Zhang", editor = true } 52 | skyzh = { avatar = "/static/avatar/skyzh.jpeg" } 53 | 54 | [authors.folyd] 55 | editor = true 56 | avatar = "/static/avatar/folyd.jpeg" 57 | bio = """ 58 | The author of [Rust Search Extension](https://github.com/huhu/rust-search-extension) and [zine](https://github.com/zineland/zine). 59 | 60 | - [GitHub](https://github.com/folyd) 61 | - [Blog](https://folyd.com) 62 | """ 63 | 64 | [authors.yukang] 65 | bio = """ 66 | - [GitHub](https://github.com/chenyukang) 67 | - [Blog](https://catcoding.me) 68 | """ 69 | 70 | [authors.weihanglo] 71 | name = "Weihang Lo" 72 | avatar = "/static/avatar/weihanglo.png" 73 | 74 | [authors.sunli] 75 | avatar = "/static/avatar/sunli.jpeg" 76 | 77 | [authors.shijicheng] 78 | name = "Jicheng Shi" 79 | 80 | [authors.waynexia] 81 | name = "Xia Ruihang" 82 | bio = """ 83 | Working on [GreptimeDB](https://github.com/greptimeteam/greptimedb) 84 | 85 | - [GitHub](https://github.com/waynexia) 86 | - [Blog](https://blog.waynest.com) 87 | """ 88 | 89 | [authors.greptime-team] 90 | name = "Greptime Team" 91 | team = true 92 | bio = """ 93 | Greptime was founded in April 2022 with two main offerings: [GreptimeDB](https://github.com/greptimeteam/greptimedb) and Greptime Cloud. 94 | 95 | - Greptime DB is an open-source, cloud-native time series database with powerful analytical features; 96 | - Greptime Cloud is a SaaS solution built on GreptimeDB. 97 | 98 | We are passionate about helping users find real-time values from their time series data with our tools. 99 | 100 | - [GitHub](https://github.com/greptimeteam) 101 | - [Website](https://greptime.com) 102 | """ 103 | 104 | [authors.xxchan] 105 | name = "xxchan" 106 | avatar = "/static/avatar/xxchan.png" 107 | bio = """ 108 | - [GitHub](https://github.com/xxchan) 109 | - [Blog](https://xxchan.github.io) 110 | """ 111 | 112 | [authors.manpacket] 113 | name = "Michael Baykov" 114 | bio = """ 115 | As an experienced Haskell programmer and contributor, I want to introduce Category Theory and 116 | bring a new flexible and practical development approach to Rust. 117 | 118 | - [GitHub](https://github.com/pacak) 119 | """ 120 | 121 | [authors.xuanwo] 122 | avatar = "/static/avatar/xuanwo.jpeg" 123 | editor = true 124 | bio = """ 125 | - [GitHub](https://github.com/Xuanwo) 126 | - [Blog](https://xuanwo.io) 127 | """ 128 | 129 | [authors.wangrunji] 130 | name = "wangrunji" 131 | avatar = "/static/avatar/wangrunji.jpeg" 132 | bio = """ 133 | - [GitHub](https://github.com/wangrunji0408) 134 | """ 135 | 136 | [authors.boshen] 137 | name = "Boshen" 138 | bio = """ 139 | - [GitHub](https://github.com/Boshen) 140 | """ 141 | 142 | [authors.robjtede] 143 | avatar = "/static/avatar/robjtede.jpeg" 144 | 145 | [authors.PsiACE] 146 | name = "PsiACE" 147 | avatar = "/static/avatar/psiace.jpg" 148 | bio = """ 149 | - [GitHub](https://github.com/psiace) 150 | """ 151 | 152 | [authors.databend] 153 | name = "Databend" 154 | avatar = "/static/avatar/databend.svg" 155 | team = true 156 | bio = """ 157 | [Databend](https://github.com/datafuselabs/databend/) is a modern cloud data warehouse focusing on reducing cost and complexity for your massive-scale analytics needs. Open source alternative to Snowflake. 158 | 159 | Also available in the cloud: 160 | """ 161 | 162 | [authors.tennyzhuang] 163 | name = "tennyzhuang" 164 | bio = """ 165 | - [GitHub](https://github.com/TennyZhuang) 166 | """ 167 | 168 | [authors.lei] 169 | name = "Lei Huang" 170 | bio = """ 171 | Working on [GreptimeDB](https://github.com/greptimeteam/greptimedb) 172 | 173 | - [GitHub](https://github.com/v0y4g3r) 174 | - [Blog](https://huanglei.rocks/) 175 | """ 176 | 177 | [authors.jhpratt] 178 | avatar="/static/avatar/jhpratt.png" 179 | 180 | [authors.TOETOE55] 181 | name = "TOETOE55" 182 | bio = """ 183 | - [GitHub](https://github.com/TOETOE55) 184 | """ 185 | 186 | [authors.jiacai2050] 187 | name = "Jiacai Liu" 188 | bio = """ 189 | - [GitHub](https://github.com/jiacai2050) 190 | - [Mastodon](https://mastodon.social/@liujiacai) 191 | - [Twitter](https://twitter.com/liujiacai) 192 | - [Blog](https://en.liujiacai.net/) 193 | """ --------------------------------------------------------------------------------