├── .github
    ├── CODEOWNERS
    ├── ISSUE_TEMPLATE
    │   ├── bug_report.md
    │   └── feature_request.md
    ├── PULL_REQUEST_TEMPLATE.md
    └── workflows
    │   ├── ci.yaml
    │   ├── dependency-review.yaml
    │   └── security.yaml
├── .gitignore
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── Cargo.toml
├── LICENSE
├── README.md
├── README_CN.md
├── _typos.toml
├── benches
    └── bench.rs
├── examples
    ├── Cargo.toml
    └── src
    │   └── hello_world
    │       ├── greeter_client.rs
    │       └── greeter_server.rs
├── licenses
    ├── LICENSE-shmipc-go
    └── LICENSE-shmipc-spec
├── rustfmt.toml
├── src
    ├── buffer
    │   ├── buf.rs
    │   ├── linked.rs
    │   ├── list.rs
    │   ├── manager.rs
    │   ├── mod.rs
    │   └── slice.rs
    ├── config.rs
    ├── consts.rs
    ├── error.rs
    ├── lib.rs
    ├── listener.rs
    ├── protocol
    │   ├── adapter.rs
    │   ├── block_io.rs
    │   ├── event.rs
    │   ├── header.rs
    │   ├── initializer
    │   │   ├── mod.rs
    │   │   ├── v2.rs
    │   │   └── v3.rs
    │   └── mod.rs
    ├── queue.rs
    ├── session
    │   ├── config.rs
    │   ├── ext.rs
    │   ├── manager.rs
    │   ├── mod.rs
    │   └── pool.rs
    ├── stats.rs
    ├── stream.rs
    └── util.rs
└── tests
    └── test.rs


/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | # For more information, please refer to https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
2 | 
3 | *   @cloudwego/volo-reviewers @cloudwego/volo-approvers @cloudwego/volo-maintainers
4 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: 🐛 Bug Report
 3 | about: If something isn't working as expected 🤔.
 4 | ---
 5 | 
 6 | ## Bug Report
 7 | 
 8 | <!--
 9 | Thank you for reporting an issue.
10 | 
11 | Please fill in as much of the template below as you're able.
12 | -->
13 | 
14 | ### Version
15 | 
16 | <!--
17 | List the versions of `shmipc` crate you are using. The easiest way to get
18 | this information is using `cargo-tree`.
19 | 
20 | `cargo install cargo-tree`
21 | (see install here: https://github.com/sfackler/cargo-tree)
22 | 
23 | Then:
24 | 
25 | `cargo tree | grep shmipc`
26 | -->
27 | 
28 | ### Platform
29 | 
30 | <!---
31 | Output of `uname -a` (UNIX), or version and 32 or 64-bit (Windows)
32 | -->
33 | 
34 | ### Description
35 | 
36 | <!--
37 | 
38 | Enter your issue details below this comment.
39 | 
40 | One way to structure the description:
41 | 
42 | <short summary of the bug>
43 | 
44 | I tried this code:
45 | 
46 | <code sample that causes the bug>
47 | 
48 | I expected to see this happen: <explanation>
49 | 
50 | Instead, this happened: <explanation>
51 | -->
52 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: 💡 Feature Request
 3 | about: I have a suggestion (and may want to implement it 🙂)!
 4 | ---
 5 | 
 6 | ## Feature Request
 7 | 
 8 | ### Motivation
 9 | 
10 | <!--
11 | Please describe the use case(s) or other motivation for the new feature.
12 | -->
13 | 
14 | ### Proposal
15 | 
16 | <!--
17 | How should the new feature be implemented, and why? Add any considered
18 | drawbacks.
19 | -->
20 | 
21 | ### Alternatives
22 | 
23 | <!--
24 | Are there other ways to solve this problem that you've considered? What are
25 | their potential drawbacks? Why was the proposed solution chosen over these
26 | alternatives?
27 | -->
28 | 


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 | <!--
 2 | Thank you for your Pull Request. Please provide a description above and review
 3 | the requirements below.
 4 | 
 5 | Bug fixes and new features should include tests.
 6 | 
 7 | Contributors guide: https://github.com/cloudwego/shmipc-rs/blob/main/CONTRIBUTING.md
 8 | -->
 9 | 
10 | ## Motivation
11 | 
12 | <!--
13 | Explain the context and why you're making that change. What is the problem
14 | you're trying to solve? If a new feature is being added, describe the intended
15 | use case that feature fulfills.
16 | -->
17 | 
18 | ## Solution
19 | 
20 | <!--
21 | Summarize the solution and provide any necessary context needed to understand
22 | the code change.
23 | -->
24 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yaml:
--------------------------------------------------------------------------------
 1 | name: "CI"
 2 | on:
 3 |   pull_request:
 4 |   push:
 5 |     branches:
 6 |       - main
 7 | 
 8 | env:
 9 |   RUST_BACKTRACE: 1
10 |   CARGO_TERM_COLOR: always
11 | 
12 | jobs:
13 |   ci-pass:
14 |     name: CI is green
15 |     runs-on: ubuntu-latest
16 |     needs:
17 |       - test-linux
18 |       - lint
19 |       - docs-check
20 |     steps:
21 |       - run: exit 0
22 | 
23 |   lint:
24 |     runs-on: [self-hosted, Linux, amd64]
25 | 
26 |     strategy:
27 |       matrix:
28 |         rust: [nightly]
29 |     steps:
30 |       - uses: actions/checkout@v4
31 |       - uses: dtolnay/rust-toolchain@master
32 |         with:
33 |           components: rustfmt,clippy
34 |           toolchain: ${{matrix.rust}}
35 |       - uses: actions-rs/clippy-check@v1
36 |         with:
37 |           token: ${{ secrets.GITHUB_TOKEN }}
38 |       - name: Format check
39 |         run: |
40 |           cargo fmt -- --check
41 | 
42 |   docs-check:
43 |     runs-on: [self-hosted, Linux, amd64]
44 | 
45 |     strategy:
46 |       matrix:
47 |         rust: [nightly]
48 |     steps:
49 |       - uses: actions/checkout@v4
50 |       - uses: dtolnay/rust-toolchain@master
51 |         with:
52 |           components: rustfmt,clippy
53 |           toolchain: ${{matrix.rust}}
54 |       - name: Docs check
55 |         run: |
56 |           cargo rustdoc --all-features -- --deny warnings
57 | 
58 |   test-linux:
59 |     runs-on: [self-hosted, Linux, amd64]
60 | 
61 |     strategy:
62 |       matrix:
63 |         rust: [nightly, stable]
64 |     steps:
65 |       - uses: actions/checkout@v4
66 |       - uses: dtolnay/rust-toolchain@master
67 |         with:
68 |           components: rustfmt,clippy
69 |           toolchain: ${{matrix.rust}}
70 |       - name: Run tests
71 |         run: |
72 |           cargo clippy --all-features -- --deny warnings
73 |           cargo test --all-features
74 | 


--------------------------------------------------------------------------------
/.github/workflows/dependency-review.yaml:
--------------------------------------------------------------------------------
 1 | name: "Dependency Review"
 2 | on: [pull_request]
 3 | 
 4 | permissions:
 5 |   contents: read
 6 | 
 7 | jobs:
 8 |   dependency-review:
 9 |     runs-on: [self-hosted, Linux, amd64]
10 |     steps:
11 |       - name: "Checkout Repository"
12 |         uses: actions/checkout@v4
13 |       - name: "Dependency Review"
14 |         uses: actions/dependency-review-action@v1
15 | 


--------------------------------------------------------------------------------
/.github/workflows/security.yaml:
--------------------------------------------------------------------------------
 1 | name: "Security Audit"
 2 | on:
 3 |   pull_request:
 4 |   push:
 5 |     paths:
 6 |       - "**/Cargo.toml"
 7 |       - "**/Cargo.lock"
 8 | jobs:
 9 |   security-audit:
10 |     runs-on: [self-hosted, Linux, amd64]
11 |     steps:
12 |       - uses: actions/checkout@v4
13 |       - uses: dtolnay/rust-toolchain@stable
14 |       - uses: actions-rs/audit-check@v1
15 |         with:
16 |           token: ${{ secrets.GITHUB_TOKEN }}
17 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /Cargo.lock
3 | 


--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
  1 | # Contributor Covenant Code of Conduct
  2 | 
  3 | ## Our Pledge
  4 | 
  5 | We as members, contributors, and leaders pledge to make participation in our
  6 | community a harassment-free experience for everyone, regardless of age, body
  7 | size, visible or invisible disability, ethnicity, sex characteristics, gender
  8 | identity and expression, level of experience, education, socio-economic status,
  9 | nationality, personal appearance, race, religion, or sexual identity
 10 | and orientation.
 11 | 
 12 | We pledge to act and interact in ways that contribute to an open, welcoming,
 13 | diverse, inclusive, and healthy community.
 14 | 
 15 | ## Our Standards
 16 | 
 17 | Examples of behavior that contributes to a positive environment for our
 18 | community include:
 19 | 
 20 | * Demonstrating empathy and kindness toward other people
 21 | * Being respectful of differing opinions, viewpoints, and experiences
 22 | * Giving and gracefully accepting constructive feedback
 23 | * Accepting responsibility and apologizing to those affected by our mistakes,
 24 |   and learning from the experience
 25 | * Focusing on what is best not just for us as individuals, but for the
 26 |   overall community
 27 | 
 28 | Examples of unacceptable behavior include:
 29 | 
 30 | * The use of sexualized language or imagery, and sexual attention or
 31 |   advances of any kind
 32 | * Trolling, insulting or derogatory comments, and personal or political attacks
 33 | * Public or private harassment
 34 | * Publishing others' private information, such as a physical or email
 35 |   address, without their explicit permission
 36 | * Other conduct which could reasonably be considered inappropriate in a
 37 |   professional setting
 38 | 
 39 | ## Enforcement Responsibilities
 40 | 
 41 | Community leaders are responsible for clarifying and enforcing our standards of
 42 | acceptable behavior and will take appropriate and fair corrective action in
 43 | response to any behavior that they deem inappropriate, threatening, offensive,
 44 | or harmful.
 45 | 
 46 | Community leaders have the right and responsibility to remove, edit, or reject
 47 | comments, commits, code, wiki edits, issues, and other contributions that are
 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation
 49 | decisions when appropriate.
 50 | 
 51 | ## Scope
 52 | 
 53 | This Code of Conduct applies within all community spaces, and also applies when
 54 | an individual is officially representing the community in public spaces.
 55 | Examples of representing our community include using an official e-mail address,
 56 | posting via an official social media account, or acting as an appointed
 57 | representative at an online or offline event.
 58 | 
 59 | ## Enforcement
 60 | 
 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
 62 | reported to the community leaders responsible for enforcement at
 63 | conduct@cloudwego.io.
 64 | All complaints will be reviewed and investigated promptly and fairly.
 65 | 
 66 | All community leaders are obligated to respect the privacy and security of the
 67 | reporter of any incident.
 68 | 
 69 | ## Enforcement Guidelines
 70 | 
 71 | Community leaders will follow these Community Impact Guidelines in determining
 72 | the consequences for any action they deem in violation of this Code of Conduct:
 73 | 
 74 | ### 1. Correction
 75 | 
 76 | **Community Impact**: Use of inappropriate language or other behavior deemed
 77 | unprofessional or unwelcome in the community.
 78 | 
 79 | **Consequence**: A private, written warning from community leaders, providing
 80 | clarity around the nature of the violation and an explanation of why the
 81 | behavior was inappropriate. A public apology may be requested.
 82 | 
 83 | ### 2. Warning
 84 | 
 85 | **Community Impact**: A violation through a single incident or series
 86 | of actions.
 87 | 
 88 | **Consequence**: A warning with consequences for continued behavior. No
 89 | interaction with the people involved, including unsolicited interaction with
 90 | those enforcing the Code of Conduct, for a specified period of time. This
 91 | includes avoiding interactions in community spaces as well as external channels
 92 | like social media. Violating these terms may lead to a temporary or
 93 | permanent ban.
 94 | 
 95 | ### 3. Temporary Ban
 96 | 
 97 | **Community Impact**: A serious violation of community standards, including
 98 | sustained inappropriate behavior.
 99 | 
100 | **Consequence**: A temporary ban from any sort of interaction or public
101 | communication with the community for a specified period of time. No public or
102 | private interaction with the people involved, including unsolicited interaction
103 | with those enforcing the Code of Conduct, is allowed during this period.
104 | Violating these terms may lead to a permanent ban.
105 | 
106 | ### 4. Permanent Ban
107 | 
108 | **Community Impact**: Demonstrating a pattern of violation of community
109 | standards, including sustained inappropriate behavior,  harassment of an
110 | individual, or aggression toward or disparagement of classes of individuals.
111 | 
112 | **Consequence**: A permanent ban from any sort of public interaction within
113 | the community.
114 | 
115 | ## Attribution
116 | 
117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage],
118 | version 2.0, available at
119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
120 | 
121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct
122 | enforcement ladder](https://github.com/mozilla/diversity).
123 | 
124 | [homepage]: https://www.contributor-covenant.org
125 | 
126 | For answers to common questions about this code of conduct, see the FAQ at
127 | https://www.contributor-covenant.org/faq. Translations are available at
128 | https://www.contributor-covenant.org/translations.
129 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # How to Contribute
 2 | 
 3 | ## Your First Pull Request
 4 | We use github for our codebase. You can start by reading [How To Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests).
 5 | 
 6 | ## Branch Organization
 7 | We use [git-flow](https://nvie.com/posts/a-successful-git-branching-model/) as our branch organization, as known as [FDD](https://en.wikipedia.org/wiki/Feature-driven_development)
 8 | 
 9 | ## Bugs
10 | ### 1. How to Find Known Issues
11 | We are using [Github Issues](https://github.com/cloudwego/shmipc-rs/issues) for our public bugs. We keep a close eye on this and try to make it clear when we have an internal fix in progress. Before filing a new task, try to make sure your problem doesn’t already exist.
12 | 
13 | ### 2. Reporting New Issues
14 | Providing a reduced test code is a recommended way for reporting issues. Then can placed in:
15 | - Just in issues
16 | 
17 | ### 3. Security Bugs
18 | Please do not report the safe disclosure of bugs to public issues. Contact us by [Support Email](mailto:conduct@cloudwego.io)
19 | 
20 | ## How to Get in Touch
21 | - [Email](mailto:conduct@cloudwego.io)
22 | 
23 | ## Submit a Pull Request
24 | Before you submit your Pull Request (PR) consider the following guidelines:
25 | 1. Search [GitHub](https://github.com/cloudwego/shmipc-rs/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate existing efforts.
26 | 2. Be sure that an issue describes the problem you're fixing, or documents the design for the feature you'd like to add. Discussing the design upfront helps to ensure that we're ready to accept your work.
27 | 3. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the cloudwego/shmipc-rs repo.
28 | 4. In your forked repository, make your changes in a new git branch:
29 |     ```
30 |     git checkout -b my-fix-branch develop
31 |     ```
32 | 5. Create your patch, including appropriate test cases.
33 | 6. Follow our [Style Guides](#code-style-guides).
34 | 7. Commit your changes using a descriptive commit message that follows [AngularJS Git Commit Message Conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit).
35 |    Adherence to these conventions is necessary because release notes are automatically generated from these messages.
36 | 8. Push your branch to GitHub:
37 |     ```
38 |     git push origin my-fix-branch
39 |     ```
40 | 9. In GitHub, send a pull request to `shmipc-rs:develop`
41 | 
42 | ## Contribution Prerequisites
43 | - You are familiar with [Github](https://github.com)
44 | - Maybe you need familiar with [Actions](https://github.com/features/actions)(our default workflow tool).
45 | 
46 | ## Code Style Guides
47 | We use `rustfmt` tool.
48 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "shmipc"
 3 | version = "0.0.1"
 4 | edition = "2024"
 5 | authors = ["Volo Team <volo@cloudwego.io>"]
 6 | license = "Apache-2.0"
 7 | readme = "README.md"
 8 | repository = "https://github.com/cloudwego/shmipc-rs"
 9 | description = "Shared memory IPC for Rust"
10 | documentation = "https://docs.rs/shmipc"
11 | categories = ["asynchronous", "network-programming"]
12 | keywords = ["async", "network", "ipc"]
13 | 
14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
15 | 
16 | [dependencies]
17 | motore = "0.4"
18 | volo = "0.10"
19 | 
20 | anyhow = "1"
21 | async-trait = "0.1"
22 | arc-swap = "1"
23 | bytes = "1"
24 | fs2 = "0.4"
25 | futures = "0.3"
26 | memmap2 = "0.9"
27 | nix = { version = "0.29", features = ["fs"] }
28 | thiserror = "2"
29 | tokio = { version = "1", features = ["full"] }
30 | tracing = "0.1"
31 | pin-project = "1"
32 | socket2 = { version = "0.5", features = ["all"] }
33 | serde = { version = "1", features = ["derive"] }
34 | 
35 | [dev-dependencies]
36 | criterion = { version = "0.5", features = ["async_tokio"] }
37 | futures = "0.3"
38 | rand = "0.9"
39 | tokio-scoped = "0.2"
40 | tracing-subscriber = "0.3"
41 | 
42 | 
43 | [profile.dev]
44 | overflow-checks = false
45 | 
46 | [profile.release]
47 | opt-level = 3
48 | debug = true
49 | rpath = false
50 | lto = true
51 | debug-assertions = false
52 | codegen-units = 1
53 | panic = 'unwind'
54 | incremental = false
55 | overflow-checks = false
56 | 
57 | [workspace]
58 | members = ["examples"]
59 | 
60 | [[bench]]
61 | name = "bench"
62 | harness = false
63 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 |    


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Shmipc
  2 | 
  3 | [![Crates.io](https://img.shields.io/crates/v/shmipc)](https://crates.io/crates/shmipc)
  4 | [![Documentation](https://docs.rs/shmipc/badge.svg)](https://docs.rs/shmipc)
  5 | [![Website](https://img.shields.io/website?up_message=cloudwego&url=https%3A%2F%2Fwww.cloudwego.io%2F)](https://www.cloudwego.io/)
  6 | [![License](https://img.shields.io/crates/l/shmipc)](#license)
  7 | [![Build Status][actions-badge]][actions-url]
  8 | 
  9 | [actions-badge]: https://github.com/cloudwego/shmipc-rs/actions/workflows/ci.yaml/badge.svg
 10 | [actions-url]: https://github.com/cloudwego/shmipc-rs/actions
 11 | 
 12 | English | [中文](README_CN.md)
 13 | 
 14 | ## Introduction
 15 | 
 16 | Shmipc is a high performance inter-process communication library developed by ByteDance.
 17 | It is built on Linux's shared memory technology and uses unix or tcp connection to do process synchronization and finally implements zero copy communication across inter-processes. 
 18 | In IO-intensive or large-package scenarios, it has better performance.
 19 | 
 20 | ## Features
 21 | 
 22 | ### Zero copy
 23 | 
 24 | In an industrial production environment, the unix domain socket and tcp loopback are often used in inter-process communication.The read operation or the write operation need copy data between user space buffer and kernel space buffer. But shmipc directly store data to the share memory, so no copy compared to the former.
 25 | 
 26 | ### Batch IO
 27 | 
 28 | An IO queue was mapped to share memory, which describe the metadata of communication data.
 29 | so that a process could put many request to the IO queue, and other process could handle a batch IO per synchronization. It could effectively reduce the system calls which was brought by process synchronization.
 30 | 
 31 | ## Performance Testing
 32 | 
 33 | The source code bench.rs, doing a performance comparison between shmipc and unix domain socket in ping-pong scenario with different package size. The result is as follows: having a performance improvement whatever small package or large package.
 34 | 
 35 | ```
 36 | benchmark_parallel_ping_pong_by_shmipc_64b
 37 |                         time:   [1.0227 µs 1.0662 µs 1.1158 µs]
 38 | benchmark_parallel_ping_pong_by_uds_64b
 39 |                         time:   [2.4879 µs 2.6609 µs 2.8643 µs]
 40 | 
 41 | benchmark_parallel_ping_pong_by_shmipc_512b
 42 |                         time:   [973.36 ns 1.0128 µs 1.0572 µs]
 43 | benchmark_parallel_ping_pong_by_uds_512b
 44 |                         time:   [2.3158 µs 2.4003 µs 2.4921 µs]
 45 | 
 46 | benchmark_parallel_ping_pong_by_shmipc_1024b
 47 |                         time:   [1.0084 µs 1.0509 µs 1.0996 µs]
 48 | benchmark_parallel_ping_pong_by_uds_1024b
 49 |                         time:   [2.3272 µs 2.4259 µs 2.5353 µs]
 50 | 
 51 | benchmark_parallel_ping_pong_by_shmipc_4096b
 52 |                         time:   [988.93 ns 1.0183 µs 1.0495 µs]
 53 | benchmark_parallel_ping_pong_by_uds_4096b
 54 |                         time:   [3.4969 µs 3.5927 µs 3.6835 µs]
 55 | 
 56 | benchmark_parallel_ping_pong_by_shmipc_16384b
 57 |                         time:   [1.1775 µs 1.2264 µs 1.2806 µs]
 58 | benchmark_parallel_ping_pong_by_uds_16384b
 59 |                         time:   [5.3147 µs 5.8754 µs 6.5797 µs]
 60 | 
 61 | benchmark_parallel_ping_pong_by_shmipc_32768b
 62 |                         time:   [1.1225 µs 1.1844 µs 1.2498 µs]
 63 | benchmark_parallel_ping_pong_by_uds_32768b
 64 |                         time:   [6.9058 µs 7.0930 µs 7.3091 µs]
 65 | 
 66 | benchmark_parallel_ping_pong_by_shmipc_65536b
 67 |                         time:   [1.1807 µs 1.2538 µs 1.3238 µs]
 68 | benchmark_parallel_ping_pong_by_uds_65536b
 69 |                         time:   [15.082 µs 15.419 µs 15.802 µs]
 70 | 
 71 | benchmark_parallel_ping_pong_by_shmipc_262144b
 72 |                         time:   [1.1137 µs 1.1673 µs 1.2253 µs]
 73 | benchmark_parallel_ping_pong_by_uds_262144b
 74 |                         time:   [68.750 µs 70.805 µs 73.121 µs]
 75 | 
 76 | benchmark_parallel_ping_pong_by_shmipc_524288b
 77 |                         time:   [1.1722 µs 1.2599 µs 1.3426 µs]
 78 | benchmark_parallel_ping_pong_by_uds_524288b
 79 |                         time:   [146.55 µs 150.90 µs 155.72 µs]
 80 | 
 81 | benchmark_parallel_ping_pong_by_shmipc_1048576b
 82 |                         time:   [2.3027 µs 2.3872 µs 2.4821 µs]
 83 | benchmark_parallel_ping_pong_by_uds_1048576b
 84 |                         time:   [341.92 µs 354.70 µs 368.29 µs]
 85 |                         
 86 | benchmark_parallel_ping_pong_by_shmipc_4194304b
 87 |                         time:   [4.7158 µs 4.8180 µs 4.9205 µs]
 88 | benchmark_parallel_ping_pong_by_uds_4194304b
 89 |                         time:   [2.1126 ms 2.3210 ms 2.5974 ms]
 90 | ```
 91 | 
 92 | - BenchmarkParallelPingPongByUds, the ping-pong communication base on unix domain socket.
 93 | - BenchmarkParallelPingPongByShmipc, the ping-pong communication base on shmipc.
 94 | - The suffix of the testing case name is the package size of communication, which from 64 Byte to 4 MB.
 95 | 
 96 | ### Quick start
 97 | 
 98 | #### HelloWorld
 99 | 
100 | - [HelloWorldClient](examples/src/hello_world/greeter_client.rs)
101 | - [HelloWorldServer](examples/src/hello_world/greeter_server.rs)
102 | 


--------------------------------------------------------------------------------
/README_CN.md:
--------------------------------------------------------------------------------
 1 | # Shmipc
 2 | 
 3 | [![Crates.io](https://img.shields.io/crates/v/shmipc)](https://crates.io/crates/shmipc)
 4 | [![Documentation](https://docs.rs/shmipc/badge.svg)](https://docs.rs/shmipc)
 5 | [![Website](https://img.shields.io/website?up_message=cloudwego&url=https%3A%2F%2Fwww.cloudwego.io%2F)](https://www.cloudwego.io/)
 6 | [![License](https://img.shields.io/crates/l/shmipc)](#license)
 7 | [![Build Status][actions-badge]][actions-url]
 8 | 
 9 | [actions-badge]: https://github.com/cloudwego/shmipc-rs/actions/workflows/ci.yaml/badge.svg
10 | [actions-url]: https://github.com/cloudwego/shmipc-rs/actions
11 | 
12 | [English](README.md) | 中文 
13 | 
14 | ## 简介
15 | 
16 | Shmipc 是一个由字节跳动开发的高性能进程间通讯库。它基于 Linux 的共享内存构建，使用 unix/tcp 连接进行进程同步，实现进程间通讯零拷贝。在 IO 密集型场景或大包场景能够获得显著的性能收益。
17 | 
18 | ## 特性
19 | 
20 | ### 零拷贝
21 | 
22 | 在工业生产环境中，Unix domain socket 和 Tcp loopback 常用于进程间通讯，读写均涉及通讯数据在用户态 buffer 与内核态 buffer 的来回拷贝。而 Shmipc 使用共享内存存放通讯数据，相对于前者没有数据拷贝。
23 | 
24 | ### 批量收割IO
25 | 
26 | Shmipc 在共享内存中引入了一个 IO 队列来描述通讯数据的元信息，一个进程可以并发地将多个请求的元信息放入 IO 队列，另外一个进程只要需要一次同步就能批量收割 IO.这在 IO 密集的场景下能够有效减少进程同步带来的 system call。
27 | 
28 | ## 性能测试
29 | 
30 | 源码中 bench.rs 进行了 Shmipc 与 Unix domain socket 在 ping-pong 场景下不同数据包大小的性能对比，结果如下所示: 从小包到大包均有性能提升。
31 | 
32 | ```
33 | benchmark_parallel_ping_pong_by_shmipc_64b
34 |                         time:   [1.0227 µs 1.0662 µs 1.1158 µs]
35 | benchmark_parallel_ping_pong_by_uds_64b
36 |                         time:   [2.4879 µs 2.6609 µs 2.8643 µs]
37 | 
38 | benchmark_parallel_ping_pong_by_shmipc_512b
39 |                         time:   [973.36 ns 1.0128 µs 1.0572 µs]
40 | benchmark_parallel_ping_pong_by_uds_512b
41 |                         time:   [2.3158 µs 2.4003 µs 2.4921 µs]
42 | 
43 | benchmark_parallel_ping_pong_by_shmipc_1024b
44 |                         time:   [1.0084 µs 1.0509 µs 1.0996 µs]
45 | benchmark_parallel_ping_pong_by_uds_1024b
46 |                         time:   [2.3272 µs 2.4259 µs 2.5353 µs]
47 | 
48 | benchmark_parallel_ping_pong_by_shmipc_4096b
49 |                         time:   [988.93 ns 1.0183 µs 1.0495 µs]
50 | benchmark_parallel_ping_pong_by_uds_4096b
51 |                         time:   [3.4969 µs 3.5927 µs 3.6835 µs]
52 | 
53 | benchmark_parallel_ping_pong_by_shmipc_16384b
54 |                         time:   [1.1775 µs 1.2264 µs 1.2806 µs]
55 | benchmark_parallel_ping_pong_by_uds_16384b
56 |                         time:   [5.3147 µs 5.8754 µs 6.5797 µs]
57 | 
58 | benchmark_parallel_ping_pong_by_shmipc_32768b
59 |                         time:   [1.1225 µs 1.1844 µs 1.2498 µs]
60 | benchmark_parallel_ping_pong_by_uds_32768b
61 |                         time:   [6.9058 µs 7.0930 µs 7.3091 µs]
62 | 
63 | benchmark_parallel_ping_pong_by_shmipc_65536b
64 |                         time:   [1.1807 µs 1.2538 µs 1.3238 µs]
65 | benchmark_parallel_ping_pong_by_uds_65536b
66 |                         time:   [15.082 µs 15.419 µs 15.802 µs]
67 | 
68 | benchmark_parallel_ping_pong_by_shmipc_262144b
69 |                         time:   [1.1137 µs 1.1673 µs 1.2253 µs]
70 | benchmark_parallel_ping_pong_by_uds_262144b
71 |                         time:   [68.750 µs 70.805 µs 73.121 µs]
72 | 
73 | benchmark_parallel_ping_pong_by_shmipc_524288b
74 |                         time:   [1.1722 µs 1.2599 µs 1.3426 µs]
75 | benchmark_parallel_ping_pong_by_uds_524288b
76 |                         time:   [146.55 µs 150.90 µs 155.72 µs]
77 | 
78 | benchmark_parallel_ping_pong_by_shmipc_1048576b
79 |                         time:   [2.3027 µs 2.3872 µs 2.4821 µs]
80 | benchmark_parallel_ping_pong_by_uds_1048576b
81 |                         time:   [341.92 µs 354.70 µs 368.29 µs]
82 |                         
83 | benchmark_parallel_ping_pong_by_shmipc_4194304b
84 |                         time:   [4.7158 µs 4.8180 µs 4.9205 µs]
85 | benchmark_parallel_ping_pong_by_uds_4194304b
86 |                         time:   [2.1126 ms 2.3210 ms 2.5974 ms]
87 | ```
88 | 
89 | - BenchmarkParallelPingPongByUds，基于 Unix Domain Socket 进行 ping-pong 通讯。
90 | - BenchmarkParallelPingPongByShmipc，基于 Shmipc 进行 ping-pong 通讯。
91 | - 后缀为 ping-pong 的数据包大小， 从 64Byte ~ 4MB 不等。
92 | 
93 | ### 快速开始
94 | 
95 | #### HelloWorld
96 | 
97 | - [HelloWorld客户端](examples/src/hello_world/greeter_client.rs)
98 | - [HelloWorld服务端](examples/src/hello_world/greeter_server.rs)
99 | 


--------------------------------------------------------------------------------
/_typos.toml:
--------------------------------------------------------------------------------
1 | # Typo check: https://github.com/crate-ci/typos
2 | 


--------------------------------------------------------------------------------
/benches/bench.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::os::unix::net::SocketAddr;
 16 | 
 17 | use criterion::{Criterion, criterion_group, criterion_main};
 18 | use shmipc::{
 19 |     AsyncReadShm, AsyncWriteShm, BufferReader, BufferSlice, Error, LinkedBuffer, Listener,
 20 |     SessionManager, SessionManagerConfig, Stream, config::SizePercentPair, consts::MemMapType,
 21 | };
 22 | use tokio::{
 23 |     io::{AsyncReadExt, AsyncWriteExt},
 24 |     net::{UnixListener, UnixStream},
 25 |     sync::oneshot,
 26 |     time::Instant,
 27 | };
 28 | 
 29 | fn criterion_benchmark(c: &mut Criterion) {
 30 |     let mut group = c.benchmark_group("shmipc");
 31 |     let sizes = [
 32 |         64,
 33 |         512,
 34 |         1024,
 35 |         4096,
 36 |         16 << 10,
 37 |         32 << 10,
 38 |         64 << 10,
 39 |         256 << 10,
 40 |         512 << 10,
 41 |         1 << 20,
 42 |         4 << 20,
 43 |     ];
 44 |     for size in sizes {
 45 |         group.bench_function(
 46 |             format!("benchmark_parallel_ping_pong_by_shmipc_{}b", size),
 47 |             |b| {
 48 |                 b.to_async(
 49 |                     tokio::runtime::Builder::new_multi_thread()
 50 |                         .enable_all()
 51 |                         .build()
 52 |                         .unwrap(),
 53 |                 )
 54 |                 .iter_custom(|iters| async move {
 55 |                     let rand = rand::random::<u64>();
 56 |                     let path = format!("/dev/shm/shmipc{}.sock", rand);
 57 |                     let mut sm_config = benchmark_config();
 58 |                     if size >= 4 << 20 {
 59 |                         sm_config.config_mut().share_memory_buffer_cap = 1 << 30;
 60 |                     }
 61 |                     sm_config.config_mut().buffer_slice_sizes = vec![
 62 |                         SizePercentPair {
 63 |                             size: size + 256,
 64 |                             percent: 70,
 65 |                         },
 66 |                         SizePercentPair {
 67 |                             size: (16 << 10) + 256,
 68 |                             percent: 20,
 69 |                         },
 70 |                         SizePercentPair {
 71 |                             size: (64 << 10) + 256,
 72 |                             percent: 10,
 73 |                         },
 74 |                     ];
 75 |                     sm_config
 76 |                         .config_mut()
 77 |                         .share_memory_path_prefix
 78 |                         .push_str(rand.to_string().as_str());
 79 |                     sm_config = sm_config.with_session_num(1);
 80 |                     let mut server = Listener::new(
 81 |                         SocketAddr::from_pathname(path.clone()).unwrap(),
 82 |                         sm_config.config().clone(),
 83 |                     )
 84 |                     .await
 85 |                     .unwrap();
 86 |                     let (tx, rx) = oneshot::channel();
 87 |                     let (tx2, mut rx2) = oneshot::channel();
 88 |                     tokio::spawn(async move {
 89 |                         loop {
 90 |                             tokio::select! {
 91 |                                 r = server.accept() => {
 92 |                                     let mut stream = r.unwrap().unwrap();
 93 |                                     tokio::spawn(async move {
 94 |                                         loop {
 95 |                                             if !must_read(&mut stream, size).await {
 96 |                                                 return;
 97 |                                             }
 98 |                                             stream.recv_buf().release_previous_read();
 99 |                                             must_write(&mut stream, size).await;
100 |                                         }
101 |                                     });
102 |                                 }
103 |                                 _ = &mut rx2 => {
104 |                                     server.close().await;
105 |                                     tx.send(()).unwrap();
106 |                                     return;
107 |                                 }
108 | 
109 |                             }
110 |                         }
111 |                     });
112 | 
113 |                     let client =
114 |                         SessionManager::new(sm_config, SocketAddr::from_pathname(path).unwrap())
115 |                             .await
116 |                             .unwrap();
117 |                     let mut handlers = Vec::with_capacity(200);
118 |                     let start = Instant::now();
119 |                     for _ in 0..199 {
120 |                         let client = client.clone();
121 |                         handlers.push(tokio::spawn(async move {
122 |                             let mut stream = client.get_stream().unwrap();
123 |                             for _ in 0..iters / 200 {
124 |                                 must_write(&mut stream, size).await;
125 |                                 must_read(&mut stream, size).await;
126 |                                 stream.release_read_and_reuse();
127 |                             }
128 |                             stream.close().await.unwrap();
129 |                         }));
130 |                     }
131 |                     for handler in handlers {
132 |                         handler.await.unwrap();
133 |                     }
134 |                     let cost = start.elapsed();
135 |                     client.close().await;
136 |                     tx2.send(()).unwrap();
137 |                     rx.await.unwrap();
138 |                     cost
139 |                 })
140 |             },
141 |         );
142 | 
143 |         group.bench_function(
144 |             format!("benchmark_parallel_ping_pong_by_uds_{}b", size),
145 |             |b| {
146 |                 b.to_async(
147 |                     tokio::runtime::Builder::new_multi_thread()
148 |                         .enable_all()
149 |                         .build()
150 |                         .unwrap(),
151 |                 )
152 |                 .iter_custom(|iters| async move {
153 |                     let rand = rand::random::<u64>();
154 |                     let (tx, rx) = oneshot::channel();
155 |                     tokio::spawn(async move {
156 |                         let listener =
157 |                             UnixListener::bind(format!("/dev/shm/uds{}.sock", rand)).unwrap();
158 |                         _ = tx.send(());
159 |                         loop {
160 |                             let (mut conn, _) = listener.accept().await.unwrap();
161 |                             tokio::spawn(async move {
162 |                                 let mut read_buf = vec![0u8; size as usize];
163 |                                 let write_buf = vec![0u8; size as usize];
164 |                                 loop {
165 |                                     if let Err(err) = conn.read_exact(&mut read_buf).await {
166 |                                         if err.kind() == std::io::ErrorKind::UnexpectedEof {
167 |                                             return;
168 |                                         }
169 |                                         panic!("read err: {}", err);
170 |                                     }
171 |                                     conn.write_all(&write_buf).await.unwrap();
172 |                                     conn.flush().await.unwrap();
173 |                                 }
174 |                             });
175 |                         }
176 |                     });
177 |                     _ = rx.await;
178 |                     let mut handlers = Vec::with_capacity(200);
179 |                     let start = Instant::now();
180 |                     for _ in 0..199 {
181 |                         handlers.push(tokio::spawn(async move {
182 |                             let mut stream =
183 |                                 UnixStream::connect(format!("/dev/shm/uds{}.sock", rand))
184 |                                     .await
185 |                                     .unwrap();
186 |                             let mut read_buf = vec![0u8; size as usize];
187 |                             let write_buf = vec![0u8; size as usize];
188 |                             for _ in 0..iters / 200 {
189 |                                 stream.write_all(&write_buf).await.unwrap();
190 |                                 stream.flush().await.unwrap();
191 |                                 stream.read_exact(&mut read_buf).await.unwrap();
192 |                             }
193 |                         }));
194 |                     }
195 |                     for handler in handlers {
196 |                         handler.await.unwrap();
197 |                     }
198 |                     start.elapsed()
199 |                 })
200 |             },
201 |         );
202 |     }
203 | }
204 | 
205 | criterion_group!(benches, criterion_benchmark);
206 | criterion_main!(benches);
207 | 
208 | fn benchmark_config() -> SessionManagerConfig {
209 |     let mut c = SessionManagerConfig::new();
210 |     c.config_mut().queue_cap = 65536;
211 |     c.config_mut().connection_write_timeout = std::time::Duration::from_secs(1);
212 |     c.config_mut().share_memory_buffer_cap = 256 << 20;
213 |     c.config_mut().mem_map_type = MemMapType::MemMapTypeMemFd;
214 |     c
215 | }
216 | 
217 | async fn must_write(s: &mut Stream, size: u32) {
218 |     write_empty_buffer(s.send_buf(), size);
219 |     loop {
220 |         match s.flush(false).await {
221 |             Err(e) => match e {
222 |                 Error::QueueFull => {
223 |                     tokio::time::sleep(std::time::Duration::from_micros(1)).await;
224 |                     continue;
225 |                 }
226 |                 _ => {
227 |                     panic!("must write err:{}", e);
228 |                 }
229 |             },
230 |             Ok(_) => {
231 |                 return;
232 |             }
233 |         }
234 |     }
235 | }
236 | 
237 | fn write_empty_buffer(l: &mut LinkedBuffer, size: u32) {
238 |     if size == 0 {
239 |         return;
240 |     }
241 |     let mut wrote = 0;
242 |     loop {
243 |         if l.slice_list().write_slice.is_none() {
244 |             l.alloc(size - wrote);
245 |             l.slice_list_mut().write_slice = l.slice_list().front_slice;
246 |         }
247 |         wrote += write_empty_slice(l.slice_list().write_mut().unwrap(), size - wrote);
248 |         if wrote < size {
249 |             if l.slice_list().write().unwrap().next().is_none() {
250 |                 l.alloc(size - wrote);
251 |             }
252 |             l.slice_list_mut().write_slice = l.slice_list().write().unwrap().next_slice;
253 |         } else {
254 |             break;
255 |         }
256 |     }
257 |     *l.len_mut() += size as usize;
258 | }
259 | 
260 | fn write_empty_slice(slice: &mut BufferSlice, size: u32) -> u32 {
261 |     slice.write_index += size as usize;
262 |     if slice.write_index > slice.cap as usize {
263 |         let wrote = slice.cap - (slice.write_index as u32 - size);
264 |         slice.write_index = slice.cap as usize;
265 |         return wrote;
266 |     }
267 |     size
268 | }
269 | 
270 | async fn must_read(s: &mut Stream, size: u32) -> bool {
271 |     match s.discard(size as usize).await {
272 |         Err(e) => match e {
273 |             Error::StreamClosed | Error::EndOfStream => false,
274 |             _ => {
275 |                 panic!("must read err:{}", e);
276 |             }
277 |         },
278 |         Ok(_) => true,
279 |     }
280 | }
281 | 


--------------------------------------------------------------------------------
/examples/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "examples"
 3 | version = "0.1.0"
 4 | edition = "2024"
 5 | publish = false
 6 | 
 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 8 | 
 9 | [dependencies]
10 | nix = "0.29"
11 | shmipc = { path = "../" }
12 | tokio = { version = "1", features = ["full"] }
13 | tracing-subscriber = "0.3"
14 | 
15 | [[bin]]
16 | name = "greeter_client"
17 | path = "src/hello_world/greeter_client.rs"
18 | 
19 | [[bin]]
20 | name = "greeter_server"
21 | path = "src/hello_world/greeter_server.rs"
22 | 


--------------------------------------------------------------------------------
/examples/src/hello_world/greeter_client.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::os::unix::net::SocketAddr;
16 | 
17 | use shmipc::{AsyncReadShm, AsyncWriteShm, SessionManager, SessionManagerConfig};
18 | 
19 | #[tokio::main]
20 | async fn main() {
21 |     tracing_subscriber::fmt::init();
22 |     let dir = std::env::current_dir().unwrap();
23 |     let uds_path = SocketAddr::from_pathname(dir.join("../ipc_test.sock")).unwrap();
24 | 
25 |     let mut conf = SessionManagerConfig::new();
26 |     conf.config_mut().mem_map_type = shmipc::consts::MemMapType::MemMapTypeMemFd;
27 |     conf.config_mut().share_memory_path_prefix = "/dev/shm/client.ipc.shm".to_string();
28 |     #[cfg(target_os = "macos")]
29 |     {
30 |         conf.config.share_memory_path_prefix = "/tmp/client.ipc.shm".to_string();
31 |         conf.config.queue_path = "/tmp/client.ipc.shm_queue".to_string();
32 |     }
33 | 
34 |     let sm = SessionManager::new(conf, uds_path).await.unwrap();
35 |     let mut stream = sm.get_stream().unwrap();
36 |     let request_msg = "client say hello world!!!";
37 |     println!(
38 |         "size: {}",
39 |         stream.write_bytes(request_msg.as_bytes()).unwrap()
40 |     );
41 |     stream.flush(true).await.unwrap();
42 |     let resp_msg = stream
43 |         .read_bytes("server hello world!!!".len())
44 |         .await
45 |         .unwrap();
46 |     println!(
47 |         "client stream receive response {}",
48 |         String::from_utf8(resp_msg.to_vec()).unwrap()
49 |     );
50 |     sm.put_back(stream).await;
51 |     sm.close().await;
52 | }
53 | 


--------------------------------------------------------------------------------
/examples/src/hello_world/greeter_server.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::os::unix::net::SocketAddr;
16 | 
17 | use nix::unistd::unlink;
18 | use shmipc::{AsyncReadShm, AsyncWriteShm, Listener, config::Config};
19 | 
20 | #[tokio::main]
21 | async fn main() {
22 |     tracing_subscriber::fmt::init();
23 |     let dir = std::env::current_dir().unwrap();
24 |     let binding = dir.join("../ipc_test.sock");
25 |     let uds_path = binding.to_str().unwrap();
26 |     _ = unlink(uds_path);
27 | 
28 |     let mut ln = Listener::new(
29 |         SocketAddr::from_pathname(binding).unwrap(),
30 |         Config::default(),
31 |     )
32 |     .await
33 |     .unwrap();
34 |     let mut stream = ln.accept().await.unwrap().unwrap();
35 |     let req_msg = stream
36 |         .read_bytes("client say hello world!!!".len())
37 |         .await
38 |         .unwrap();
39 |     println!(
40 |         "server receive request {}",
41 |         String::from_utf8(req_msg.to_vec()).unwrap()
42 |     );
43 | 
44 |     let resp_msg = "server hello world!!!";
45 |     stream.write_bytes(resp_msg.as_bytes()).unwrap();
46 |     stream.flush(true).await.unwrap();
47 |     tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
48 |     stream.close().await.unwrap();
49 | }
50 | 


--------------------------------------------------------------------------------
/licenses/LICENSE-shmipc-go:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.


--------------------------------------------------------------------------------
/licenses/LICENSE-shmipc-spec:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.


--------------------------------------------------------------------------------
/rustfmt.toml:
--------------------------------------------------------------------------------
 1 | comment_width = 100
 2 | edition = "2024"
 3 | format_code_in_doc_comments = true
 4 | format_strings = true
 5 | group_imports = "StdExternalCrate"
 6 | imports_granularity = "Crate"
 7 | normalize_comments = true
 8 | normalize_doc_attributes = true
 9 | wrap_comments = true
10 | 


--------------------------------------------------------------------------------
/src/buffer/buf.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{borrow::Borrow, cmp, hash, ops::Deref};
 16 | 
 17 | use bytes::Bytes;
 18 | 
 19 | #[derive(Debug)]
 20 | pub enum Buf<'shm> {
 21 |     Shm(&'shm [u8]),
 22 |     Exm(Bytes),
 23 | }
 24 | 
 25 | impl Buf<'_> {
 26 |     /// # Safety
 27 |     ///
 28 |     /// The caller must ensure that the shm buf is valid before the return value is out of usage.
 29 |     pub unsafe fn into_bytes(self) -> Bytes {
 30 |         match self {
 31 |             Buf::Shm(buf) => {
 32 |                 Bytes::from_static(unsafe { std::mem::transmute::<&[u8], &[u8]>(buf) })
 33 |             }
 34 |             Buf::Exm(buf) => buf,
 35 |         }
 36 |     }
 37 | }
 38 | 
 39 | impl Deref for Buf<'_> {
 40 |     type Target = [u8];
 41 | 
 42 |     fn deref(&self) -> &[u8] {
 43 |         match self {
 44 |             Buf::Shm(buf) => buf,
 45 |             Buf::Exm(buf) => buf,
 46 |         }
 47 |     }
 48 | }
 49 | 
 50 | impl AsRef<[u8]> for Buf<'_> {
 51 |     #[inline]
 52 |     fn as_ref(&self) -> &[u8] {
 53 |         match self {
 54 |             Buf::Shm(buf) => buf,
 55 |             Buf::Exm(buf) => buf,
 56 |         }
 57 |     }
 58 | }
 59 | 
 60 | impl hash::Hash for Buf<'_> {
 61 |     fn hash<H>(&self, state: &mut H)
 62 |     where
 63 |         H: hash::Hasher,
 64 |     {
 65 |         match self {
 66 |             Buf::Shm(buf) => buf.hash(state),
 67 |             Buf::Exm(buf) => buf.hash(state),
 68 |         }
 69 |     }
 70 | }
 71 | 
 72 | impl Borrow<[u8]> for Buf<'_> {
 73 |     fn borrow(&self) -> &[u8] {
 74 |         match self {
 75 |             Buf::Shm(buf) => buf,
 76 |             Buf::Exm(buf) => buf,
 77 |         }
 78 |     }
 79 | }
 80 | 
 81 | impl PartialEq for Buf<'_> {
 82 |     fn eq(&self, other: &Buf<'_>) -> bool {
 83 |         self.as_ref().eq(other.as_ref())
 84 |     }
 85 | }
 86 | 
 87 | impl PartialOrd for Buf<'_> {
 88 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
 89 |         Some(self.cmp(other))
 90 |     }
 91 | }
 92 | 
 93 | impl Ord for Buf<'_> {
 94 |     fn cmp(&self, other: &Buf<'_>) -> cmp::Ordering {
 95 |         self.as_ref().cmp(other.as_ref())
 96 |     }
 97 | }
 98 | 
 99 | impl Eq for Buf<'_> {}
100 | 
101 | impl PartialEq<[u8]> for Buf<'_> {
102 |     fn eq(&self, other: &[u8]) -> bool {
103 |         self.as_ref() == other
104 |     }
105 | }
106 | 
107 | impl PartialOrd<[u8]> for Buf<'_> {
108 |     fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
109 |         self.as_ref().partial_cmp(other)
110 |     }
111 | }
112 | 
113 | impl PartialEq<Buf<'_>> for [u8] {
114 |     fn eq(&self, other: &Buf<'_>) -> bool {
115 |         *other == *self
116 |     }
117 | }
118 | 
119 | impl PartialOrd<Buf<'_>> for [u8] {
120 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
121 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
122 |     }
123 | }
124 | 
125 | impl PartialEq<str> for Buf<'_> {
126 |     fn eq(&self, other: &str) -> bool {
127 |         self.as_ref() == other.as_bytes()
128 |     }
129 | }
130 | 
131 | impl PartialOrd<str> for Buf<'_> {
132 |     fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
133 |         self.as_ref().partial_cmp(other.as_bytes())
134 |     }
135 | }
136 | 
137 | impl PartialEq<Buf<'_>> for str {
138 |     fn eq(&self, other: &Buf<'_>) -> bool {
139 |         *other == *self
140 |     }
141 | }
142 | 
143 | impl PartialOrd<Buf<'_>> for str {
144 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
145 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
146 |     }
147 | }
148 | 
149 | impl PartialEq<Vec<u8>> for Buf<'_> {
150 |     fn eq(&self, other: &Vec<u8>) -> bool {
151 |         *self == other[..]
152 |     }
153 | }
154 | 
155 | impl PartialOrd<Vec<u8>> for Buf<'_> {
156 |     fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
157 |         self.as_ref().partial_cmp(&other[..])
158 |     }
159 | }
160 | 
161 | impl PartialEq<Buf<'_>> for Vec<u8> {
162 |     fn eq(&self, other: &Buf<'_>) -> bool {
163 |         *other == *self
164 |     }
165 | }
166 | 
167 | impl PartialOrd<Buf<'_>> for Vec<u8> {
168 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
169 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
170 |     }
171 | }
172 | 
173 | impl PartialEq<String> for Buf<'_> {
174 |     fn eq(&self, other: &String) -> bool {
175 |         *self == other[..]
176 |     }
177 | }
178 | 
179 | impl PartialOrd<String> for Buf<'_> {
180 |     fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
181 |         self.as_ref().partial_cmp(other.as_bytes())
182 |     }
183 | }
184 | 
185 | impl PartialEq<Buf<'_>> for String {
186 |     fn eq(&self, other: &Buf<'_>) -> bool {
187 |         *other == *self
188 |     }
189 | }
190 | 
191 | impl PartialOrd<Buf<'_>> for String {
192 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
193 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
194 |     }
195 | }
196 | 
197 | impl PartialEq<Buf<'_>> for &[u8] {
198 |     fn eq(&self, other: &Buf<'_>) -> bool {
199 |         other.as_ref() == *self
200 |     }
201 | }
202 | 
203 | impl PartialOrd<Buf<'_>> for &[u8] {
204 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
205 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
206 |     }
207 | }
208 | 
209 | impl PartialEq<Buf<'_>> for &str {
210 |     fn eq(&self, other: &Buf<'_>) -> bool {
211 |         *other.as_ref() == *self.as_bytes()
212 |     }
213 | }
214 | 
215 | impl PartialOrd<Buf<'_>> for &str {
216 |     fn partial_cmp(&self, other: &Buf<'_>) -> Option<cmp::Ordering> {
217 |         <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
218 |     }
219 | }
220 | 


--------------------------------------------------------------------------------
/src/buffer/list.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::sync::atomic::{AtomicI32, AtomicU32, Ordering};
 16 | 
 17 | use anyhow::anyhow;
 18 | use memmap2::MmapMut;
 19 | 
 20 | use super::slice::{BufferHeader, BufferSlice};
 21 | use crate::{
 22 |     Error,
 23 |     buffer::manager::{
 24 |         BUFFER_DATA_START_OFFSET, BUFFER_FLAG_OFFSET, BUFFER_HEADER_SIZE, BUFFER_SIZE_OFFSET,
 25 |         HAS_NEXT_BUFFER_FLAG, NEXT_BUFFER_OFFSET,
 26 |     },
 27 | };
 28 | 
 29 | /// size 4 | cap 4 | head 4 | tail 4 | capPerBuffer 4 | push count 8 | pop count 8
 30 | pub const BUFFER_LIST_HEADER_SIZE: u32 = 36;
 31 | 
 32 | /// BufferList's layout in share memory: size 4 byte | cap 4 byte | head 4 byte | tail 4 byte |
 33 | /// cap_per_buffer 4 byte | buffer_region n byte
 34 | ///
 35 | /// Thread safe, lock free. Support push & pop concurrently even cross different process.
 36 | #[derive(Clone, Copy, Debug)]
 37 | pub struct BufferList {
 38 |     /// the number of free buffer in list
 39 |     pub(crate) size: *const AtomicI32,
 40 |     /// the capacity of list
 41 |     pub(crate) cap: *const AtomicU32,
 42 |     /// it points to the first free buffer, whose offset in buffer_region
 43 |     pub(crate) head: *const AtomicU32,
 44 |     /// it points to the last free buffer, whose offset in buffer_region
 45 |     pub(crate) tail: *const AtomicU32,
 46 |     /// the capacity of per buffer
 47 |     pub(crate) cap_per_buffer: *mut u32,
 48 |     pub(crate) counter: *const AtomicI32,
 49 |     /// underlying memory
 50 |     buffer_region: *mut u8,
 51 |     buffer_region_len: u32,
 52 |     buffer_region_offset_in_shm: u32,
 53 |     #[allow(dead_code)]
 54 |     /// the buffer_list's location offset in share memory
 55 |     pub(crate) offset_in_shm: u32,
 56 | }
 57 | 
 58 | unsafe impl Send for BufferList {}
 59 | unsafe impl Sync for BufferList {}
 60 | 
 61 | impl BufferList {
 62 |     /// Create a new BufferList in share memory, used for client side.
 63 |     pub fn create(
 64 |         buffer_num: u32,
 65 |         cap_per_buffer: u32,
 66 |         mem: &MmapMut,
 67 |         offset_in_mem: u32,
 68 |     ) -> Result<Self, anyhow::Error> {
 69 |         if buffer_num == 0 || cap_per_buffer == 0 {
 70 |             return Err(anyhow!(
 71 |                 "buffer_num:{} and cap_per_buffer:{} could not be 0",
 72 |                 buffer_num,
 73 |                 cap_per_buffer
 74 |             ));
 75 |         }
 76 |         let at_least_size = count_buffer_list_mem_size(buffer_num, cap_per_buffer);
 77 |         let mem_len = mem.len() as u32;
 78 |         if mem_len < (offset_in_mem + at_least_size)
 79 |             || offset_in_mem > mem_len
 80 |             || at_least_size > mem_len
 81 |         {
 82 |             return Err(anyhow!(
 83 |                 "mem's size is at least:{} but:{} offset_in_mem:{} at_least_size:{}",
 84 |                 offset_in_mem + at_least_size,
 85 |                 mem_len,
 86 |                 offset_in_mem,
 87 |                 at_least_size
 88 |             ));
 89 |         }
 90 | 
 91 |         let buffer_region_start = offset_in_mem + BUFFER_LIST_HEADER_SIZE;
 92 |         let buffer_region_end = offset_in_mem + at_least_size;
 93 |         if buffer_region_end <= buffer_region_start {
 94 |             return Err(anyhow!(
 95 |                 "buffer_region_start:{} buffer_region_end:{} slice bounds out of range",
 96 |                 buffer_region_start,
 97 |                 buffer_region_end
 98 |             ));
 99 |         }
100 |         let ptr = mem.as_ptr();
101 | 
102 |         let offset = offset_in_mem as isize;
103 |         let b = Self {
104 |             size: unsafe { ptr.offset(offset) as *const AtomicI32 },
105 |             cap: unsafe { ptr.offset(offset + 4) as *const AtomicU32 },
106 |             head: unsafe { ptr.offset(offset + 8) as *const AtomicU32 },
107 |             tail: unsafe { ptr.offset(offset + 12) as *const AtomicU32 },
108 |             cap_per_buffer: unsafe { ptr.offset(offset + 16) as *mut u32 },
109 |             counter: unsafe { ptr.offset(offset + 20) as *const AtomicI32 },
110 |             buffer_region: unsafe {
111 |                 ptr.offset(offset + BUFFER_LIST_HEADER_SIZE as isize) as *mut u8
112 |             },
113 |             buffer_region_len: at_least_size - BUFFER_LIST_HEADER_SIZE,
114 |             buffer_region_offset_in_shm: offset_in_mem + BUFFER_LIST_HEADER_SIZE,
115 |             offset_in_shm: offset_in_mem,
116 |         };
117 |         unsafe {
118 |             (*b.size).store(buffer_num as i32, Ordering::SeqCst);
119 |             (*b.cap).store(buffer_num, Ordering::SeqCst);
120 |             (*b.head).store(0, Ordering::SeqCst);
121 |             (*b.tail).store(
122 |                 (buffer_num - 1) * (cap_per_buffer + BUFFER_HEADER_SIZE),
123 |                 Ordering::SeqCst,
124 |             );
125 |             *b.cap_per_buffer = cap_per_buffer;
126 |             (*b.counter).store(0, Ordering::SeqCst);
127 |         }
128 | 
129 |         tracing::info!(
130 |             "create buffer list: buffer_num:{} cap_per_buffer:{} offset_in_mem:{} need_size:{} \
131 |              buffer_region_len:{}",
132 |             buffer_num,
133 |             cap_per_buffer,
134 |             offset_in_mem,
135 |             at_least_size,
136 |             at_least_size - BUFFER_LIST_HEADER_SIZE,
137 |         );
138 | 
139 |         let mut current = 0;
140 |         let mut next;
141 |         for i in 0..buffer_num {
142 |             next = current + cap_per_buffer + BUFFER_HEADER_SIZE;
143 |             unsafe {
144 |                 *(b.buffer_region.offset(current as isize) as *mut u32) = cap_per_buffer;
145 |                 *(b.buffer_region
146 |                     .offset((current + BUFFER_SIZE_OFFSET) as isize)
147 |                     as *mut u32) = 0;
148 |                 *(b.buffer_region
149 |                     .offset((current + BUFFER_DATA_START_OFFSET) as isize)
150 |                     as *mut u32) = 0;
151 |                 if i < (buffer_num - 1) {
152 |                     *(b.buffer_region
153 |                         .offset((current + NEXT_BUFFER_OFFSET) as isize)
154 |                         as *mut u32) = next;
155 |                     *(b.buffer_region
156 |                         .offset((current + BUFFER_FLAG_OFFSET) as isize)
157 |                         as *mut u32) = 0;
158 |                     *b.buffer_region
159 |                         .offset((current + BUFFER_FLAG_OFFSET) as isize) |= HAS_NEXT_BUFFER_FLAG;
160 |                 }
161 |             }
162 |             current = next;
163 |         }
164 |         unsafe {
165 |             let tail = b
166 |                 .buffer_region
167 |                 .offset((*b.tail).load(Ordering::SeqCst) as isize);
168 |             *(tail.offset(BUFFER_FLAG_OFFSET as isize) as *mut u32) = 0;
169 |         }
170 | 
171 |         Ok(b)
172 |     }
173 | 
174 |     /// Mapping a BufferList from share memory, used for server side.
175 |     pub fn mapping(mem: &MmapMut, offset_in_shm: u32) -> Result<Self, anyhow::Error> {
176 |         if mem.len() < (offset_in_shm + BUFFER_LIST_HEADER_SIZE) as usize {
177 |             return Err(anyhow!(
178 |                 "mapping buffer list failed, mem's size is at least {}, ",
179 |                 offset_in_shm + BUFFER_LIST_HEADER_SIZE,
180 |             ));
181 |         }
182 | 
183 |         let ptr = mem.as_ptr();
184 |         let offset = offset_in_shm as isize;
185 | 
186 |         let size = unsafe { ptr.offset(offset) as *const AtomicI32 };
187 |         let cap = unsafe { ptr.offset(offset + 4) as *const AtomicU32 };
188 |         let head = unsafe { ptr.offset(offset + 8) as *const AtomicU32 };
189 |         let tail = unsafe { ptr.offset(offset + 12) as *const AtomicU32 };
190 |         let cap_per_buffer = unsafe { ptr.offset(offset + 16) as *mut u32 };
191 |         let counter = unsafe { ptr.offset(offset + 24) as *const AtomicI32 };
192 | 
193 |         let need_size =
194 |             count_buffer_list_mem_size(unsafe { (*cap).load(Ordering::SeqCst) }, unsafe {
195 |                 *cap_per_buffer
196 |             });
197 |         if offset_in_shm + need_size > mem.len() as u32
198 |             || offset_in_shm + need_size < offset_in_shm + BUFFER_LIST_HEADER_SIZE
199 |         {
200 |             return Err(unsafe {
201 |                 anyhow!(
202 |                     "mapping buffer list failed, size:{} cap:{} head:{} tail:{} cap_per_buffer:{} \
203 |                      err: mem's size is at least {} but:{}",
204 |                     (*size).load(Ordering::SeqCst),
205 |                     (*cap).load(Ordering::SeqCst),
206 |                     (*head).load(Ordering::SeqCst),
207 |                     (*tail).load(Ordering::SeqCst),
208 |                     *cap_per_buffer,
209 |                     need_size,
210 |                     mem.len()
211 |                 )
212 |             });
213 |         }
214 | 
215 |         Ok(Self {
216 |             size,
217 |             cap,
218 |             head,
219 |             tail,
220 |             cap_per_buffer,
221 |             counter,
222 |             buffer_region: unsafe {
223 |                 ptr.offset(offset + BUFFER_LIST_HEADER_SIZE as isize) as *mut u8
224 |             },
225 |             buffer_region_len: need_size - BUFFER_LIST_HEADER_SIZE,
226 |             buffer_region_offset_in_shm: offset_in_shm + BUFFER_LIST_HEADER_SIZE,
227 |             offset_in_shm,
228 |         })
229 |     }
230 | 
231 |     /// Push a buffer to the list.
232 |     pub fn push(&self, mut buffer: BufferSlice) {
233 |         buffer.reset();
234 |         loop {
235 |             let old_tail = unsafe { (*self.tail).load(Ordering::SeqCst) };
236 |             let new_tail = buffer.offset_in_shm - self.buffer_region_offset_in_shm;
237 |             unsafe {
238 |                 if (*self.tail)
239 |                     .compare_exchange(old_tail, new_tail, Ordering::SeqCst, Ordering::SeqCst)
240 |                     .is_ok()
241 |                 {
242 |                     BufferHeader(self.buffer_region.offset(old_tail as isize)).link_next(new_tail);
243 |                     (*self.size).fetch_add(1, Ordering::SeqCst);
244 |                     (*self.counter).fetch_sub(1, Ordering::SeqCst);
245 | 
246 |                     return;
247 |                 }
248 |             }
249 |         }
250 |     }
251 | 
252 |     /// Pop a buffer from the list.
253 |     ///
254 |     /// When data races occurred, it will retry 200 times at most. if still failed to pop, return
255 |     /// NoMoreBuffer error.
256 |     pub fn pop(&self) -> Result<BufferSlice, Error> {
257 |         let mut old_head = unsafe { (*self.head).load(Ordering::SeqCst) };
258 |         let remain = unsafe { (*self.size).fetch_sub(1, Ordering::SeqCst) };
259 | 
260 |         if remain <= 1
261 |             || unsafe {
262 |                 old_head + BUFFER_HEADER_SIZE + *self.cap_per_buffer > self.buffer_region_len
263 |             }
264 |         {
265 |             unsafe {
266 |                 (*self.size).fetch_add(1, Ordering::SeqCst);
267 |             }
268 |             return Err(Error::NoMoreBuffer);
269 |         }
270 | 
271 |         // when data races occurred, max retry 200 times.
272 |         for _ in 0..200 {
273 |             unsafe {
274 |                 let bh = BufferHeader(self.buffer_region.offset(old_head as isize));
275 | 
276 |                 if bh.has_next() {
277 |                     if (*self.head)
278 |                         .compare_exchange(
279 |                             old_head,
280 |                             bh.next_buffer_offset(),
281 |                             Ordering::SeqCst,
282 |                             Ordering::SeqCst,
283 |                         )
284 |                         .is_ok()
285 |                     {
286 |                         let h = BufferHeader(self.buffer_region.offset(old_head as isize));
287 |                         h.clear_flag();
288 |                         h.set_in_used();
289 |                         (*self.counter).fetch_add(1, Ordering::SeqCst);
290 |                         return Ok(BufferSlice::new(
291 |                             Some(h),
292 |                             std::slice::from_raw_parts_mut(
293 |                                 self.buffer_region
294 |                                     .offset((old_head + BUFFER_HEADER_SIZE) as isize),
295 |                                 *self.cap_per_buffer as usize,
296 |                             ),
297 |                             old_head + self.buffer_region_offset_in_shm,
298 |                             true,
299 |                         ));
300 |                     }
301 |                 } else {
302 |                     // don't alloc the last slice
303 |                     if (*self.size).load(Ordering::SeqCst) <= 1 {
304 |                         (*self.size).fetch_add(1, Ordering::SeqCst);
305 |                         return Err(Error::NoMoreBuffer);
306 |                     }
307 |                 }
308 |                 old_head = (*self.head).load(Ordering::SeqCst);
309 |             }
310 |         }
311 |         unsafe {
312 |             (*self.size).fetch_add(1, Ordering::SeqCst);
313 |         }
314 |         Err(Error::NoMoreBuffer)
315 |     }
316 | 
317 |     #[allow(unused)]
318 |     pub fn remain(&self) -> isize {
319 |         // when the size is 1, not allow pop for solving problem about concurrent operating.
320 |         unsafe { (*self.size).load(Ordering::SeqCst) as isize - 1 }
321 |     }
322 | }
323 | 
324 | pub fn count_buffer_list_mem_size(buffer_num: u32, cap_per_buffer: u32) -> u32 {
325 |     BUFFER_LIST_HEADER_SIZE + buffer_num * (cap_per_buffer + BUFFER_HEADER_SIZE)
326 | }
327 | 
328 | #[cfg(test)]
329 | mod tests {
330 |     use memmap2::MmapOptions;
331 | 
332 |     use crate::buffer::list::{BufferList, count_buffer_list_mem_size};
333 | 
334 |     #[test]
335 |     fn test_buffer_list_put_pop() {
336 |         let cap_per_buffer = 4096;
337 |         let buffer_num = 1000;
338 |         let mem = MmapOptions::new()
339 |             .len(count_buffer_list_mem_size(buffer_num, cap_per_buffer) as usize)
340 |             .map_anon()
341 |             .unwrap();
342 | 
343 |         let l = BufferList::create(buffer_num, cap_per_buffer, &mem, 0).unwrap();
344 | 
345 |         let mut buffers = Vec::with_capacity(1024);
346 |         let origin_size = l.remain();
347 |         while l.remain() > 0 {
348 |             let b = l.pop().unwrap();
349 |             assert_eq!(cap_per_buffer, b.capacity() as u32);
350 |             assert_eq!(0, b.size());
351 |             assert!(!b.buffer_header.as_ref().unwrap().has_next());
352 |             buffers.push(b);
353 |         }
354 | 
355 |         for buffer in buffers {
356 |             l.push(buffer);
357 |         }
358 | 
359 |         buffers = Vec::with_capacity(1024);
360 |         assert_eq!(origin_size, l.remain());
361 |         while l.remain() > 0 {
362 |             let b = l.pop().unwrap();
363 |             assert_eq!(cap_per_buffer, b.capacity() as u32);
364 |             assert_eq!(0, b.size());
365 |             assert!(!b.buffer_header.as_ref().unwrap().has_next());
366 |             buffers.push(b);
367 |         }
368 |     }
369 | 
370 |     #[tokio::test(flavor = "multi_thread")]
371 |     async fn test_buffer_list_concurrent_put_pop() {
372 |         let cap_per_buffer = 16;
373 |         let buffer_num = 100;
374 |         let mem = MmapOptions::new()
375 |             .len(count_buffer_list_mem_size(buffer_num, cap_per_buffer) as usize)
376 |             .map_anon()
377 |             .unwrap();
378 |         let l = BufferList::create(buffer_num, cap_per_buffer, &mem, 0).unwrap();
379 | 
380 |         let concurrency = 10;
381 |         let mut join_handle = Vec::new();
382 |         for _ in 0..concurrency {
383 |             join_handle.push(tokio::spawn({
384 |                 async move {
385 |                     for _ in 0..10000 {
386 |                         let mut b = l.pop();
387 |                         while b.is_err() {
388 |                             tokio::time::sleep(std::time::Duration::from_millis(1)).await;
389 |                             b = l.pop();
390 |                         }
391 |                         let b = b.unwrap();
392 |                         assert_eq!(cap_per_buffer, b.capacity() as u32);
393 |                         assert_eq!(0, b.size());
394 |                         assert!(
395 |                             !b.buffer_header.as_ref().unwrap().has_next(),
396 |                             "offset:{} next:{}",
397 |                             b.offset_in_shm,
398 |                             b.buffer_header.as_ref().unwrap().next_buffer_offset(),
399 |                         );
400 |                         l.push(b);
401 |                     }
402 |                 }
403 |             }));
404 |         }
405 |         for handle in join_handle {
406 |             handle.await.unwrap();
407 |         }
408 |         assert_eq!(
409 |             buffer_num,
410 |             unsafe { (*l.size).load(std::sync::atomic::Ordering::SeqCst) } as u32
411 |         );
412 |     }
413 | 
414 |     #[test]
415 |     fn test_buffer_list_create_and_mapping_free_buffer_list() {
416 |         let cap_per_buffer = 16;
417 |         let buffer_num = 10;
418 |         let mem = MmapOptions::new()
419 |             .len(count_buffer_list_mem_size(buffer_num, cap_per_buffer) as usize)
420 |             .map_anon()
421 |             .unwrap();
422 |         let l = BufferList::create(0, cap_per_buffer, &mem, 0);
423 |         assert!(l.is_err());
424 | 
425 |         let l = BufferList::create(buffer_num + 1, cap_per_buffer, &mem, 0);
426 |         assert!(l.is_err());
427 | 
428 |         let _ = BufferList::create(buffer_num, cap_per_buffer, &mem, 0).unwrap();
429 | 
430 |         let test_mem = std::sync::Arc::new(MmapOptions::new().len(10).map_anon().unwrap());
431 |         let ml = BufferList::mapping(&test_mem, 0);
432 |         assert!(ml.is_err());
433 | 
434 |         let ml = BufferList::mapping(&mem, 8);
435 |         assert!(ml.is_err());
436 | 
437 |         let _ = BufferList::mapping(&mem, 0).unwrap();
438 |     }
439 | 
440 |     #[test]
441 |     fn test_create_free_buffer_list() {
442 |         assert!(
443 |             BufferList::create(
444 |                 4294967295,
445 |                 4294967295,
446 |                 &MmapOptions::new().len(1).map_anon().unwrap(),
447 |                 4294967279
448 |             )
449 |             .is_err()
450 |         )
451 |     }
452 | }
453 | 


--------------------------------------------------------------------------------
/src/buffer/mod.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | pub mod buf;
16 | pub mod linked;
17 | pub mod list;
18 | pub mod manager;
19 | pub mod slice;
20 | 
21 | use buf::Buf;
22 | 
23 | use crate::error::Error;
24 | 
25 | pub trait BufferReader {
26 |     /// Read `size` bytes from share memory
27 |     ///
28 |     /// if `release_previous_read()` is called , the results of previous `read_bytes()` will be
29 |     /// invalid.
30 |     fn read_bytes(&mut self, size: usize) -> Result<Buf<'_>, Error>;
31 | 
32 |     /// Peek `size` byte from share memory.
33 |     ///
34 |     /// The difference between `peek()` and `read_bytes()` is that
35 |     /// `peek()` don't influence the return value of length, but the `read_bytes()` will decrease
36 |     /// the unread size.
37 |     ///
38 |     /// Results of previous `peek()` call is valid until `release_previous_read()` is called.
39 |     fn peek(&mut self, size: usize) -> Result<Buf<'_>, Error>;
40 | 
41 |     /// Drop data of given size.
42 |     fn discard(&mut self, size: usize) -> Result<usize, Error>;
43 | 
44 |     /// Call `release_previous_read()` when it is safe to drop all previous result of `read_bytes()`
45 |     /// and `peek()`, otherwise shm memory will leak.
46 |     fn release_previous_read(&mut self);
47 | }
48 | 
49 | pub trait BufferWriter {
50 |     /// Reserve `size` bytes share memory space, use it to implement zero copy write.
51 |     fn reserve(&mut self, size: usize) -> Result<&mut [u8], Error>;
52 | 
53 |     /// Copy data to share memory, return the copy size if success.
54 |     fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error>;
55 | }
56 | 


--------------------------------------------------------------------------------
/src/buffer/slice.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use core::slice;
 16 | use std::{cmp::min, ptr::NonNull};
 17 | 
 18 | use crate::{
 19 |     buffer::manager::{
 20 |         BUFFER_CAP_OFFSET, BUFFER_DATA_START_OFFSET, BUFFER_FLAG_OFFSET, BUFFER_SIZE_OFFSET,
 21 |         HAS_NEXT_BUFFER_FLAG, NEXT_BUFFER_OFFSET, SLICE_IN_USED_FLAG,
 22 |     },
 23 |     error::Error,
 24 | };
 25 | 
 26 | #[derive(Default, Debug)]
 27 | pub struct SliceList {
 28 |     pub front_slice: Option<NonNull<BufferSlice>>,
 29 |     pub write_slice: Option<NonNull<BufferSlice>>,
 30 |     pub back_slice: Option<NonNull<BufferSlice>>,
 31 |     len: usize,
 32 | }
 33 | 
 34 | impl SliceList {
 35 |     pub fn new() -> Self {
 36 |         Self::default()
 37 |     }
 38 | 
 39 |     #[inline]
 40 |     pub fn front(&self) -> Option<&BufferSlice> {
 41 |         unsafe { self.front_slice.map(|node| &(*node.as_ptr())) }
 42 |     }
 43 | 
 44 |     #[inline]
 45 |     pub fn front_mut(&self) -> Option<&mut BufferSlice> {
 46 |         unsafe { self.front_slice.map(|node| &mut (*node.as_ptr())) }
 47 |     }
 48 | 
 49 |     #[inline]
 50 |     pub fn back(&self) -> Option<&BufferSlice> {
 51 |         unsafe { self.back_slice.map(|node| &(*node.as_ptr())) }
 52 |     }
 53 | 
 54 |     #[inline]
 55 |     pub fn back_mut(&self) -> Option<&mut BufferSlice> {
 56 |         unsafe { self.back_slice.map(|node| &mut (*node.as_ptr())) }
 57 |     }
 58 | 
 59 |     #[inline]
 60 |     pub fn write(&self) -> Option<&BufferSlice> {
 61 |         unsafe { self.write_slice.map(|node| &(*node.as_ptr())) }
 62 |     }
 63 | 
 64 |     #[inline]
 65 |     pub fn write_mut(&self) -> Option<&mut BufferSlice> {
 66 |         unsafe { self.write_slice.map(|node| &mut (*node.as_ptr())) }
 67 |     }
 68 | 
 69 |     #[inline]
 70 |     pub fn size(&self) -> usize {
 71 |         self.len
 72 |     }
 73 | 
 74 |     pub fn push_back(&mut self, s: BufferSlice) {
 75 |         unsafe {
 76 |             let new = NonNull::new_unchecked(Box::into_raw(Box::new(s)));
 77 |             if self.len > 0 {
 78 |                 self.back_slice.unwrap().as_mut().next_slice = Some(new);
 79 |             } else {
 80 |                 self.front_slice = Some(new);
 81 |             }
 82 |             self.back_slice = Some(new);
 83 |             self.len += 1;
 84 |         }
 85 |     }
 86 | 
 87 |     pub fn pop_front(&mut self) -> Option<BufferSlice> {
 88 |         unsafe {
 89 |             let r = self.front_slice;
 90 |             if self.len > 0 {
 91 |                 self.len -= 1;
 92 |                 self.front_slice = self.front_slice.unwrap().as_ref().next_slice;
 93 |             }
 94 |             if self.len == 0 {
 95 |                 self.front_slice = None;
 96 |                 self.back_slice = None;
 97 |             }
 98 |             r.map(|node| *Box::from_raw(node.as_ptr()))
 99 |         }
100 |     }
101 | 
102 |     pub fn split_from_write(&mut self) -> Option<BufferSlice> {
103 |         unsafe {
104 |             self.write_slice.and_then(|slice| {
105 |                 let next_list_head = (*slice.as_ptr()).next_slice;
106 |                 self.back_slice = Some(slice);
107 |                 (*slice.as_ptr()).next_slice = None;
108 |                 let mut next_list_size = 0;
109 |                 let mut s = next_list_head;
110 |                 while s.is_some() {
111 |                     next_list_size += 1;
112 |                     s = (*s.unwrap_unchecked().as_ptr()).next_slice;
113 |                 }
114 |                 self.len -= next_list_size;
115 |                 next_list_head.map(|head| *Box::from_raw(head.as_ptr()))
116 |             })
117 |         }
118 |     }
119 | }
120 | 
121 | #[derive(Debug, Eq, PartialEq, Clone)]
122 | pub struct BufferSlice {
123 |     /// BufferHeader layout: cap 4 byte | size 4 byte | start 4 byte | next 4 byte | flag 2 byte |
124 |     /// unused 2 byte
125 |     pub buffer_header: Option<BufferHeader>,
126 |     pub data: *mut u8,
127 |     pub cap: u32,
128 |     /// use for prepend
129 |     pub start: u32,
130 |     pub offset_in_shm: u32,
131 |     pub read_index: usize,
132 |     pub write_index: usize,
133 |     pub is_from_shm: bool,
134 |     pub next_slice: Option<NonNull<BufferSlice>>,
135 | }
136 | 
137 | unsafe impl Send for BufferSlice {}
138 | unsafe impl Sync for BufferSlice {}
139 | 
140 | impl BufferSlice {
141 |     pub fn new(
142 |         header: Option<BufferHeader>,
143 |         data: &mut [u8],
144 |         offset_in_shm: u32,
145 |         is_from_shm: bool,
146 |     ) -> Self {
147 |         debug_assert!(!data.is_empty());
148 | 
149 |         let len = data.len() as u32;
150 |         let mut s = Self {
151 |             buffer_header: None,
152 |             data: data.as_mut_ptr(),
153 |             cap: 0,
154 |             start: 0,
155 |             offset_in_shm,
156 |             read_index: 0,
157 |             write_index: 0,
158 |             is_from_shm,
159 |             next_slice: None,
160 |         };
161 |         if is_from_shm && header.is_some() {
162 |             let buffer_header = header.unwrap();
163 |             s.cap = buffer_header.cap();
164 |             s.start = buffer_header.start();
165 |             s.write_index = (s.start + buffer_header.size()) as usize;
166 |             s.buffer_header = Some(buffer_header);
167 |         } else {
168 |             s.cap = len;
169 |         }
170 |         s
171 |     }
172 | 
173 |     pub fn update(&self) {
174 |         if let Some(buffer_header) = &self.buffer_header {
175 |             buffer_header.set_size(self.size() as u32);
176 |             buffer_header.set_start(self.start);
177 | 
178 |             if let Some(next_slice) = &self.next_slice {
179 |                 unsafe {
180 |                     buffer_header.link_next((*next_slice.as_ptr()).offset_in_shm);
181 |                 }
182 |             }
183 |         }
184 |     }
185 | 
186 |     pub fn reset(&mut self) {
187 |         if let Some(buffer_header) = &self.buffer_header {
188 |             buffer_header.set_size(0);
189 |             buffer_header.set_start(0);
190 |             buffer_header.clear_flag()
191 |         }
192 |         self.write_index = 0;
193 |         self.read_index = 0;
194 |         self.next_slice = None;
195 |     }
196 | 
197 |     pub fn size(&self) -> usize {
198 |         self.write_index - self.read_index
199 |     }
200 | 
201 |     pub fn remain(&self) -> usize {
202 |         self.cap as usize - self.write_index
203 |     }
204 | 
205 |     pub fn capacity(&self) -> usize {
206 |         self.cap as usize
207 |     }
208 | 
209 |     pub fn reserve(&mut self, size: usize) -> Result<&mut [u8], Error> {
210 |         let start = self.write_index;
211 |         let remain = self.remain();
212 |         if remain >= size {
213 |             self.write_index += size;
214 |             return Ok(unsafe { slice::from_raw_parts_mut(self.data.add(start), size) });
215 |         }
216 |         Err(Error::NoMoreBuffer)
217 |     }
218 | 
219 |     pub fn append(&mut self, data: &[u8]) -> usize {
220 |         if data.is_empty() {
221 |             return 0;
222 |         }
223 |         let copy_size = min(data.len(), self.remain());
224 |         unsafe {
225 |             self.data
226 |                 .add(self.write_index)
227 |                 .copy_from_nonoverlapping(data.as_ptr(), copy_size)
228 |         };
229 |         self.write_index += copy_size;
230 |         copy_size
231 |     }
232 | 
233 |     #[must_use]
234 |     pub fn read(&mut self, mut size: usize) -> &[u8] {
235 |         size = min(size, self.size());
236 |         let data = unsafe { slice::from_raw_parts(self.data.add(self.read_index), size) };
237 |         self.read_index += size;
238 |         data
239 |     }
240 | 
241 |     pub fn peek(&mut self, mut size: usize) -> &[u8] {
242 |         size = min(size, self.size());
243 |         unsafe { slice::from_raw_parts(self.data.add(self.read_index), size) }
244 |     }
245 | 
246 |     pub fn skip(&mut self, size: usize) -> usize {
247 |         let un_read = self.size();
248 |         if un_read > size {
249 |             self.read_index += size;
250 |             return size;
251 |         }
252 |         self.read_index += un_read;
253 |         un_read
254 |     }
255 | 
256 |     #[inline]
257 |     pub fn next(&self) -> Option<&BufferSlice> {
258 |         unsafe { self.next_slice.map(|node| &(*node.as_ptr())) }
259 |     }
260 | 
261 |     #[inline]
262 |     pub fn next_mut(&self) -> Option<&mut BufferSlice> {
263 |         unsafe { self.next_slice.map(|node| &mut (*node.as_ptr())) }
264 |     }
265 | }
266 | 
267 | /// BufferHeader is the header of a buffer slice.
268 | ///
269 | /// Layout: cap 4 byte | size 4 byte | start 4 byte | next 4 byte | flag 2 byte | unused 2 byte
270 | ///
271 | /// # Safety
272 | ///
273 | /// Make sure it is well initialized before use and see ptr.offset method safety requirements.
274 | #[derive(Eq, PartialEq, Debug, Clone)]
275 | pub struct BufferHeader(pub *mut u8);
276 | 
277 | impl BufferHeader {
278 |     #[inline]
279 |     pub fn next_buffer_offset(&self) -> u32 {
280 |         unsafe { *(self.0.offset(NEXT_BUFFER_OFFSET as isize) as *const u32) }
281 |     }
282 | 
283 |     #[inline]
284 |     pub fn has_next(&self) -> bool {
285 |         unsafe { *self.0.offset(BUFFER_FLAG_OFFSET as isize) & HAS_NEXT_BUFFER_FLAG > 0 }
286 |     }
287 | 
288 |     #[inline]
289 |     pub fn clear_flag(&self) {
290 |         unsafe {
291 |             *self.0.offset(BUFFER_FLAG_OFFSET as isize) = 0u8;
292 |         }
293 |     }
294 | 
295 |     #[inline]
296 |     pub fn set_in_used(&self) {
297 |         unsafe {
298 |             *self.0.offset(BUFFER_FLAG_OFFSET as isize) |= SLICE_IN_USED_FLAG;
299 |         }
300 |     }
301 | 
302 |     #[inline]
303 |     pub fn is_in_used(&self) -> bool {
304 |         unsafe { *self.0.offset(BUFFER_FLAG_OFFSET as isize) & SLICE_IN_USED_FLAG > 0 }
305 |     }
306 | 
307 |     #[inline]
308 |     pub fn link_next(&self, next: u32) {
309 |         unsafe {
310 |             *(self.0.offset(NEXT_BUFFER_OFFSET as isize) as *mut u32) = next;
311 |             *self.0.offset(BUFFER_FLAG_OFFSET as isize) |= HAS_NEXT_BUFFER_FLAG;
312 |         }
313 |     }
314 | 
315 |     #[inline]
316 |     pub fn cap(&self) -> u32 {
317 |         unsafe { *(self.0.offset(BUFFER_CAP_OFFSET as isize) as *const u32) }
318 |     }
319 | 
320 |     #[inline]
321 |     pub fn size(&self) -> u32 {
322 |         unsafe { *(self.0.offset(BUFFER_SIZE_OFFSET as isize) as *const u32) }
323 |     }
324 | 
325 |     #[inline]
326 |     pub fn set_size(&self, size: u32) {
327 |         unsafe {
328 |             *(self.0.offset(BUFFER_SIZE_OFFSET as isize) as *mut u32) = size;
329 |         }
330 |     }
331 | 
332 |     #[inline]
333 |     pub fn start(&self) -> u32 {
334 |         unsafe { *(self.0.offset(BUFFER_DATA_START_OFFSET as isize) as *const u32) }
335 |     }
336 | 
337 |     #[inline]
338 |     pub fn set_start(&self, start: u32) {
339 |         unsafe {
340 |             *(self.0.offset(BUFFER_DATA_START_OFFSET as isize) as *mut u32) = start;
341 |         }
342 |     }
343 | }
344 | 
345 | #[cfg(test)]
346 | mod tests {
347 |     use core::slice;
348 | 
349 |     use memmap2::MmapOptions;
350 |     use rand::Rng;
351 | 
352 |     use super::{BufferSlice, SliceList};
353 |     use crate::{
354 |         buffer::{
355 |             manager::{BUFFER_CAP_OFFSET, BUFFER_HEADER_SIZE, BufferManager},
356 |             slice::BufferHeader,
357 |         },
358 |         config::SizePercentPair,
359 |     };
360 | 
361 |     #[test]
362 |     fn test_buffer_slice_read_write() {
363 |         const SIZE: usize = 8192;
364 | 
365 |         let mut buf = [0u8; SIZE];
366 |         let mut slice = BufferSlice::new(None, &mut buf, 0, false);
367 |         for i in 0..SIZE {
368 |             let n = slice.append(&[i as u8]);
369 |             assert_eq!(n, 1);
370 |         }
371 |         let n = slice.append(&[10]);
372 |         assert_eq!(n, 0);
373 | 
374 |         let data = slice.read(SIZE * 10);
375 |         assert_eq!(data.len(), SIZE);
376 | 
377 |         // vertfy read data.
378 |         (0..SIZE).for_each(|i| {
379 |             assert_eq!(data[i], i as u8);
380 |         });
381 |     }
382 | 
383 |     #[test]
384 |     fn test_buffer_slice_skip() {
385 |         const SIZE: usize = 8192;
386 | 
387 |         let mut buf = [0u8; SIZE];
388 |         let mut slice = BufferSlice::new(None, &mut buf, 0, false);
389 |         slice.append(&[0u8; SIZE]);
390 |         let mut remain = slice.capacity();
391 | 
392 |         let n = slice.skip(10);
393 |         remain -= n;
394 |         assert_eq!(remain, slice.size());
395 | 
396 |         let n = slice.skip(100);
397 |         remain -= n;
398 |         assert_eq!(remain, slice.size());
399 | 
400 |         _ = slice.skip(10000);
401 |         assert_eq!(0, slice.size());
402 |     }
403 | 
404 |     #[test]
405 |     fn test_buffer_slice_reserve() {
406 |         const SIZE: usize = 8192;
407 | 
408 |         let mut buf = [0u8; SIZE];
409 |         let mut slice = BufferSlice::new(None, &mut buf, 0, false);
410 |         let data1 = slice.reserve(100).unwrap();
411 |         assert_eq!(100, data1.len());
412 | 
413 |         (0..data1.len()).for_each(|i| data1[i] = i as u8);
414 |         let data1 = unsafe { slice::from_raw_parts(data1.as_ptr(), data1.len()) };
415 | 
416 |         let data2 = slice.reserve(SIZE);
417 |         assert!(data2.is_err());
418 | 
419 |         let read_data = slice.read(100);
420 |         assert_eq!(100, read_data.len());
421 | 
422 |         (0..100).for_each(|i| {
423 |             assert_eq!(read_data[i], data1[i]);
424 |         });
425 | 
426 |         let read_data = slice.read(10000);
427 |         assert_eq!(read_data.len(), 0);
428 |     }
429 | 
430 |     #[test]
431 |     fn test_buffer_slice_update() {
432 |         const SIZE: usize = 8192;
433 | 
434 |         let mut buf = [0u8; SIZE];
435 |         let mut header = [0u8; BUFFER_HEADER_SIZE as usize];
436 |         unsafe {
437 |             *(header.as_mut_ptr().offset(BUFFER_CAP_OFFSET as isize) as *mut u32) = 8192u32;
438 |         }
439 |         let mut slice =
440 |             BufferSlice::new(Some(BufferHeader(header.as_mut_ptr())), &mut buf, 0, true);
441 | 
442 |         let n = slice.append(&[0u8; SIZE]);
443 |         assert_eq!(n, SIZE);
444 |         slice.update();
445 |         assert_eq!(SIZE, slice.buffer_header.as_ref().unwrap().size() as usize);
446 |     }
447 | 
448 |     #[test]
449 |     fn test_buffer_slice_linked_next() {
450 |         const SIZE: usize = 8192;
451 |         const SLICE_NUM: usize = 100;
452 | 
453 |         let mut slices = Vec::with_capacity(SLICE_NUM);
454 |         let mem = MmapOptions::new().len(10 << 20).map_anon().unwrap();
455 |         let bm = BufferManager::create(
456 |             &[SizePercentPair {
457 |                 size: 8192,
458 |                 percent: 100,
459 |             }],
460 |             "",
461 |             mem,
462 |             0,
463 |         )
464 |         .unwrap();
465 | 
466 |         let mut write_data_array = Vec::with_capacity(100);
467 | 
468 |         for _ in 0..SLICE_NUM {
469 |             let mut s = bm.alloc_shm_buffer(SIZE as u32).unwrap();
470 |             let mut rng = rand::rng();
471 |             let data: Vec<u8> = (0..SIZE).map(|_| rng.random()).collect();
472 |             assert_eq!(s.append(&data), SIZE);
473 |             s.update();
474 |             slices.push(s);
475 |             write_data_array.push(data);
476 |         }
477 | 
478 |         for i in 0..slices.len() - 1 {
479 |             slices[i]
480 |                 .buffer_header
481 |                 .as_ref()
482 |                 .unwrap()
483 |                 .link_next(slices[i + 1].offset_in_shm);
484 |         }
485 | 
486 |         let mut next = slices[0].offset_in_shm;
487 |         (0..SLICE_NUM).for_each(|i| {
488 |             let mut s = bm.read_buffer_slice(next).unwrap();
489 |             assert_eq!(s.capacity(), SIZE);
490 |             assert_eq!(s.size(), SIZE);
491 |             let read_data = s.read(SIZE);
492 |             assert_eq!(read_data.len(), SIZE);
493 |             (0..SIZE).for_each(|j| {
494 |                 assert_eq!(read_data[j], write_data_array[i][j]);
495 |             });
496 |             let is_last_slice = i == SLICE_NUM - 1;
497 |             assert_eq!(s.buffer_header.as_ref().unwrap().has_next(), !is_last_slice);
498 |             next = s.buffer_header.as_ref().unwrap().next_buffer_offset();
499 |         });
500 |     }
501 | 
502 |     #[test]
503 |     fn test_slice_list_push_pop() {
504 |         // 1. twice push, twice pop
505 |         let mut l = SliceList::new();
506 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
507 |         assert_eq!(l.front(), l.back());
508 |         assert_eq!(1, l.size());
509 | 
510 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
511 |         assert_eq!(2, l.size());
512 |         assert_ne!(l.front(), l.back());
513 | 
514 |         l.pop_front();
515 |         assert_eq!(1, l.size());
516 |         assert_eq!(l.front(), l.back());
517 | 
518 |         l.pop_front();
519 |         assert_eq!(0, l.size());
520 |         assert!(l.front().is_none());
521 |         assert!(l.back().is_none());
522 | 
523 |         // multi push and pop
524 |         for i in 0..100 {
525 |             l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
526 |             assert_eq!(i + 1, l.size());
527 |         }
528 |         for i in 0..100 {
529 |             l.pop_front();
530 |             assert_eq!(100 - (i + 1), l.size());
531 |         }
532 |         assert_eq!(0, l.size());
533 |         assert!(l.front().is_none());
534 |         assert!(l.back().is_none());
535 |     }
536 | 
537 |     #[test]
538 |     fn test_slice_list_split_from_write() {
539 |         // 1. sliceList's size == 1
540 |         let mut l = SliceList::new();
541 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
542 |         l.write_slice = l.front_slice;
543 |         assert!(l.split_from_write().is_none());
544 |         assert_eq!(1, l.size());
545 |         assert_eq!(l.front(), l.back());
546 |         assert_eq!(l.back(), l.write());
547 | 
548 |         // 2. sliceList's size == 2, writeSlice's index is 0
549 |         let mut l = SliceList::new();
550 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
551 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
552 |         l.write_slice = l.front_slice;
553 |         l.split_from_write();
554 | 
555 |         assert_eq!(1, l.size());
556 |         assert_eq!(l.front(), l.back());
557 |         assert_eq!(l.back(), l.write());
558 | 
559 |         // 2. sliceList's size == 2, writeSlice's index is 1
560 |         let mut l = SliceList::new();
561 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
562 |         l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
563 |         l.write_slice = l.back_slice;
564 |         assert!(l.split_from_write().is_none());
565 |         assert_eq!(2, l.size());
566 |         assert_eq!(l.back(), l.write());
567 | 
568 |         // 3. sliceList's size == 3, writeSlice's index is 50
569 |         let mut l = SliceList::new();
570 |         for i in 0..100 {
571 |             l.push_back(BufferSlice::new(None, &mut [0; 1024], 0, false));
572 |             if i == 50 {
573 |                 l.write_slice = l.back_slice;
574 |             }
575 |         }
576 |         l.split_from_write();
577 |         assert_eq!(l.back(), l.write());
578 | 
579 |         assert_eq!(51, l.size());
580 |     }
581 | }
582 | 


--------------------------------------------------------------------------------
/src/config.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{fmt::Debug, time::Duration};
 16 | 
 17 | use anyhow::anyhow;
 18 | 
 19 | use crate::{
 20 |     buffer::manager::BUFFER_HEADER_SIZE,
 21 |     consts::{DEFAULT_QUEUE_CAP, DEFAULT_SHARE_MEMORY_CAP, MemMapType, SESSION_REBUILD_INTERVAL},
 22 | };
 23 | 
 24 | #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 25 | pub struct SizePercentPair {
 26 |     pub size: u32,
 27 |     pub percent: u32,
 28 | }
 29 | 
 30 | #[derive(Clone, Debug)]
 31 | /// Config is used to tune the shmipc session
 32 | pub struct Config {
 33 |     /// connection_write_timeout is meant to be a "safety value" timeout after
 34 |     /// which we will suspect a problem with the underlying connection and
 35 |     /// close it. This is only applied to writes, where there's generally
 36 |     /// an expectation that things will move along quickly.
 37 |     pub connection_write_timeout: Duration,
 38 | 
 39 |     pub connection_read_timeout: Option<Duration>,
 40 | 
 41 |     pub connection_timeout: Option<Duration>,
 42 | 
 43 |     /// initialize_timeout is meant timeout during server and client exchange config phase
 44 |     pub initialize_timeout: Duration,
 45 | 
 46 |     /// the max number of pending request
 47 |     pub queue_cap: u32,
 48 | 
 49 |     /// share memory path of the underlying queue
 50 |     pub queue_path: String,
 51 | 
 52 |     /// the capacity of buffer in share memory
 53 |     pub share_memory_buffer_cap: u32,
 54 | 
 55 |     /// the share memory path prefix of buffer
 56 |     pub share_memory_path_prefix: String,
 57 | 
 58 |     /// guess request or response's size for improving performance, and the default value is 4096
 59 |     pub buffer_slice_sizes: Vec<SizePercentPair>,
 60 | 
 61 |     /// mmap map type, MemMapTypeDevShmFile or MemMapTypeMemFd (server set)
 62 |     pub mem_map_type: MemMapType,
 63 | 
 64 |     /// client rebuild session interval
 65 |     pub rebuild_interval: Duration,
 66 | 
 67 |     pub max_stream_num: usize,
 68 | }
 69 | 
 70 | impl Default for Config {
 71 |     fn default() -> Self {
 72 |         Self::new()
 73 |     }
 74 | }
 75 | 
 76 | impl Config {
 77 |     pub fn new() -> Self {
 78 |         Self {
 79 |             connection_write_timeout: Duration::from_secs(10),
 80 |             connection_read_timeout: None,
 81 |             connection_timeout: None,
 82 |             initialize_timeout: Duration::from_millis(1000),
 83 |             queue_cap: DEFAULT_QUEUE_CAP,
 84 |             queue_path: "/dev/shm/shmipc_queue".to_owned(),
 85 |             share_memory_buffer_cap: DEFAULT_SHARE_MEMORY_CAP,
 86 |             share_memory_path_prefix: "/dev/shm/shmipc".to_owned(),
 87 |             buffer_slice_sizes: vec![
 88 |                 SizePercentPair {
 89 |                     size: 8192 - BUFFER_HEADER_SIZE,
 90 |                     percent: 50,
 91 |                 },
 92 |                 SizePercentPair {
 93 |                     size: 32 * 1024 - BUFFER_HEADER_SIZE,
 94 |                     percent: 30,
 95 |                 },
 96 |                 SizePercentPair {
 97 |                     size: 128 * 1024 - BUFFER_HEADER_SIZE,
 98 |                     percent: 20,
 99 |                 },
100 |             ],
101 |             mem_map_type: MemMapType::MemMapTypeMemFd,
102 |             rebuild_interval: SESSION_REBUILD_INTERVAL,
103 |             max_stream_num: 4096,
104 |         }
105 |     }
106 | 
107 |     pub fn verify(&self) -> Result<(), anyhow::Error> {
108 |         if self.share_memory_buffer_cap < (1 << 20) {
109 |             return Err(anyhow!(
110 |                 "share memory size is too small:{}, must greater than {}",
111 |                 self.share_memory_buffer_cap,
112 |                 1 << 20
113 |             ));
114 |         }
115 |         if self.buffer_slice_sizes.is_empty() {
116 |             return Err(anyhow!("buffer_slice_sizes could not be nil"));
117 |         }
118 | 
119 |         let mut sum = 0;
120 |         for pair in self.buffer_slice_sizes.iter() {
121 |             sum += pair.percent;
122 |             if pair.size > self.share_memory_buffer_cap {
123 |                 return Err(anyhow!(
124 |                     "buffer_slice_sizes's size:{} couldn't greater than share_memory_buffer_cap:{}",
125 |                     pair.size,
126 |                     self.share_memory_buffer_cap
127 |                 ));
128 |             }
129 | 
130 |             #[cfg(any(target_arch = "arm", target_arch = "arm64ec"))]
131 |             if pair.size % 4 != 0 {
132 |                 return Err(anyhow!(
133 |                     "the size_percent_pair.size must be a multiple of 4"
134 |                 ));
135 |             }
136 |         }
137 | 
138 |         if sum != 100 {
139 |             return Err(anyhow!(
140 |                 "the sum of buffer_slice_sizes's percent should be 100",
141 |             ));
142 |         }
143 | 
144 |         #[cfg(any(target_arch = "arm", target_arch = "arm64ec"))]
145 |         if self.queue_cap % 8 != 0 {
146 |             return Err(anyhow!("the queue_cap must be a multiple of 8"));
147 |         }
148 | 
149 |         if self.share_memory_path_prefix.is_empty() || self.queue_path.is_empty() {
150 |             return Err(anyhow!("buffer path or queue path could not be nil"));
151 |         }
152 | 
153 |         #[cfg(not(target_os = "linux"))]
154 |         {
155 |             return Err(anyhow!("shmipc just support linux OS now"));
156 |         }
157 | 
158 |         #[cfg(not(any(target_arch = "x86_64", target_arch = "arm64ec")))]
159 |         {
160 |             return Err(anyhow!("shmipc just support amd64 or arm64 arch"));
161 |         }
162 | 
163 |         Ok(())
164 |     }
165 | }
166 | 


--------------------------------------------------------------------------------
/src/consts.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::time::Duration;
16 | 
17 | pub const PROTO_VERSION: u8 = 2;
18 | pub const MAX_SUPPORT_PROTO_VERSION: u8 = 3;
19 | pub const MAGIC_NUMBER: u16 = 0x7758;
20 | 
21 | #[repr(u8)]
22 | #[derive(Default, Clone, Copy, Debug)]
23 | pub enum MemMapType {
24 |     #[default]
25 |     MemMapTypeDevShmFile = 0,
26 |     MemMapTypeMemFd,
27 | }
28 | 
29 | #[derive(Default, Clone, Copy, Debug)]
30 | #[repr(u32)]
31 | pub enum StateType {
32 |     #[default]
33 |     DefaultState = 0,
34 |     HotRestartState,
35 |     HotRestartDoneState,
36 | }
37 | 
38 | pub const MEMFD_CREATE_NAME: &str = "shmipc";
39 | pub const BUFFER_PATH_SUFFIX: &str = "_buffer";
40 | 
41 | pub const MEMFD_DATA_LEN: usize = 4;
42 | pub const MEMFD_COUNT: usize = 2;
43 | 
44 | pub const BUFER_PATH_SUFFIX: &str = "_buffer";
45 | pub const UNIX_NETWORK: &str = "unix";
46 | 
47 | pub const HOT_RESTART_CHECK_TIMEOUT: Duration = Duration::from_secs(2);
48 | pub const HOT_RESTART_CHECK_INTERVAL: Duration = Duration::from_millis(100);
49 | 
50 | pub const SESSION_REBUILD_INTERVAL: Duration = Duration::from_secs(60);
51 | 
52 | pub const EPOCH_ID_LEN: usize = 8;
53 | 
54 | /// linux file name max length
55 | pub const FILE_NAME_MAX_LEN: usize = 255;
56 | /// The buffer path will concatenate epoch information and end with [_epoch_{epochId uint64}_{randId
57 | /// uint64}], so the maximum length of epoch information is 1+5+1+20+1+20
58 | pub const EPOCH_INFO_MAX_LEN: usize = 7 + 20 + 1 + 20;
59 | /// _queue_{sessionId int}
60 | pub const QUEUE_INFO_MAX_LEN: usize = 7 + 20;
61 | 
62 | pub const DEFAULT_QUEUE_CAP: u32 = 8192;
63 | pub const DEFAULT_SHARE_MEMORY_CAP: u32 = 32 * 1024 * 1024;
64 | pub const DEFAULT_SINGLE_BUFFER_SIZE: i64 = 4096;
65 | pub const QUEUE_ELEMENT_LEN: usize = 12;
66 | pub const QUEUE_COUNT: usize = 2;
67 | 
68 | pub const SIZE_OF_LENGTH: usize = 4;
69 | pub const SIZE_OF_MAGIC: usize = 2;
70 | pub const SIZE_OF_VERSION: usize = 1;
71 | pub const SIZE_OF_TYPE: usize = 1;
72 | 
73 | pub const HEADER_SIZE: usize = SIZE_OF_LENGTH + SIZE_OF_MAGIC + SIZE_OF_VERSION + SIZE_OF_TYPE;
74 | 


--------------------------------------------------------------------------------
/src/error.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | #[derive(thiserror::Error, Debug)]
 16 | pub enum Error {
 17 |     /// InvalidVersion means that we received a frame with an invalid version.
 18 |     #[error("invalid protocol version")]
 19 |     InvalidVersion,
 20 | 
 21 |     /// InvalidMsgType means that we received a frame with an invalid message type.
 22 |     #[error("invalid msg type")]
 23 |     InvalidMsgType,
 24 | 
 25 |     /// SessionShutdown is used if there is a shutdown during an operation.
 26 |     #[error("session shutdown")]
 27 |     SessionShutdown,
 28 | 
 29 |     /// StreamsExhausted is returned if we have no more stream ids to issue.
 30 |     #[error("streams exhausted")]
 31 |     StreamsExhausted,
 32 | 
 33 |     /// DuplicateStream is used if a duplicate stream is opened inbound.
 34 |     #[error("duplicate stream initiated")]
 35 |     DuplicateStream,
 36 | 
 37 |     /// Timeout is used when we reach an IO deadline.
 38 |     #[error("i/o deadline reached")]
 39 |     Timeout,
 40 | 
 41 |     /// StreamClosed is returned when using a closed stream.
 42 |     #[error("stream closed")]
 43 |     StreamClosed,
 44 | 
 45 |     /// StreamResetByPeer is returned when the peer reset the stream.
 46 |     #[error("stream reset by peer")]
 47 |     StreamReset,
 48 | 
 49 |     /// ConnectionWriteTimeout indicates that we hit the "safety valve" timeout writing to the
 50 |     /// underlying stream connection.
 51 |     #[error("connection write timeout")]
 52 |     ConnectionWriteTimeout,
 53 | 
 54 |     #[error("connection timeout")]
 55 |     ConnectionTimeout,
 56 | 
 57 |     /// KeepAliveTimeout is sent if a missed keepalive caused the stream close.
 58 |     #[error("keepalive timeout")]
 59 |     KeepAliveTimeout,
 60 | 
 61 |     /// EndOfStream means that the stream is end, user shouldn't read from the stream.
 62 |     #[error("end of stream")]
 63 |     EndOfStream,
 64 | 
 65 |     /// SessionUnhealthy occurred at `session.open_stream()`, which means that the session is
 66 |     /// overload.
 67 |     ///
 68 |     /// User should retry after 60 seconds, and the following situation will result in
 69 |     /// SessionUnhealthy.
 70 |     ///
 71 |     /// 1. When local share memory is not enough, client send request data via unix domain socket.
 72 |     ///
 73 |     /// 2. When peer share memory is not enough, client receive response data from unix domain
 74 |     ///    socket.
 75 |     #[error("now the session is unhealthy, please retry later")]
 76 |     SessionUnhealthy,
 77 | 
 78 |     /// NotEnoughData means that the real read size < expect read size.
 79 |     ///
 80 |     /// In general, which happened on application protocol is buffered.
 81 |     #[error("current buffer is not enough data to read")]
 82 |     NotEnoughData,
 83 | 
 84 |     /// NoMoreBuffer means that the share memory is busy, and no more buffer to allocate.
 85 |     #[error("share memory no more buffer")]
 86 |     NoMoreBuffer,
 87 | 
 88 |     /// SizeTooLarge means that the allocated size exceeded.
 89 |     #[error("alloc size exceed")]
 90 |     SizeTooLarge,
 91 | 
 92 |     /// BrokenBuffer means that the share memory's layout had broken, which happens in that the
 93 |     /// share memory was alter by external or internal bug.
 94 |     #[error("share memory's buffer had broken")]
 95 |     BrokenBuffer,
 96 | 
 97 |     #[error("share memory had not left space")]
 98 |     ShareMemoryHadNotLeftSpace,
 99 | 
100 |     #[error("stream callbacks had existed")]
101 |     StreamCallbackHadExisted,
102 | 
103 |     /// ExchangeConfig means message type error during exchange config phase.
104 |     #[error("exchange config protocol invalid")]
105 |     ExchangeConfig,
106 | 
107 |     /// ExchangeConfigTimeout means client exchange config timeout.
108 |     #[error("exchange config timeout")]
109 |     ExchangeConfigTimeout,
110 | 
111 |     #[error("shmipc just support linux OS now")]
112 |     OSNonSupported,
113 | 
114 |     #[error("shmipc just support amd64 or arm64 arch")]
115 |     ArchNonSupported,
116 | 
117 |     /// Ensure once hot restart succeed
118 |     #[error("hot restart in progress, try again later")]
119 |     HotRestartInProgress,
120 | 
121 |     #[error("session in handshake stage, try again later")]
122 |     InHandshakeStage,
123 | 
124 |     /// File name max len 255
125 |     #[error("share memory path prefix too long")]
126 |     FileNameTooLong,
127 | 
128 |     #[error("the queue is empty")]
129 |     QueueEmpty,
130 | 
131 |     #[error("the queue is full")]
132 |     QueueFull,
133 | 
134 |     #[error("stream pool is full")]
135 |     StreamPoolFull,
136 | 
137 |     #[error(transparent)]
138 |     Io(#[from] std::io::Error),
139 | 
140 |     #[error(transparent)]
141 |     Others(#[from] anyhow::Error),
142 | }
143 | 


--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
 1 | #![cfg_attr(not(doctest), doc = include_str!("../README.md"))]
 2 | 
 3 | // Copyright 2025 CloudWeGo Authors
 4 | //
 5 | // Licensed under the Apache License, Version 2.0 (the "License");
 6 | // you may not use this file except in compliance with the License.
 7 | // You may obtain a copy of the License at
 8 | //
 9 | //     http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 | 
17 | mod buffer;
18 | pub use buffer::{BufferReader, BufferWriter};
19 | pub mod config;
20 | pub mod consts;
21 | mod error;
22 | pub use error::Error;
23 | mod protocol;
24 | mod queue;
25 | mod session;
26 | pub use session::{config::SessionManagerConfig, manager::SessionManager};
27 | mod stream;
28 | pub use stream::{AsyncReadShm, AsyncWriteShm, Stream};
29 | mod listener;
30 | pub use listener::Listener;
31 | pub mod stats;
32 | mod util;
33 | pub use buffer::{linked::LinkedBuffer, slice::BufferSlice};
34 | 


--------------------------------------------------------------------------------
/src/listener.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{
 16 |     io,
 17 |     sync::{Arc, Mutex},
 18 | };
 19 | 
 20 | use tokio::sync::mpsc;
 21 | use volo::net::{Address, DefaultIncoming, MakeIncoming, incoming::Incoming};
 22 | 
 23 | use crate::{config::Config, session::Session, stream::Stream};
 24 | 
 25 | #[derive(Debug)]
 26 | pub struct Listener {
 27 |     stream_rx: mpsc::UnboundedReceiver<io::Result<Option<Stream>>>,
 28 |     sessions: Arc<Mutex<Vec<Session>>>,
 29 | }
 30 | 
 31 | impl Listener {
 32 |     pub async fn new(addr: impl Into<Address>, config: Config) -> anyhow::Result<Self> {
 33 |         let addr = addr.into();
 34 |         if let Address::Unix(socket) = &addr {
 35 |             if let Some(path) = socket.as_pathname() {
 36 |                 if path.exists() {
 37 |                     std::fs::remove_file(path)?;
 38 |                 }
 39 |             }
 40 |         }
 41 |         let incoming = addr.make_incoming().await?;
 42 |         let (tx, rx) = mpsc::unbounded_channel();
 43 |         let sessions = Arc::new(Mutex::new(Vec::new()));
 44 |         tokio::spawn(Self::accept_loop(incoming, config, tx, sessions.clone()));
 45 |         Ok(Self {
 46 |             stream_rx: rx,
 47 |             sessions,
 48 |         })
 49 |     }
 50 | 
 51 |     pub async fn accept(&mut self) -> io::Result<Option<Stream>> {
 52 |         self.stream_rx.recv().await.unwrap_or(Ok(None))
 53 |     }
 54 | 
 55 |     async fn accept_loop(
 56 |         mut incoming: DefaultIncoming,
 57 |         config: Config,
 58 |         stream_tx: mpsc::UnboundedSender<io::Result<Option<Stream>>>,
 59 |         sessions: Arc<Mutex<Vec<Session>>>,
 60 |     ) {
 61 |         loop {
 62 |             tokio::select! {
 63 |                 _ = stream_tx.closed() => {
 64 |                     return;
 65 |                 }
 66 |                 res = incoming.accept() => {
 67 |                     match res {
 68 |                         Ok(Some(conn)) => {
 69 |                             let (tx, rx) = mpsc::channel::<Stream>(config.max_stream_num);
 70 | 
 71 |                             match Session::server(config.clone(), conn.stream, tx).await {
 72 |                                 Ok(session) => {
 73 |                                     sessions.lock().unwrap().push(session.clone());
 74 |                                     tokio::spawn(session.recv_loop(rx, stream_tx.clone()));
 75 |                                 }
 76 |                                 Err(err) => {
 77 |                                     tracing::warn!("failed to create shmipc session: {}", err);
 78 |                                     continue;
 79 |                                 }
 80 |                             }
 81 |                         }
 82 |                         Ok(None) => {
 83 |                             _ = stream_tx.send(Ok(None));
 84 |                             return;
 85 |                         }
 86 |                         Err(err) => {
 87 |                             _ = stream_tx.send(Err(err));
 88 |                             return;
 89 |                         }
 90 |                     }
 91 |                 }
 92 |             }
 93 |         }
 94 |     }
 95 | 
 96 |     pub async fn close(self) {
 97 |         let sessions = self.sessions.lock().unwrap().clone();
 98 |         for session in sessions {
 99 |             session.close().await;
100 |         }
101 |     }
102 | }
103 | 


--------------------------------------------------------------------------------
/src/protocol/adapter.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::os::fd::RawFd;
 16 | 
 17 | use anyhow::anyhow;
 18 | 
 19 | use super::{
 20 |     block_io::block_write_full,
 21 |     event::EventType,
 22 |     header::Header,
 23 |     initializer::{
 24 |         ProtocolInitializer, block_read_event_header,
 25 |         v2::{Client as V2Client, ProtocolInitializerV2, Server as V2Server},
 26 |         v3::{Client as V3Client, ProtocolInitializerV3, Server as V3Server},
 27 |         wait_event_header,
 28 |     },
 29 |     protocol_trace,
 30 | };
 31 | use crate::consts::{HEADER_SIZE, MAX_SUPPORT_PROTO_VERSION, MemMapType};
 32 | 
 33 | pub struct ClientProtocolAdapter {
 34 |     conn_fd: RawFd,
 35 |     mem_map_type: MemMapType,
 36 |     buffer_path: String,
 37 |     queue_path: String,
 38 |     buffer_fd: RawFd,
 39 |     queue_fd: RawFd,
 40 | }
 41 | 
 42 | impl ClientProtocolAdapter {
 43 |     pub fn new(
 44 |         conn_fd: RawFd,
 45 |         mem_map_type: MemMapType,
 46 |         buffer_path: String,
 47 |         queue_path: String,
 48 |         buffer_fd: RawFd,
 49 |         queue_fd: RawFd,
 50 |     ) -> Self {
 51 |         Self {
 52 |             conn_fd,
 53 |             mem_map_type,
 54 |             buffer_path,
 55 |             queue_path,
 56 |             buffer_fd,
 57 |             queue_fd,
 58 |         }
 59 |     }
 60 | 
 61 |     pub fn get_initializer(self) -> Result<ProtocolInitializer, anyhow::Error> {
 62 |         // temporarily ensure version compatibility.
 63 |         if let MemMapType::MemMapTypeDevShmFile = self.mem_map_type {
 64 |             return Ok(ProtocolInitializer::V2(ProtocolInitializerV2::Client(
 65 |                 V2Client {
 66 |                     conn_fd: self.conn_fd,
 67 |                     buffer_path: self.buffer_path,
 68 |                     queue_path: self.queue_path,
 69 |                 },
 70 |             )));
 71 |         }
 72 |         // send version to peer
 73 |         let mut h = Header([0; HEADER_SIZE].as_mut_ptr());
 74 |         let client_version = MAX_SUPPORT_PROTO_VERSION;
 75 |         h.encode(
 76 |             HEADER_SIZE as u32,
 77 |             client_version,
 78 |             EventType::TYPE_EXCHANGE_PROTO_VERSION,
 79 |         );
 80 |         protocol_trace(&h, &[], true);
 81 |         block_write_full(self.conn_fd, unsafe {
 82 |             std::slice::from_raw_parts(h.0, HEADER_SIZE)
 83 |         })?;
 84 |         // recv peer's version
 85 |         let mut buf = [0u8; HEADER_SIZE];
 86 |         let recv_header = wait_event_header(
 87 |             self.conn_fd,
 88 |             EventType::TYPE_EXCHANGE_PROTO_VERSION,
 89 |             &mut buf,
 90 |         )
 91 |         .map_err(|err| {
 92 |             anyhow!(
 93 |                 "protrocol_initializer_v3 client_init failed, reason:{}",
 94 |                 err
 95 |             )
 96 |         })?;
 97 |         let server_version = recv_header.version();
 98 |         match std::cmp::min(client_version, server_version) {
 99 |             2 => Ok(ProtocolInitializer::V2(ProtocolInitializerV2::Client(
100 |                 V2Client {
101 |                     conn_fd: self.conn_fd,
102 |                     buffer_path: self.buffer_path,
103 |                     queue_path: self.queue_path,
104 |                 },
105 |             ))),
106 |             3 => Ok(ProtocolInitializer::V3(ProtocolInitializerV3::Client(
107 |                 V3Client {
108 |                     conn_fd: self.conn_fd,
109 |                     mem_map_type: self.mem_map_type,
110 |                     buffer_fd: self.buffer_fd,
111 |                     queue_fd: self.queue_fd,
112 |                     buffer_path: self.buffer_path,
113 |                     queue_path: self.queue_path,
114 |                 },
115 |             ))),
116 |             version => Err(anyhow!(
117 |                 "not support the protocol version:{}, max_support_version is {}",
118 |                 version,
119 |                 MAX_SUPPORT_PROTO_VERSION
120 |             )),
121 |         }
122 |     }
123 | }
124 | 
125 | pub struct ServerProtocolAdapter {
126 |     conn_fd: RawFd,
127 | }
128 | 
129 | impl ServerProtocolAdapter {
130 |     pub fn new(conn_fd: RawFd) -> Self {
131 |         Self { conn_fd }
132 |     }
133 | 
134 |     pub fn get_initializer(self) -> Result<ProtocolInitializer, anyhow::Error> {
135 |         // ensure version compatibility
136 |         let mut buf = vec![0u8; HEADER_SIZE];
137 |         let h = block_read_event_header(self.conn_fd, &mut buf)?;
138 |         std::mem::forget(buf);
139 |         match h.version() {
140 |             2 => Ok(ProtocolInitializer::V2(ProtocolInitializerV2::Server(
141 |                 V2Server {
142 |                     conn_fd: self.conn_fd,
143 |                     first_event: h,
144 |                 },
145 |             ))),
146 |             3 => Ok(ProtocolInitializer::V3(ProtocolInitializerV3::Server(
147 |                 V3Server {
148 |                     conn_fd: self.conn_fd,
149 |                     first_event: h,
150 |                 },
151 |             ))),
152 |             version => Err(anyhow!(
153 |                 "not support the protocol version:{}, max_support_version is {}",
154 |                 version,
155 |                 MAX_SUPPORT_PROTO_VERSION
156 |             )),
157 |         }
158 |     }
159 | }
160 | 


--------------------------------------------------------------------------------
/src/protocol/block_io.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{
 16 |     io::{IoSlice, IoSliceMut},
 17 |     os::fd::{BorrowedFd, RawFd},
 18 | };
 19 | 
 20 | use anyhow::anyhow;
 21 | use nix::{
 22 |     cmsg_space,
 23 |     libc::EINVAL,
 24 |     sys::socket::{
 25 |         ControlMessage, ControlMessageOwned, MsgFlags, getsockopt, recvmsg, sendmsg,
 26 |         sockopt::SockType,
 27 |     },
 28 |     unistd::{read, write},
 29 | };
 30 | 
 31 | use crate::consts::MEMFD_COUNT;
 32 | 
 33 | pub(crate) fn block_read_full(conn_fd: RawFd, data: &mut [u8]) -> Result<(), anyhow::Error> {
 34 |     let mut read_size = 0;
 35 |     while read_size < data.len() {
 36 |         let n = read(conn_fd, &mut data[read_size..]).map_err(|e| {
 37 |             anyhow!(
 38 |                 "read_full failed, had read_size:{read_size}, reason:{}",
 39 |                 e.desc()
 40 |             )
 41 |         })?;
 42 |         read_size += n;
 43 |         if n == 0 {
 44 |             return Err(anyhow!("EOF"));
 45 |         }
 46 |     }
 47 |     Ok(())
 48 | }
 49 | 
 50 | pub(crate) fn block_write_full(conn_fd: RawFd, data: &[u8]) -> Result<(), anyhow::Error> {
 51 |     let mut written = 0;
 52 |     while written < data.len() {
 53 |         let n = write(unsafe { BorrowedFd::borrow_raw(conn_fd) }, &data[written..])?;
 54 |         written += n;
 55 |     }
 56 |     Ok(())
 57 | }
 58 | 
 59 | pub(crate) fn send_fd(conn_fd: RawFd, fds: &[RawFd]) -> Result<(), anyhow::Error> {
 60 |     let mut iov = [IoSlice::new(&[0u8; 0])];
 61 |     let mut cmsgs = Vec::with_capacity(1);
 62 |     if !fds.is_empty() {
 63 |         let borrowed_fd = unsafe { BorrowedFd::borrow_raw(conn_fd) };
 64 |         let sock_type = getsockopt(&borrowed_fd, SockType)?;
 65 |         if sock_type != nix::sys::socket::SockType::Datagram {
 66 |             iov[0] = IoSlice::new(&[0u8; 1]);
 67 |         }
 68 |         cmsgs.push(ControlMessage::ScmRights(fds))
 69 |     }
 70 |     Ok(sendmsg::<()>(
 71 |         conn_fd,
 72 |         iov.as_slice(),
 73 |         cmsgs.as_slice(),
 74 |         MsgFlags::empty(),
 75 |         None,
 76 |     )
 77 |     .map(|_| ())?)
 78 | }
 79 | 
 80 | pub(crate) fn block_read_out_of_bound_for_fd(conn_fd: RawFd) -> Result<Vec<RawFd>, anyhow::Error> {
 81 |     let mut iov = [IoSliceMut::new(&mut [0u8; 0])];
 82 | 
 83 |     let borrowed_fd = unsafe { BorrowedFd::borrow_raw(conn_fd) };
 84 |     let sock_type = getsockopt(&borrowed_fd, SockType)?;
 85 |     let mut buf = [0u8; 1];
 86 |     if sock_type != nix::sys::socket::SockType::Datagram {
 87 |         iov[0] = IoSliceMut::new(&mut buf);
 88 |     }
 89 |     let mut cmsg_buffer = cmsg_space!([RawFd; MEMFD_COUNT]);
 90 | 
 91 |     let recv_msg = recvmsg::<()>(
 92 |         conn_fd,
 93 |         iov.as_mut_slice(),
 94 |         Some(cmsg_buffer.as_mut()),
 95 |         MsgFlags::empty(),
 96 |     )
 97 |     .map_err(|err| anyhow!("try recv fd from peer failed, reason:{}", err))?;
 98 |     tracing::info!("recvmsg finished");
 99 | 
100 |     if let Some(msgs) = recv_msg.cmsgs()?.next() {
101 |         if let ControlMessageOwned::ScmRights(fds) = msgs {
102 |             Ok(fds)
103 |         } else {
104 |             Err(anyhow!(
105 |                 "parse fd from unix domain failed, reason errno:{}",
106 |                 EINVAL
107 |             ))
108 |         }
109 |     } else {
110 |         Err(anyhow!("parse socket control message ret is nil"))
111 |     }
112 | }
113 | 


--------------------------------------------------------------------------------
/src/protocol/event.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{fmt::Display, ptr::copy_nonoverlapping, sync::LazyLock};
 16 | 
 17 | use super::header::Header;
 18 | use crate::{
 19 |     consts::{HEADER_SIZE, MAGIC_NUMBER, MAX_SUPPORT_PROTO_VERSION},
 20 |     error::Error,
 21 | };
 22 | 
 23 | pub const MIN_EVENT_TYPE: EventType = EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH;
 24 | pub const MAX_EVENT_TYPE: EventType = EventType::TYPE_HOT_RESTART_ACK;
 25 | 
 26 | pub static POLLING_EVENT_WITH_VERSION: LazyLock<Vec<Vec<u8>>> = LazyLock::new(|| {
 27 |     let mut events = Vec::with_capacity(MAX_SUPPORT_PROTO_VERSION as usize + 1);
 28 |     for i in 0..MAX_SUPPORT_PROTO_VERSION + 1 {
 29 |         let mut v = vec![0u8; HEADER_SIZE];
 30 |         let mut event = Header(v.as_mut_ptr());
 31 |         event.encode(HEADER_SIZE as u32, i, EventType::TYPE_POLLING);
 32 |         events.push(v);
 33 |     }
 34 |     events
 35 | });
 36 | 
 37 | /// EventType for internal implements
 38 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 39 | pub struct EventType(u8);
 40 | 
 41 | impl EventType {
 42 |     pub const TYPE_SHARE_MEMORY_BY_FILE_PATH: Self = Self(0);
 43 | 
 44 |     /// notify peer start consume
 45 |     pub const TYPE_POLLING: Self = Self(1);
 46 | 
 47 |     /// stream level event notify peer stream close
 48 |     pub const TYPE_STREAM_CLOSE: Self = Self(2);
 49 | 
 50 |     /// typePing
 51 |     pub const TYPE_FALLBACK_DATA: Self = Self(3);
 52 | 
 53 |     /// exchange proto version
 54 |     pub const TYPE_EXCHANGE_PROTO_VERSION: Self = Self(4);
 55 | 
 56 |     /// query the mem map type supported by the server
 57 |     pub const TYPE_SHARE_MEMORY_BY_MEMFD: Self = Self(5);
 58 | 
 59 |     /// when server mapping share memory success, give the ack to client.
 60 |     pub const TYPE_ACK_SHARE_MEMORY: Self = Self(6);
 61 | 
 62 |     pub const TYPE_ACK_READY_RECV_FD: Self = Self(7);
 63 | 
 64 |     pub const TYPE_HOT_RESTART: Self = Self(8);
 65 | 
 66 |     pub const TYPE_HOT_RESTART_ACK: Self = Self(9);
 67 | 
 68 |     pub fn inner(&self) -> u8 {
 69 |         self.0
 70 |     }
 71 | }
 72 | 
 73 | impl From<u8> for EventType {
 74 |     fn from(v: u8) -> Self {
 75 |         match v {
 76 |             0 => EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH,
 77 |             1 => EventType::TYPE_POLLING,
 78 |             2 => EventType::TYPE_STREAM_CLOSE,
 79 |             3 => EventType::TYPE_FALLBACK_DATA,
 80 |             4 => EventType::TYPE_EXCHANGE_PROTO_VERSION,
 81 |             5 => EventType::TYPE_SHARE_MEMORY_BY_MEMFD,
 82 |             6 => EventType::TYPE_ACK_SHARE_MEMORY,
 83 |             7 => EventType::TYPE_ACK_READY_RECV_FD,
 84 |             8 => EventType::TYPE_HOT_RESTART,
 85 |             9 => EventType::TYPE_HOT_RESTART_ACK,
 86 |             i => Self(i),
 87 |         }
 88 |     }
 89 | }
 90 | 
 91 | impl Display for EventType {
 92 |     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 93 |         let mut unset = "UNSET".to_owned();
 94 |         let s = match self.0 {
 95 |             0 => "ShareMemoryByFilePath",
 96 |             1 => "Polling",
 97 |             2 => "StreamClose",
 98 |             3 => "FallbackData",
 99 |             4 => "ExchangeProtoVersion",
100 |             5 => "ShareMemoryByMemfd",
101 |             6 => "AckShareMemory",
102 |             7 => "AckReadyRecvFD",
103 |             8 => "HotRestart",
104 |             9 => "HotRestartAck",
105 |             i => {
106 |                 unset.push_str(&format!("{i}"));
107 |                 &unset
108 |             }
109 |         };
110 |         write!(f, "{}", s)
111 |     }
112 | }
113 | 
114 | /// header | seq_id | status
115 | #[derive(Clone, Debug)]
116 | pub struct FallbackDataEvent(pub *mut u8);
117 | 
118 | unsafe impl Sync for FallbackDataEvent {}
119 | unsafe impl Send for FallbackDataEvent {}
120 | 
121 | impl FallbackDataEvent {
122 |     pub fn encode(&mut self, length: u32, version: u8, seq_id: u32, status: u32) {
123 |         unsafe {
124 |             copy_nonoverlapping(length.to_be_bytes().as_ptr(), self.0, 4);
125 |             copy_nonoverlapping(MAGIC_NUMBER.to_be_bytes().as_ptr(), self.0.offset(4), 2);
126 | 
127 |             *self.0.offset(6) = version;
128 |             *self.0.offset(7) = EventType::TYPE_FALLBACK_DATA.0;
129 | 
130 |             copy_nonoverlapping(seq_id.to_be_bytes().as_ptr(), self.0.offset(8), 4);
131 |             copy_nonoverlapping(status.to_be_bytes().as_ptr(), self.0.offset(12), 4);
132 |         }
133 |     }
134 | 
135 |     #[inline]
136 |     pub fn as_slice(&self) -> &[u8] {
137 |         unsafe { std::slice::from_raw_parts(self.0, HEADER_SIZE + 8) }
138 |     }
139 | }
140 | 
141 | pub fn check_event_valid(hdr: &Header) -> Result<(), Error> {
142 |     // Verify the magic & version
143 |     if hdr.magic() != MAGIC_NUMBER || hdr.version() == 0 {
144 |         tracing::error!(
145 |             "shmipc: Invalid magic or version {} {}",
146 |             hdr.magic(),
147 |             hdr.version()
148 |         );
149 |         return Err(Error::InvalidVersion);
150 |     }
151 |     let mt = hdr.msg_type();
152 |     if mt < MIN_EVENT_TYPE || mt > MAX_EVENT_TYPE {
153 |         tracing::error!("shmipc, invalid protocol header: {}", hdr);
154 |         return Err(Error::InvalidMsgType);
155 |     }
156 |     Ok(())
157 | }
158 | 


--------------------------------------------------------------------------------
/src/protocol/header.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::{fmt::Display, ptr::copy_nonoverlapping};
16 | 
17 | use super::event::EventType;
18 | use crate::consts::{HEADER_SIZE, MAGIC_NUMBER};
19 | 
20 | #[derive(Clone, Debug)]
21 | pub struct Header(pub *mut u8);
22 | 
23 | unsafe impl Sync for Header {}
24 | unsafe impl Send for Header {}
25 | 
26 | impl Header {
27 |     #[inline]
28 |     pub fn length(&self) -> u32 {
29 |         unsafe { u32::from_be_bytes(*(self.0.cast_const() as *const [u8; 4])) }
30 |     }
31 | 
32 |     #[inline]
33 |     pub fn magic(&self) -> u16 {
34 |         unsafe { u16::from_be_bytes(*(self.0.offset(4).cast_const() as *const [u8; 2])) }
35 |     }
36 | 
37 |     #[inline]
38 |     pub fn version(&self) -> u8 {
39 |         unsafe { *self.0.offset(6) }
40 |     }
41 | 
42 |     #[inline]
43 |     pub fn msg_type(&self) -> EventType {
44 |         unsafe { EventType::from(*self.0.offset(7)) }
45 |     }
46 | 
47 |     pub fn encode(&mut self, length: u32, version: u8, msg_type: EventType) {
48 |         unsafe {
49 |             copy_nonoverlapping(length.to_be_bytes().as_ptr(), self.0, 4);
50 |             copy_nonoverlapping(MAGIC_NUMBER.to_be_bytes().as_ptr(), self.0.offset(4), 2);
51 |             *self.0.offset(6) = version;
52 |             *self.0.offset(7) = msg_type.inner();
53 |         }
54 |     }
55 | 
56 |     #[inline]
57 |     pub fn as_slice(&self) -> &[u8] {
58 |         unsafe { std::slice::from_raw_parts(self.0, HEADER_SIZE) }
59 |     }
60 | }
61 | 
62 | impl Display for Header {
63 |     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64 |         let msg_type = self.msg_type();
65 |         write!(
66 |             f,
67 |             "Header {{ length: {}, magic: {}, version: {}, msg_type: {} }}",
68 |             self.length(),
69 |             self.magic(),
70 |             self.version(),
71 |             msg_type
72 |         )
73 |     }
74 | }
75 | 


--------------------------------------------------------------------------------
/src/protocol/initializer/mod.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | pub mod v2;
 16 | pub mod v3;
 17 | 
 18 | use std::{os::fd::RawFd, sync::Arc};
 19 | 
 20 | use anyhow::anyhow;
 21 | 
 22 | use self::{v2::ProtocolInitializerV2, v3::ProtocolInitializerV3};
 23 | use super::{block_io::block_write_full, event::check_event_valid, header::Header, protocol_trace};
 24 | use crate::{
 25 |     buffer::manager::BufferManager,
 26 |     consts::{HEADER_SIZE, MAX_SUPPORT_PROTO_VERSION, MEMFD_COUNT},
 27 |     protocol::{
 28 |         block_io::{block_read_full, block_read_out_of_bound_for_fd, send_fd},
 29 |         event::EventType,
 30 |     },
 31 |     queue::QueueManager,
 32 | };
 33 | 
 34 | pub enum ProtocolInitializer {
 35 |     V2(ProtocolInitializerV2),
 36 |     V3(ProtocolInitializerV3),
 37 | }
 38 | 
 39 | impl ProtocolInitializer {
 40 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
 41 |         match self {
 42 |             ProtocolInitializer::V2(v2) => v2.init(),
 43 |             ProtocolInitializer::V3(v3) => v3.init(),
 44 |         }
 45 |     }
 46 | 
 47 |     pub fn version(&self) -> u8 {
 48 |         match self {
 49 |             ProtocolInitializer::V2(_) => ProtocolInitializerV2::version(),
 50 |             ProtocolInitializer::V3(_) => ProtocolInitializerV3::version(),
 51 |         }
 52 |     }
 53 | }
 54 | 
 55 | pub fn handle_exchange_version(conn_fd: RawFd) -> Result<(), anyhow::Error> {
 56 |     let mut resp_header = Header([0; HEADER_SIZE].as_mut_ptr());
 57 |     resp_header.encode(
 58 |         HEADER_SIZE as u32,
 59 |         MAX_SUPPORT_PROTO_VERSION,
 60 |         EventType::TYPE_EXCHANGE_PROTO_VERSION,
 61 |     );
 62 |     protocol_trace(&resp_header, &[], true);
 63 |     block_write_full(conn_fd, unsafe {
 64 |         std::slice::from_raw_parts(resp_header.0, HEADER_SIZE)
 65 |     })
 66 | }
 67 | 
 68 | pub fn handle_share_memory_by_memfd(
 69 |     conn_fd: RawFd,
 70 |     h: &Header,
 71 |     version: u8,
 72 | ) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
 73 |     tracing::info!("recv memfd, header:{}", h);
 74 |     // 1.recv shm metadata
 75 |     let mut body = vec![0u8; h.length() as usize - HEADER_SIZE];
 76 |     block_read_full(conn_fd, &mut body)
 77 |         .map_err(|err| anyhow!("read shm metadata failed, reason:{}", err))?;
 78 |     let (buffer_path, queue_path) = extract_shm_metadata(&body);
 79 | 
 80 |     // 2.send AckReadyRecvFD
 81 |     let mut ack = Header([0; HEADER_SIZE].as_mut_ptr());
 82 |     ack.encode(
 83 |         HEADER_SIZE as u32,
 84 |         version,
 85 |         EventType::TYPE_ACK_READY_RECV_FD,
 86 |     );
 87 |     tracing::info!("response typeAckReadyRecvFD");
 88 |     block_write_full(conn_fd, unsafe {
 89 |         std::slice::from_raw_parts(ack.0, HEADER_SIZE)
 90 |     })
 91 |     .map_err(|err| anyhow!("send ack TypeAckReadyRecvFD failed reason:{}", err))?;
 92 |     tracing::info!("TypeAckReadyRecvFD send finished");
 93 | 
 94 |     // 3. recv fd
 95 |     tracing::info!("send ack finished");
 96 |     let fds = block_read_out_of_bound_for_fd(conn_fd)?;
 97 |     if fds.len() < MEMFD_COUNT {
 98 |         tracing::warn!("ParseUnixRights len fds:{}", fds.len());
 99 |         return Err(anyhow!("the number of memfd received is wrong"));
100 |     }
101 | 
102 |     let (buffer_fd, queue_fd) = (fds[0], fds[1]);
103 |     tracing::info!(
104 |         "recv memfd, buffer_path:{} queue_path:{} buffer_fd:{} queue_fd:{}",
105 |         buffer_path,
106 |         queue_path,
107 |         buffer_fd,
108 |         queue_fd
109 |     );
110 | 
111 |     // 4. mapping share memory
112 |     let qm = QueueManager::mapping_with_memfd(queue_path, queue_fd)?;
113 |     let bm = BufferManager::get_with_memfd(buffer_path, buffer_fd, 0, false, &mut []).inspect_err(
114 |         |_| {
115 |             qm.unmap();
116 |         },
117 |     )?;
118 |     tracing::info!("handle_share_memory_by_memfd done");
119 |     Ok(Some((bm, qm)))
120 | }
121 | 
122 | pub fn send_memfd_to_peer(
123 |     conn_fd: RawFd,
124 |     buffer_path: &str,
125 |     buffer_fd: RawFd,
126 |     queue_path: &str,
127 |     queue_fd: RawFd,
128 |     version: u8,
129 | ) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
130 |     let mut event = generate_shm_metadata(
131 |         EventType::TYPE_SHARE_MEMORY_BY_MEMFD,
132 |         buffer_path,
133 |         queue_path,
134 |         version,
135 |     );
136 |     let h = Header(event.as_mut_ptr());
137 |     tracing::info!(
138 |         "send_memfd_to_peer buffer fd:{} queue fd:{} header:{}",
139 |         buffer_fd,
140 |         queue_fd,
141 |         h
142 |     );
143 |     protocol_trace(&h, &event[HEADER_SIZE..], true);
144 | 
145 |     block_write_full(conn_fd, &event)?;
146 |     let mut buf = [0u8; HEADER_SIZE];
147 |     wait_event_header(conn_fd, EventType::TYPE_ACK_READY_RECV_FD, &mut buf)?;
148 |     send_fd(conn_fd, &[buffer_fd, queue_fd])?;
149 |     Ok(None)
150 | }
151 | 
152 | pub fn wait_event_header(
153 |     conn_fd: RawFd,
154 |     expect_event_type: EventType,
155 |     buf: &mut [u8],
156 | ) -> Result<Header, anyhow::Error> {
157 |     let h = block_read_event_header(conn_fd, buf)?;
158 |     if h.msg_type() != expect_event_type {
159 |         return Err(anyhow!(
160 |             "expect event_type:{} {}, but:{}",
161 |             expect_event_type.inner(),
162 |             expect_event_type,
163 |             h.msg_type(),
164 |         ));
165 |     }
166 | 
167 |     Ok(h)
168 | }
169 | 
170 | pub fn block_read_event_header(conn_fd: RawFd, buf: &mut [u8]) -> Result<Header, anyhow::Error> {
171 |     block_read_full(conn_fd, buf)?;
172 |     let h = Header(buf.as_mut_ptr());
173 |     check_event_valid(&h)?;
174 |     protocol_trace(&h, &[], false);
175 |     Ok(h)
176 | }
177 | 
178 | pub fn handle_share_memory_by_file_path(
179 |     conn_fd: RawFd,
180 |     hdr: &Header,
181 | ) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
182 |     tracing::info!("handle_share_memory_by_file_path head:{:?}", hdr);
183 |     let mut body = vec![0u8; hdr.length() as usize - HEADER_SIZE];
184 |     if let Err(err) = block_read_full(conn_fd, body.as_mut()) {
185 |         if !err.to_string().eq("EOF")
186 |             && !err.to_string().contains("closed")
187 |             && !err.to_string().contains("reset by peer")
188 |         {
189 |             tracing::error!("shmipc: failed to read pathlen: {}", err);
190 |         }
191 |         return Err(err);
192 |     }
193 |     let (buffer_path, queue_path) = extract_shm_metadata(&body);
194 |     let qm = QueueManager::mapping_with_file(queue_path).map_err(|err| {
195 |         anyhow!(
196 |             "handle_share_memory_by_file_path mappingQueueManager failed,queuePathLen:{} path:{} \
197 |              err={}",
198 |             queue_path.len(),
199 |             queue_path,
200 |             err
201 |         )
202 |     })?;
203 | 
204 |     let bm = BufferManager::get_with_file(buffer_path, 0, false, &mut []).map_err(|err| {
205 |         qm.unmap();
206 |         anyhow!(
207 |             "handle_share_memory_by_file_path mapping_buffer_manager failed, buffer_path_len:{} \
208 |              path:{} err={}",
209 |             buffer_path.len(),
210 |             buffer_path,
211 |             err
212 |         )
213 |     })?;
214 | 
215 |     Ok(Some((bm, qm)))
216 | }
217 | 
218 | fn extract_shm_metadata(body: &[u8]) -> (&str, &str) {
219 |     let mut offset = 0;
220 |     let queue_path_len = u16::from_be_bytes([body[offset], body[offset + 1]]);
221 |     offset += 2;
222 |     let queue_path =
223 |         unsafe { std::str::from_utf8_unchecked(&body[offset..offset + queue_path_len as usize]) };
224 |     offset += queue_path_len as usize;
225 | 
226 |     let buffer_path_len = u16::from_be_bytes([body[offset], body[offset + 1]]);
227 |     offset += 2;
228 |     let buffer_path =
229 |         unsafe { std::str::from_utf8_unchecked(&body[offset..offset + buffer_path_len as usize]) };
230 |     (buffer_path, queue_path)
231 | }
232 | 
233 | pub fn send_share_memory_by_file_path(
234 |     conn_fd: RawFd,
235 |     buffer_path: &str,
236 |     queue_path: &str,
237 |     version: u8,
238 | ) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
239 |     let mut data = generate_shm_metadata(
240 |         EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH,
241 |         buffer_path,
242 |         queue_path,
243 |         version,
244 |     );
245 |     let h = Header(data.as_mut_ptr());
246 |     protocol_trace(&h, &data[HEADER_SIZE..], true);
247 |     block_write_full(conn_fd, &data)?;
248 |     Ok(None)
249 | }
250 | 
251 | fn generate_shm_metadata(
252 |     event_type: EventType,
253 |     buffer_path: &str,
254 |     queue_path: &str,
255 |     version: u8,
256 | ) -> Vec<u8> {
257 |     let mut data = vec![0u8; HEADER_SIZE + 2 + buffer_path.len() + 2 + queue_path.len()];
258 |     let mut h = Header(data.as_mut_ptr());
259 |     h.encode(data.len() as u32, version, event_type);
260 |     let mut offset = HEADER_SIZE;
261 |     // queue share memory path
262 |     data[offset..offset + 2].copy_from_slice(&(queue_path.len() as u16).to_be_bytes());
263 |     offset += 2;
264 |     data[offset..offset + queue_path.len()].copy_from_slice(queue_path.as_bytes());
265 |     offset += queue_path.len();
266 |     // buffer share memory path
267 |     data[offset..offset + 2].copy_from_slice(&(buffer_path.len() as u16).to_be_bytes());
268 |     offset += 2;
269 |     data[offset..offset + buffer_path.len()].copy_from_slice(buffer_path.as_bytes());
270 |     data
271 | }
272 | 


--------------------------------------------------------------------------------
/src/protocol/initializer/v2.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::{os::fd::RawFd, sync::Arc};
16 | 
17 | use anyhow::anyhow;
18 | 
19 | use super::{handle_share_memory_by_file_path, send_share_memory_by_file_path};
20 | use crate::{
21 |     buffer::manager::BufferManager,
22 |     protocol::{event::EventType, header::Header},
23 |     queue::QueueManager,
24 | };
25 | 
26 | pub enum ProtocolInitializerV2 {
27 |     Client(Client),
28 |     Server(Server),
29 | }
30 | 
31 | impl ProtocolInitializerV2 {
32 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
33 |         match self {
34 |             ProtocolInitializerV2::Client(client) => client.init(),
35 |             ProtocolInitializerV2::Server(server) => server.init(),
36 |         }
37 |     }
38 | 
39 |     pub fn version() -> u8 {
40 |         2
41 |     }
42 | }
43 | 
44 | pub struct Client {
45 |     pub(crate) conn_fd: RawFd,
46 |     pub(crate) buffer_path: String,
47 |     pub(crate) queue_path: String,
48 | }
49 | 
50 | impl Client {
51 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
52 |         send_share_memory_by_file_path(self.conn_fd, &self.buffer_path, &self.queue_path, 2)
53 |     }
54 | }
55 | 
56 | pub struct Server {
57 |     pub(crate) conn_fd: RawFd,
58 |     pub(crate) first_event: Header,
59 | }
60 | 
61 | impl Server {
62 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
63 |         if self.first_event.msg_type() != EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH {
64 |             return Err(anyhow!(
65 |                 "ProtocolInitializerV2 expect first event is:{}({}),but:{}",
66 |                 EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH.inner(),
67 |                 EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH,
68 |                 self.first_event.msg_type().inner()
69 |             ));
70 |         }
71 |         handle_share_memory_by_file_path(self.conn_fd, &self.first_event)
72 |     }
73 | }
74 | 


--------------------------------------------------------------------------------
/src/protocol/initializer/v3.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{os::fd::RawFd, sync::Arc};
 16 | 
 17 | use anyhow::anyhow;
 18 | 
 19 | use super::{
 20 |     block_read_event_header, block_write_full, handle_exchange_version,
 21 |     handle_share_memory_by_file_path, handle_share_memory_by_memfd, send_memfd_to_peer,
 22 |     send_share_memory_by_file_path, wait_event_header,
 23 | };
 24 | use crate::{
 25 |     buffer::manager::BufferManager,
 26 |     consts::{HEADER_SIZE, MemMapType},
 27 |     protocol::{event::EventType, header::Header, protocol_trace},
 28 |     queue::QueueManager,
 29 | };
 30 | 
 31 | pub enum ProtocolInitializerV3 {
 32 |     Client(Client),
 33 |     Server(Server),
 34 | }
 35 | 
 36 | impl ProtocolInitializerV3 {
 37 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
 38 |         match self {
 39 |             ProtocolInitializerV3::Client(client) => client.init(),
 40 |             ProtocolInitializerV3::Server(server) => server.init(),
 41 |         }
 42 |     }
 43 | 
 44 |     pub fn version() -> u8 {
 45 |         3
 46 |     }
 47 | }
 48 | 
 49 | pub struct Client {
 50 |     pub(crate) conn_fd: RawFd,
 51 |     pub(crate) mem_map_type: MemMapType,
 52 |     pub(crate) buffer_path: String,
 53 |     pub(crate) queue_path: String,
 54 |     pub(crate) buffer_fd: RawFd,
 55 |     pub(crate) queue_fd: RawFd,
 56 | }
 57 | 
 58 | impl Client {
 59 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
 60 |         match self.mem_map_type {
 61 |             MemMapType::MemMapTypeDevShmFile => {
 62 |                 send_share_memory_by_file_path(self.conn_fd, &self.buffer_path, &self.queue_path, 3)
 63 |             }
 64 |             MemMapType::MemMapTypeMemFd => send_memfd_to_peer(
 65 |                 self.conn_fd,
 66 |                 &self.buffer_path,
 67 |                 self.buffer_fd,
 68 |                 &self.queue_path,
 69 |                 self.queue_fd,
 70 |                 3,
 71 |             ),
 72 |         }?;
 73 |         let mut buf = [0u8; HEADER_SIZE];
 74 |         wait_event_header(self.conn_fd, EventType::TYPE_ACK_SHARE_MEMORY, &mut buf)?;
 75 |         Ok(None)
 76 |     }
 77 | }
 78 | 
 79 | pub struct Server {
 80 |     pub(crate) conn_fd: RawFd,
 81 |     pub(crate) first_event: Header,
 82 | }
 83 | 
 84 | impl Server {
 85 |     pub fn init(&self) -> Result<Option<(Arc<BufferManager>, QueueManager)>, anyhow::Error> {
 86 |         if EventType::TYPE_EXCHANGE_PROTO_VERSION != self.first_event.msg_type() {
 87 |             return Err(anyhow!(
 88 |                 "ProtocolInitializerV3 expect first event is:{}({}),but:{}",
 89 |                 EventType::TYPE_EXCHANGE_PROTO_VERSION.inner(),
 90 |                 EventType::TYPE_EXCHANGE_PROTO_VERSION,
 91 |                 self.first_event.msg_type().inner()
 92 |             ));
 93 |         }
 94 | 
 95 |         // 1. exchange version
 96 |         handle_exchange_version(self.conn_fd).map_err(|err| {
 97 |             anyhow!(
 98 |                 "ProtocolInitializerV3 exchange_version failed, reason:{}",
 99 |                 err
100 |             )
101 |         })?;
102 | 
103 |         // 2. recv and mapping share memory
104 |         let mut buf = [0u8; HEADER_SIZE];
105 |         let h = block_read_event_header(self.conn_fd, &mut buf).map_err(|err| {
106 |             anyhow!(
107 |                 "ProtocolInitializerV3 block_read_event_header failed, reason:{}",
108 |                 err
109 |             )
110 |         })?;
111 |         let r = match h.msg_type() {
112 |             EventType::TYPE_SHARE_MEMORY_BY_FILE_PATH => {
113 |                 handle_share_memory_by_file_path(self.conn_fd, &h)
114 |             }
115 |             EventType::TYPE_SHARE_MEMORY_BY_MEMFD => {
116 |                 handle_share_memory_by_memfd(self.conn_fd, &h, 3)
117 |             }
118 |             _ => {
119 |                 return Err(anyhow!(
120 |                     "expect event type is TypeShareMemoryByFilePath or TypeShareMemoryByMemfd, \
121 |                      but:{}({})",
122 |                     h.msg_type().inner(),
123 |                     h.msg_type()
124 |                 ));
125 |             }
126 |         }?;
127 |         // 3. ack share memory
128 |         let mut resp_header = Header([0u8; HEADER_SIZE].as_mut_ptr());
129 |         resp_header.encode(HEADER_SIZE as u32, 3, EventType::TYPE_ACK_SHARE_MEMORY);
130 |         protocol_trace(&resp_header, &[], true);
131 |         block_write_full(self.conn_fd, unsafe {
132 |             std::slice::from_raw_parts(resp_header.0, HEADER_SIZE)
133 |         })?;
134 |         Ok(r)
135 |     }
136 | }
137 | 


--------------------------------------------------------------------------------
/src/protocol/mod.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | pub mod adapter;
 16 | pub mod block_io;
 17 | pub mod event;
 18 | pub mod header;
 19 | pub mod initializer;
 20 | 
 21 | use std::{
 22 |     os::fd::RawFd,
 23 |     sync::{Arc, LazyLock},
 24 |     time::Duration,
 25 | };
 26 | 
 27 | use adapter::{ClientProtocolAdapter, ServerProtocolAdapter};
 28 | use anyhow::anyhow;
 29 | use header::Header;
 30 | 
 31 | use crate::{
 32 |     buffer::manager::BufferManager,
 33 |     config::Config,
 34 |     consts::{BUFFER_PATH_SUFFIX, MemMapType},
 35 |     queue::QueueManager,
 36 | };
 37 | 
 38 | pub static PROTOCOL_TRACE_MODE: LazyLock<bool> =
 39 |     LazyLock::new(|| match std::env::var("SHMIPC_PROTOCOL_TRACE") {
 40 |         Ok(val) => !val.is_empty(),
 41 |         Err(_) => false,
 42 |     });
 43 | 
 44 | pub fn protocol_trace(h: &Header, body: &[u8], send: bool) {
 45 |     if !*PROTOCOL_TRACE_MODE {
 46 |         return;
 47 |     }
 48 |     if send {
 49 |         tracing::warn!("send, header:{} body:{}", h, String::from_utf8_lossy(body));
 50 |     } else {
 51 |         tracing::warn!("recv, header:{} body:{}", h, String::from_utf8_lossy(body));
 52 |     }
 53 | }
 54 | 
 55 | pub fn init_manager(
 56 |     config: &mut Config,
 57 | ) -> Result<(Arc<BufferManager>, QueueManager), anyhow::Error> {
 58 |     Ok(match config.mem_map_type {
 59 |         MemMapType::MemMapTypeDevShmFile => {
 60 |             let bm = match BufferManager::get_with_file(
 61 |                 &format!("{}{}", config.share_memory_path_prefix, BUFFER_PATH_SUFFIX),
 62 |                 config.share_memory_buffer_cap,
 63 |                 true,
 64 |                 &mut config.buffer_slice_sizes,
 65 |             ) {
 66 |                 Ok(manager) => manager,
 67 |                 Err(err) => {
 68 |                     _ = std::fs::remove_dir(format!(
 69 |                         "{}{}",
 70 |                         config.share_memory_path_prefix, BUFFER_PATH_SUFFIX
 71 |                     ));
 72 |                     return Err(anyhow!(
 73 |                         "create share memory buffer manager failed, error={}",
 74 |                         err
 75 |                     ));
 76 |                 }
 77 |             };
 78 |             let qm = match QueueManager::create_with_file(&config.queue_path, config.queue_cap) {
 79 |                 Ok(manager) => manager,
 80 |                 Err(err) => {
 81 |                     _ = std::fs::remove_dir(&config.queue_path);
 82 |                     return Err(anyhow!(
 83 |                         "create share memory queue manager failed, error={}",
 84 |                         err
 85 |                     ));
 86 |                 }
 87 |             };
 88 |             (bm, qm)
 89 |         }
 90 |         MemMapType::MemMapTypeMemFd => {
 91 |             let bm = match BufferManager::get_with_memfd(
 92 |                 &format!("{}{}", config.share_memory_path_prefix, BUFFER_PATH_SUFFIX),
 93 |                 0,
 94 |                 config.share_memory_buffer_cap,
 95 |                 true,
 96 |                 &mut config.buffer_slice_sizes,
 97 |             ) {
 98 |                 Ok(manager) => manager,
 99 |                 Err(err) => {
100 |                     return Err(anyhow!(
101 |                         "create share memory buffer manager failed, error={}",
102 |                         err
103 |                     ));
104 |                 }
105 |             };
106 |             let qm = match QueueManager::create_with_memfd(&config.queue_path, config.queue_cap) {
107 |                 Ok(manager) => manager,
108 |                 Err(err) => {
109 |                     return Err(anyhow!(
110 |                         "create share memory queue manager failed, error={}",
111 |                         err
112 |                     ));
113 |                 }
114 |             };
115 |             (bm, qm)
116 |         }
117 |     })
118 | }
119 | 
120 | pub async fn init_client_protocol(
121 |     buffer_path: String,
122 |     buffer_memfd: RawFd,
123 |     queue_path: String,
124 |     queue_memfd: RawFd,
125 |     conn_fd: RawFd,
126 |     mem_map_type: MemMapType,
127 |     timeout: Duration,
128 | ) -> Result<u8, anyhow::Error> {
129 |     tracing::info!("starting initializes shmipc client protocol");
130 |     let handler = tokio::task::spawn_blocking(move || {
131 |         let adapter = ClientProtocolAdapter::new(
132 |             conn_fd,
133 |             mem_map_type,
134 |             buffer_path,
135 |             queue_path,
136 |             buffer_memfd,
137 |             queue_memfd,
138 |         );
139 |         let initializer = adapter.get_initializer()?;
140 |         initializer.init()?;
141 |         Ok::<_, anyhow::Error>(initializer.version())
142 |     });
143 |     tokio::select! {
144 |         res = handler => {
145 |             res?
146 |         }
147 |         _ = tokio::time::sleep(timeout) => {
148 |             Err(anyhow!("ProtocolInitializer init client timeout:{:?} ms", timeout))
149 |         }
150 |     }
151 | }
152 | 
153 | pub async fn init_server_protocol(
154 |     conn_fd: RawFd,
155 |     timeout: Duration,
156 | ) -> Result<(Arc<BufferManager>, QueueManager, u8), anyhow::Error> {
157 |     tracing::info!("starting initializes shmipc server protocol");
158 |     let handler = tokio::task::spawn_blocking(move || {
159 |         let adapter = ServerProtocolAdapter::new(conn_fd);
160 |         let initializer = adapter.get_initializer()?;
161 |         let (bm, qm) = initializer.init()?.unwrap();
162 |         Ok((bm, qm, initializer.version()))
163 |     });
164 |     tokio::select! {
165 |         res = handler => {
166 |             res?
167 |         }
168 |         _ = tokio::time::sleep(timeout) => {
169 |             Err(anyhow!("ProtocolInitializer init server timeout:{:?} ms", timeout))
170 |         }
171 |     }
172 | }
173 | 


--------------------------------------------------------------------------------
/src/queue.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{
 16 |     ffi::CString,
 17 |     fs::{self, File, OpenOptions, Permissions},
 18 |     os::{
 19 |         fd::{AsRawFd, FromRawFd, IntoRawFd, RawFd},
 20 |         unix::prelude::PermissionsExt,
 21 |     },
 22 |     path::Path,
 23 |     sync::{
 24 |         Mutex,
 25 |         atomic::{AtomicU32, Ordering},
 26 |     },
 27 | };
 28 | 
 29 | use anyhow::anyhow;
 30 | use memmap2::{MmapMut, MmapOptions};
 31 | 
 32 | use crate::{
 33 |     consts::{MemMapType, QUEUE_COUNT, QUEUE_ELEMENT_LEN},
 34 |     error::Error,
 35 |     util::can_create_on_dev_shm,
 36 | };
 37 | 
 38 | const QUEUE_HEADER_LENGTH: usize = 24;
 39 | 
 40 | #[derive(Debug)]
 41 | pub struct QueueManager {
 42 |     pub(crate) path: String,
 43 |     pub(crate) send_queue: Queue,
 44 |     pub(crate) recv_queue: Queue,
 45 |     pub(crate) memfd: RawFd,
 46 |     #[allow(dead_code)]
 47 |     mem: MmapMut,
 48 |     mmap_map_type: MemMapType,
 49 | }
 50 | 
 51 | impl QueueManager {
 52 |     pub fn create_with_memfd(queue_path_name: &str, queue_cap: u32) -> Result<Self, anyhow::Error> {
 53 |         let memfd = nix::sys::memfd::memfd_create(
 54 |             CString::new(format!("shmipc{}", queue_path_name))
 55 |                 .expect("CString::new failed")
 56 |                 .as_c_str(),
 57 |             nix::sys::memfd::MemFdCreateFlag::empty(),
 58 |         )?;
 59 | 
 60 |         let mem_size = count_queue_mem_size(queue_cap) * QUEUE_COUNT;
 61 |         nix::unistd::ftruncate(&memfd, mem_size as i64).map_err(|err| {
 62 |             anyhow!(
 63 |                 "create_queue_manager_with_memfd truncate share memory failed: {}",
 64 |                 err
 65 |             )
 66 |         })?;
 67 | 
 68 |         let mut mem = unsafe { MmapOptions::new().len(mem_size).map_mut(&memfd)? };
 69 |         mem.fill(0);
 70 | 
 71 |         Ok(Self {
 72 |             path: queue_path_name.to_owned(),
 73 |             send_queue: Queue::create_from_bytes(mem.as_mut_ptr(), queue_cap),
 74 |             recv_queue: Queue::create_from_bytes(
 75 |                 unsafe { mem.as_mut_ptr().add(mem_size / 2) },
 76 |                 queue_cap,
 77 |             ),
 78 |             mem,
 79 |             mmap_map_type: MemMapType::MemMapTypeMemFd,
 80 |             memfd: memfd.into_raw_fd(),
 81 |         })
 82 |     }
 83 | 
 84 |     pub fn create_with_file(shm_path: &str, queue_cap: u32) -> Result<Self, anyhow::Error> {
 85 |         // ignore mkdir error
 86 |         let path = Path::new(shm_path);
 87 |         _ = fs::create_dir_all(path.parent().unwrap_or(Path::new("/")));
 88 |         _ = fs::set_permissions(shm_path, Permissions::from_mode(0o777));
 89 |         if path.exists() {
 90 |             return Err(anyhow!("queue was existed, path:{}", shm_path));
 91 |         }
 92 |         let mem_size = count_queue_mem_size(queue_cap) * QUEUE_COUNT;
 93 |         if !can_create_on_dev_shm(mem_size as u64, shm_path) {
 94 |             return Err(anyhow!(
 95 |                 "err: share memory had not left space, path:{}, size:{}",
 96 |                 shm_path,
 97 |                 mem_size
 98 |             ));
 99 |         }
100 | 
101 |         let shm_file = OpenOptions::new()
102 |             .read(true)
103 |             .write(true)
104 |             .create(true)
105 |             .truncate(true)
106 |             .open(shm_path)?;
107 |         shm_file.set_permissions(Permissions::from_mode(0o777))?;
108 |         shm_file.set_len(mem_size as u64)?;
109 | 
110 |         let mut mem = unsafe {
111 |             MmapOptions::new()
112 |                 .len(mem_size)
113 |                 .map_mut(shm_file.as_raw_fd())?
114 |         };
115 |         mem.fill(0);
116 | 
117 |         Ok(Self {
118 |             path: shm_path.to_owned(),
119 |             send_queue: Queue::create_from_bytes(mem.as_mut_ptr(), queue_cap),
120 |             recv_queue: Queue::create_from_bytes(
121 |                 unsafe { mem.as_mut_ptr().add(mem_size / 2) },
122 |                 queue_cap,
123 |             ),
124 |             mem,
125 |             mmap_map_type: MemMapType::MemMapTypeDevShmFile,
126 |             memfd: 0,
127 |         })
128 |     }
129 | 
130 |     pub fn mapping_with_memfd(queue_path_name: &str, memfd: RawFd) -> Result<Self, anyhow::Error> {
131 |         let file = unsafe { File::from_raw_fd(memfd) };
132 |         let fi = file.metadata()?;
133 | 
134 |         let mapping_size = fi.len();
135 |         #[cfg(any(target_arch = "arm", target_arch = "arm64ec"))]
136 |         // a queueManager have two queue, a queue's head and tail should align to 8 byte boundary
137 |         if mapping_size % 16 != 0 {
138 |             return Err(anyhow!(
139 |                 "the memory size of queue should be a multiple of 16"
140 |             ));
141 |         }
142 | 
143 |         let mut mem = unsafe {
144 |             MmapOptions::new()
145 |                 .len(mapping_size as usize)
146 |                 .map_mut(memfd)?
147 |         };
148 |         Ok(Self {
149 |             path: queue_path_name.to_owned(),
150 |             send_queue: Queue::mapping_from_bytes(unsafe {
151 |                 mem.as_mut_ptr().offset((mapping_size / 2) as isize)
152 |             }),
153 |             recv_queue: Queue::mapping_from_bytes(mem.as_mut_ptr()),
154 |             mem,
155 |             mmap_map_type: MemMapType::MemMapTypeMemFd,
156 |             memfd,
157 |         })
158 |     }
159 | 
160 |     pub fn mapping_with_file(shm_path: &str) -> Result<Self, anyhow::Error> {
161 |         let file = OpenOptions::new().read(true).write(true).open(shm_path)?;
162 |         file.set_permissions(Permissions::from_mode(0o777))?;
163 |         let fi = file.metadata()?;
164 | 
165 |         let mapping_size = fi.len();
166 |         #[cfg(any(target_arch = "arm", target_arch = "arm64ec"))]
167 |         // a queueManager have two queue, a queue's head and tail should align to 8 byte boundary
168 |         if mapping_size % 16 != 0 {
169 |             return Err(anyhow!(
170 |                 "the memory size of queue should be a multiple of 16"
171 |             ));
172 |         }
173 | 
174 |         let mut mem = unsafe {
175 |             MmapOptions::new()
176 |                 .len(mapping_size as usize)
177 |                 .map_mut(file.as_raw_fd())?
178 |         };
179 |         Ok(Self {
180 |             path: shm_path.to_owned(),
181 |             send_queue: Queue::mapping_from_bytes(unsafe {
182 |                 mem.as_mut_ptr().offset((mapping_size / 2) as isize)
183 |             }),
184 |             recv_queue: Queue::mapping_from_bytes(mem.as_mut_ptr()),
185 |             mem,
186 |             mmap_map_type: MemMapType::MemMapTypeDevShmFile,
187 |             memfd: 0,
188 |         })
189 |     }
190 | 
191 |     pub fn unmap(&self) {
192 |         if let MemMapType::MemMapTypeDevShmFile = self.mmap_map_type {
193 |             if let Err(e) = std::fs::remove_file(&self.path) {
194 |                 tracing::warn!("queueManager remove file:{} failed, error={}", self.path, e);
195 |             } else {
196 |                 tracing::info!("queueManager remove file:{}", self.path);
197 |             }
198 |         } else if let Err(err) = nix::unistd::close(self.memfd) {
199 |             tracing::warn!("queueManager close fd:{} failed, error={}", self.memfd, err);
200 |         } else {
201 |             tracing::info!("queueManager close fd:{}", self.memfd);
202 |         }
203 |     }
204 | }
205 | 
206 | #[derive(Debug)]
207 | pub struct Queue {
208 |     // consumer write, producer read
209 |     pub(crate) head: *mut i64,
210 |     // producer write, consumer read
211 |     pub(crate) tail: *mut i64,
212 |     /// when peer is consuming the queue, the working_flag is 1, otherwise 0
213 |     working_flag: *const AtomicU32,
214 |     // it could be from share memory or process memory.
215 |     queue_bytes_on_memory: *const u8,
216 |     cap: i64,
217 |     #[allow(dead_code)]
218 |     len: usize,
219 |     lock: Mutex<()>,
220 | }
221 | 
222 | unsafe impl Send for Queue {}
223 | unsafe impl Sync for Queue {}
224 | 
225 | pub struct QueueElement {
226 |     pub(crate) seq_id: u32,
227 |     pub(crate) offset_in_shm_buf: u32,
228 |     pub(crate) status: u32,
229 | }
230 | 
231 | impl Queue {
232 |     pub fn create_from_bytes(data: *mut u8, cap: u32) -> Self {
233 |         unsafe { *(data as *mut u32) = cap };
234 |         let q = Self::mapping_from_bytes(data);
235 |         unsafe {
236 |             // Due to the previous shmipc specification, the head and tail of the queue is not align
237 |             // to 8 byte boundary
238 |             q.head.write_unaligned(0);
239 |             q.tail.write_unaligned(0);
240 |             (*q.working_flag).store(0, Ordering::SeqCst);
241 |         }
242 |         q
243 |     }
244 | 
245 |     pub fn mapping_from_bytes(data: *mut u8) -> Self {
246 |         let cap = unsafe { *(data as *mut u32) };
247 |         let queue_start_offset = QUEUE_HEADER_LENGTH;
248 |         let queue_end_offset = QUEUE_HEADER_LENGTH + QUEUE_ELEMENT_LEN * cap as usize;
249 |         #[cfg(any(target_arch = "arm", target_arch = "arm64ec"))]
250 |         unsafe {
251 |             Queue {
252 |                 cap: cap as i64,
253 |                 working_flag: data.offset(4) as *const AtomicU32,
254 |                 head: data.offset(8) as *mut i64,
255 |                 tail: data.offset(16) as *mut i64,
256 |                 queue_bytes_on_memory: data.offset(queue_start_offset as isize) as *const u8,
257 |                 len: queue_end_offset - queue_start_offset,
258 |             }
259 |         }
260 |         // TODO: unaligned head and tail
261 |         #[cfg(not(any(target_arch = "arm", target_arch = "arm64ec")))]
262 |         unsafe {
263 |             Self {
264 |                 cap: cap as i64,
265 |                 working_flag: data.offset(20) as *const AtomicU32,
266 |                 head: data.offset(4) as *mut i64,
267 |                 tail: data.offset(12) as *mut i64,
268 |                 queue_bytes_on_memory: data.add(queue_start_offset) as *const u8,
269 |                 len: queue_end_offset - queue_start_offset,
270 |                 lock: Mutex::new(()),
271 |             }
272 |         }
273 |     }
274 | 
275 |     pub fn put(&self, element: QueueElement) -> Result<(), Error> {
276 |         let _tail_lock = self.lock.lock().unwrap();
277 |         unsafe {
278 |             if self.tail.read_unaligned() - self.head.read_unaligned() >= self.cap {
279 |                 return Err(Error::QueueFull);
280 |             }
281 |             let queue_offset =
282 |                 (self.tail.read_unaligned() % self.cap) as isize * QUEUE_ELEMENT_LEN as isize;
283 |             *(self.queue_bytes_on_memory.offset(queue_offset) as *mut u32) = element.seq_id;
284 |             *(self.queue_bytes_on_memory.offset(queue_offset + 4) as *mut u32) =
285 |                 element.offset_in_shm_buf;
286 |             *(self.queue_bytes_on_memory.offset(queue_offset + 8) as *mut u32) = element.status;
287 |             self.tail.write_unaligned(self.tail.read_unaligned() + 1);
288 |         };
289 |         Ok(())
290 |     }
291 | 
292 |     pub fn pop(&self) -> Result<QueueElement, Error> {
293 |         unsafe {
294 |             if self.head.read_unaligned() >= self.tail.read_unaligned() {
295 |                 return Err(Error::QueueEmpty);
296 |             }
297 |             let queue_offset =
298 |                 (self.head.read_unaligned() % self.cap) as isize * QUEUE_ELEMENT_LEN as isize;
299 |             let element = QueueElement {
300 |                 seq_id: *(self.queue_bytes_on_memory.offset(queue_offset) as *const u32),
301 |                 offset_in_shm_buf: *(self.queue_bytes_on_memory.offset(queue_offset + 4)
302 |                     as *const u32),
303 |                 status: *(self.queue_bytes_on_memory.offset(queue_offset + 8) as *const u32),
304 |             };
305 | 
306 |             self.head.write_unaligned(self.head.read_unaligned() + 1);
307 |             Ok(element)
308 |         }
309 |     }
310 | 
311 |     #[allow(unused)]
312 |     pub fn is_full(&self) -> bool {
313 |         self.size() == self.cap
314 |     }
315 | 
316 |     #[allow(unused)]
317 |     pub fn is_empty(&self) -> bool {
318 |         self.size() == 0
319 |     }
320 | 
321 |     pub fn size(&self) -> i64 {
322 |         unsafe { self.tail.read_unaligned() - self.head.read_unaligned() }
323 |     }
324 | 
325 |     #[allow(unused)]
326 |     pub fn consumer_is_working(&self) -> bool {
327 |         unsafe { (*self.working_flag).load(Ordering::SeqCst) > 0 }
328 |     }
329 | 
330 |     pub fn mark_working(&self) -> bool {
331 |         unsafe { (*self.working_flag).compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) }
332 |             .is_ok()
333 |     }
334 | 
335 |     pub fn mark_not_working(&self) -> bool {
336 |         unsafe {
337 |             (*self.working_flag).store(0, Ordering::SeqCst);
338 |         }
339 |         if self.size() == 0 {
340 |             return true;
341 |         }
342 |         unsafe {
343 |             (*self.working_flag).store(1, Ordering::SeqCst);
344 |         }
345 |         false
346 |     }
347 | }
348 | 
349 | fn count_queue_mem_size(queue_cap: u32) -> usize {
350 |     QUEUE_HEADER_LENGTH + QUEUE_ELEMENT_LEN * queue_cap as usize
351 | }
352 | 
353 | #[cfg(test)]
354 | mod test {
355 |     use std::{sync::Arc, time::Duration};
356 | 
357 |     use super::{QUEUE_HEADER_LENGTH, Queue};
358 |     use crate::{
359 |         consts::QUEUE_ELEMENT_LEN,
360 |         queue::{QueueElement, QueueManager},
361 |     };
362 | 
363 |     #[test]
364 |     fn test_queue_manager_create_mapping() {
365 |         let path = "/tmp/ipc1.queue";
366 | 
367 |         let qm1 = QueueManager::create_with_file(path, 8192).unwrap();
368 |         let qm2 = QueueManager::mapping_with_file(path).unwrap();
369 | 
370 |         assert!(
371 |             qm1.send_queue
372 |                 .put(QueueElement {
373 |                     seq_id: 0,
374 |                     offset_in_shm_buf: 0,
375 |                     status: 0
376 |                 })
377 |                 .is_ok()
378 |         );
379 |         assert!(qm2.recv_queue.pop().is_ok());
380 | 
381 |         assert!(
382 |             qm2.send_queue
383 |                 .put(QueueElement {
384 |                     seq_id: 0,
385 |                     offset_in_shm_buf: 0,
386 |                     status: 0
387 |                 })
388 |                 .is_ok()
389 |         );
390 |         assert!(qm1.recv_queue.pop().is_ok());
391 |         qm1.unmap();
392 |     }
393 | 
394 |     #[test]
395 |     fn test_queue_operate() {
396 |         let q = create_queue(8192);
397 | 
398 |         assert!(q.is_empty());
399 |         assert!(!q.is_full());
400 |         assert_eq!(0, q.size());
401 | 
402 |         let mut put_count = 0;
403 |         for i in 0..8192 {
404 |             assert!(
405 |                 q.put(QueueElement {
406 |                     seq_id: i,
407 |                     offset_in_shm_buf: i,
408 |                     status: i
409 |                 })
410 |                 .is_ok()
411 |             );
412 |             put_count += 1;
413 |         }
414 |         let r = q.put(QueueElement {
415 |             seq_id: 1,
416 |             offset_in_shm_buf: 1,
417 |             status: 1,
418 |         });
419 |         assert!(r.is_err());
420 |         assert!(q.is_full());
421 |         assert!(!q.is_empty());
422 |         assert_eq!(put_count, q.size());
423 | 
424 |         for i in 0..8192 {
425 |             let e = q.pop().unwrap();
426 |             assert_eq!(i, e.seq_id);
427 |             assert_eq!(i, e.offset_in_shm_buf);
428 |             assert_eq!(i, e.status);
429 |         }
430 | 
431 |         let r = q.pop();
432 |         assert!(r.is_err());
433 |         assert!(q.is_empty());
434 |         assert!(!q.is_full());
435 |         assert_eq!(0, q.size());
436 | 
437 |         assert!(!q.consumer_is_working());
438 |         q.mark_working();
439 |         assert!(q.consumer_is_working());
440 |         q.mark_not_working();
441 |         assert!(!q.consumer_is_working());
442 | 
443 |         _ = q.put(QueueElement {
444 |             seq_id: 1,
445 |             offset_in_shm_buf: 1,
446 |             status: 1,
447 |         });
448 |         q.mark_not_working();
449 |         assert!(q.consumer_is_working());
450 |     }
451 | 
452 |     #[tokio::test(flavor = "multi_thread")]
453 |     async fn test_queue_multi_producer_and_single_consumer() {
454 |         let q = Arc::new(create_queue(100000));
455 |         let mut pop_count = 0;
456 |         for _ in 0..100 {
457 |             let q = q.clone();
458 |             tokio::spawn(async move {
459 |                 for _ in 0..1000 {
460 |                     q.put(QueueElement {
461 |                         seq_id: 1,
462 |                         offset_in_shm_buf: 1,
463 |                         status: 1,
464 |                     })
465 |                     .unwrap();
466 |                 }
467 |             });
468 |         }
469 | 
470 |         while pop_count != 100000 {
471 |             match q.pop() {
472 |                 Ok(_) => pop_count += 1,
473 |                 Err(_) => {
474 |                     tokio::time::sleep(Duration::from_micros(1)).await;
475 |                 }
476 |             }
477 |         }
478 |     }
479 | 
480 |     fn create_queue(cap: u32) -> Queue {
481 |         let mem_size = QUEUE_HEADER_LENGTH + QUEUE_ELEMENT_LEN * cap as usize;
482 |         let mut mem: Vec<u8> = vec![0u8; mem_size];
483 |         let queue = Queue::create_from_bytes(mem.as_mut_ptr(), cap);
484 |         std::mem::forget(mem);
485 |         queue
486 |     }
487 | }
488 | 


--------------------------------------------------------------------------------
/src/session/config.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::time::Duration;
16 | 
17 | use crate::config::Config;
18 | 
19 | #[derive(Clone, Debug)]
20 | pub struct SessionManagerConfig {
21 |     config: Config,
22 |     session_num: usize,
23 |     stream_max_idle_time: Duration,
24 | }
25 | 
26 | impl SessionManagerConfig {
27 |     pub fn new() -> Self {
28 |         Self::default()
29 |     }
30 | 
31 |     pub fn config(&self) -> &Config {
32 |         &self.config
33 |     }
34 | 
35 |     pub fn config_mut(&mut self) -> &mut Config {
36 |         &mut self.config
37 |     }
38 | 
39 |     pub fn with_config(mut self, config: Config) -> Self {
40 |         self.config = config;
41 |         self
42 |     }
43 | 
44 |     pub fn session_num(&self) -> usize {
45 |         self.session_num
46 |     }
47 | 
48 |     pub fn with_session_num(mut self, session_num: usize) -> Self {
49 |         self.session_num = session_num;
50 |         self
51 |     }
52 | 
53 |     pub fn stream_max_idle_time(&self) -> Duration {
54 |         self.stream_max_idle_time
55 |     }
56 | 
57 |     pub fn with_stream_max_idle_time(mut self, stream_max_idle_time: Duration) -> Self {
58 |         self.stream_max_idle_time = stream_max_idle_time;
59 |         self
60 |     }
61 | }
62 | 
63 | impl Default for SessionManagerConfig {
64 |     fn default() -> Self {
65 |         Self {
66 |             config: Default::default(),
67 |             session_num: 1,
68 |             stream_max_idle_time: Duration::from_secs(30),
69 |         }
70 |     }
71 | }
72 | 


--------------------------------------------------------------------------------
/src/session/ext.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::os::fd::AsRawFd;
16 | 
17 | use volo::net::conn::ConnStream;
18 | 
19 | pub trait ConnStreamExt {
20 |     fn is_tcp(&self) -> bool;
21 |     fn as_raw_fd(&self) -> i32;
22 | }
23 | 
24 | impl ConnStreamExt for ConnStream {
25 |     #[inline]
26 |     fn is_tcp(&self) -> bool {
27 |         matches!(self, Self::Tcp(_))
28 |     }
29 | 
30 |     #[inline]
31 |     fn as_raw_fd(&self) -> i32 {
32 |         match self {
33 |             Self::Tcp(s) => s.as_raw_fd(),
34 |             #[cfg(target_family = "unix")]
35 |             Self::Unix(s) => s.as_raw_fd(),
36 |         }
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/src/session/manager.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::sync::{
 16 |     Arc,
 17 |     atomic::{AtomicUsize, Ordering},
 18 | };
 19 | 
 20 | use arc_swap::ArcSwap;
 21 | use tokio::sync::Notify;
 22 | use volo::net::Address;
 23 | 
 24 | use super::{Session, config::SessionManagerConfig};
 25 | use crate::{consts::StateType, error::Error, stream::Stream};
 26 | 
 27 | const SESSION_ROUND_ROBIN_THRESHOLD: usize = 32;
 28 | 
 29 | /// SessionManager provides an implementation similar to a connection pool, which provides a pair of
 30 | /// connections(stream) for communication between two processes. When `get_stream()` returns an
 31 | /// error, you may need to fallback according to your own scenario, such as fallback to a unix
 32 | /// domain socket.
 33 | ///
 34 | /// When a client process communicates with a server process, it is best practice to share a
 35 | /// SessionManager globally in the client process to create a stream.
 36 | ///
 37 | /// When the client needs to communicate with multiple server processes, a separate
 38 | /// SessionManager should be maintained for each different server. At the same time,
 39 | /// `queue_path` and `share_memory_path_prefix` need to be kept different.
 40 | #[derive(Clone, Debug)]
 41 | pub struct SessionManager {
 42 |     inner: Arc<SessionManagerInner>,
 43 | }
 44 | 
 45 | #[allow(dead_code)]
 46 | #[derive(Debug)]
 47 | struct SessionManagerInner {
 48 |     sm_config: SessionManagerConfig,
 49 |     addr: Address,
 50 |     sessions: Vec<ArcSwap<Session>>,
 51 |     count: AtomicUsize,
 52 |     epoch: u64,
 53 |     rand_id: u64,
 54 |     state: StateType,
 55 |     shutdown_tx: Arc<Notify>,
 56 | }
 57 | 
 58 | impl SessionManager {
 59 |     /// Create a new SessionManager.
 60 |     pub async fn new(
 61 |         mut sm_config: SessionManagerConfig,
 62 |         addr: impl Into<Address>,
 63 |     ) -> Result<Self, Error> {
 64 |         let mut sessions = Vec::with_capacity(sm_config.session_num());
 65 |         let addr = addr.into();
 66 | 
 67 |         for i in 0..sm_config.session_num() {
 68 |             let session = Session::client(i, 0, 0, &mut sm_config, &addr).await?;
 69 |             sessions.push(ArcSwap::from_pointee(session));
 70 |         }
 71 |         let shutdown = Arc::new(Notify::new());
 72 |         let sm = Self {
 73 |             inner: Arc::new(SessionManagerInner {
 74 |                 sm_config: sm_config.clone(),
 75 |                 addr,
 76 |                 sessions,
 77 |                 count: AtomicUsize::new(0),
 78 |                 epoch: 0,
 79 |                 rand_id: 0,
 80 |                 state: StateType::DefaultState,
 81 |                 shutdown_tx: shutdown.clone(),
 82 |             }),
 83 |         };
 84 |         for i in 0..sm_config.session_num() {
 85 |             tokio::spawn({
 86 |                 let shutdown_rx = shutdown.clone();
 87 |                 let sm = sm.clone();
 88 |                 async move { sm.rebuild_session(shutdown_rx, i).await }
 89 |             });
 90 |         }
 91 | 
 92 |         Ok(sm)
 93 |     }
 94 | 
 95 |     /// Return a shmipc's stream from stream pool.
 96 |     ///
 97 |     /// Every stream should explicitly call `put_back()` to return it to SessionManager for next
 98 |     /// time using, otherwise it will cause resource leak.
 99 |     pub fn get_stream(&self) -> Result<Stream, Error> {
100 |         let sessions = &self.inner.sessions;
101 |         if sessions.is_empty() {
102 |             return Err(Error::SessionUnhealthy);
103 |         }
104 |         let i = ((self.inner.count.fetch_add(1, Ordering::SeqCst) + 1)
105 |             / SESSION_ROUND_ROBIN_THRESHOLD)
106 |             % sessions.len();
107 |         sessions[i].load().get_or_open_stream(i)
108 |     }
109 | 
110 |     /// Return unused stream to stream pool for next time using.
111 |     pub async fn put_back(&self, stream: Stream) {
112 |         self.inner.sessions[stream.session_id()]
113 |             .load()
114 |             .put_or_close_stream(stream)
115 |             .await;
116 |     }
117 | 
118 |     /// Close all sessions in SessionManager.
119 |     pub async fn close(&self) {
120 |         self.inner.shutdown_tx.notify_waiters();
121 |         for s in &self.inner.sessions {
122 |             s.load().close().await;
123 |         }
124 |     }
125 | 
126 |     async fn rebuild_session(self, shutdown_rx: Arc<Notify>, i: usize) {
127 |         let mut shutdown_rx = Box::pin(shutdown_rx.notified());
128 | 
129 |         loop {
130 |             let session = self.inner.sessions[i].load();
131 |             tokio::select! {
132 |                 _ = session.shared.shutdown_notify.notified() => {
133 |                     loop {
134 |                         tokio::time::sleep(self.inner.sm_config.config().rebuild_interval).await;
135 | 
136 |                         match Session::client(i, 0, 0, &mut self.inner.sm_config.clone(), &self.inner.addr).await {
137 |                             Ok(new_session) => {
138 |                                 self.inner.sessions[i].store(Arc::new(new_session));
139 |                                 tracing::info!("rebuild session {i} success");
140 |                                 break;
141 |                             }
142 |                             Err(e) => {
143 |                                 tracing::error!("rebuild session {i} error: {:?}, retry after {:?}", e, self.inner.sm_config.config().rebuild_interval);
144 |                             }
145 |                         }
146 |                     }
147 |                 }
148 |                 _ = &mut shutdown_rx => {
149 |                     return;
150 |                 }
151 |             }
152 |         }
153 |     }
154 | }
155 | 


--------------------------------------------------------------------------------
/src/session/pool.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::{collections::VecDeque, sync::Mutex};
16 | 
17 | use crate::{Error, stream::Stream};
18 | 
19 | #[derive(Debug)]
20 | pub struct StreamPool {
21 |     streams: Mutex<VecDeque<Stream>>,
22 | }
23 | 
24 | impl StreamPool {
25 |     pub fn new(pool_capacity: usize) -> Self {
26 |         Self {
27 |             streams: Mutex::new(VecDeque::with_capacity(pool_capacity)),
28 |         }
29 |     }
30 | 
31 |     pub async fn push(&self, mut stream: Stream) -> Result<(), Error> {
32 |         {
33 |             let mut streams = self.streams.lock().unwrap();
34 |             if streams.len() < streams.capacity() {
35 |                 streams.push_back(stream);
36 |                 return Ok(());
37 |             }
38 |         }
39 |         stream.safe_close_notify();
40 |         _ = stream.close().await;
41 |         Err(Error::StreamPoolFull)
42 |     }
43 | 
44 |     pub fn pop(&self) -> Option<Stream> {
45 |         self.streams.lock().unwrap().pop_front()
46 |     }
47 | 
48 |     pub async fn close(&self) {
49 |         while let Some(mut s) = self.pop() {
50 |             s.safe_close_notify();
51 |             _ = s.close().await;
52 |         }
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/src/stats.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | use std::sync::atomic::AtomicU64;
16 | 
17 | #[derive(Debug, Default)]
18 | pub struct Stats {
19 |     #[allow(dead_code)]
20 |     pub alloc_shm_error_count: AtomicU64,
21 |     pub fallback_write_count: AtomicU64,
22 |     pub fallback_read_count: AtomicU64,
23 |     pub event_conn_error_count: AtomicU64,
24 |     pub queue_full_error_count: AtomicU64,
25 |     pub recv_polling_event_count: AtomicU64,
26 |     pub send_polling_event_count: AtomicU64,
27 |     pub out_flow_bytes: AtomicU64,
28 |     pub in_flow_bytes: AtomicU64,
29 | }
30 | 


--------------------------------------------------------------------------------
/src/util.rs:
--------------------------------------------------------------------------------
 1 | // Copyright 2025 CloudWeGo Authors
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | /// Determine whether to create new share memory based on the current remaining share memory size.
16 | ///
17 | /// Currently, this only works on the /dev/shm partition of the Linux system. Other systems return
18 | /// true by default (By the way, shmipc can run on Mac OS, but has not been performance tuned).
19 | ///
20 | /// When getting /dev/shm partition information fails, returns false and outputs a log.
21 | pub fn can_create_on_dev_shm(size: u64, path: &str) -> bool {
22 |     #[cfg(target_os = "linux")]
23 |     {
24 |         if path.contains("/dev/shm") {
25 |             match fs2::free_space("/dev/shm") {
26 |                 Ok(free) => return free >= size,
27 |                 Err(err) => {
28 |                     tracing::warn!(
29 |                         "could not read /dev/shm free size, can_create_on_dev_shm default return \
30 |                          true, err: {}",
31 |                         err
32 |                     );
33 |                     return false;
34 |                 }
35 |             }
36 |         }
37 |     }
38 |     true
39 | }
40 | 
41 | #[cfg(test)]
42 | mod tests {
43 |     use crate::util::can_create_on_dev_shm;
44 | 
45 |     #[test]
46 |     fn test_can_create_on_dev_shm() {
47 |         #[cfg(target_os = "linux")]
48 |         {
49 |             // just on /dev/shm, other always return true
50 |             assert!(can_create_on_dev_shm(u64::MAX, "sdffafds"));
51 |             let free = fs2::free_space("/dev/shm").unwrap();
52 |             assert!(can_create_on_dev_shm(free, "/dev/shm/xxx"));
53 |             assert!(!can_create_on_dev_shm(free + 1, "/dev/shm/yyy"));
54 |         }
55 |         #[cfg(target_os = "macos")]
56 |         {
57 |             // always return true
58 |             assert!(can_create_on_dev_shm(33333, "sdffafds"));
59 |         }
60 |     }
61 | }
62 | 


--------------------------------------------------------------------------------
/tests/test.rs:
--------------------------------------------------------------------------------
  1 | // Copyright 2025 CloudWeGo Authors
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | use std::{
 16 |     os::unix::net::SocketAddr,
 17 |     time::{Duration, Instant},
 18 | };
 19 | 
 20 | use shmipc::{
 21 |     AsyncReadShm, AsyncWriteShm, BufferReader, BufferSlice, Error, LinkedBuffer, Listener,
 22 |     SessionManager, SessionManagerConfig, Stream, config::SizePercentPair, consts::MemMapType,
 23 | };
 24 | 
 25 | #[tokio::test(flavor = "multi_thread")]
 26 | async fn test_ping_pong_by_shmipc() {
 27 |     let rand = rand::random::<u64>();
 28 |     let path = format!("/dev/shm/shmipc{}.sock", rand);
 29 |     let mut sm_config = benchmark_config();
 30 |     let size = 1 << 20;
 31 | 
 32 |     sm_config.config_mut().buffer_slice_sizes = vec![
 33 |         SizePercentPair {
 34 |             size: size + 256,
 35 |             percent: 70,
 36 |         },
 37 |         SizePercentPair {
 38 |             size: (16 << 10) + 256,
 39 |             percent: 20,
 40 |         },
 41 |         SizePercentPair {
 42 |             size: (64 << 10) + 256,
 43 |             percent: 10,
 44 |         },
 45 |     ];
 46 |     sm_config
 47 |         .config_mut()
 48 |         .share_memory_path_prefix
 49 |         .push_str(rand.to_string().as_str());
 50 |     sm_config = sm_config.with_session_num(1);
 51 |     let mut server = Listener::new(
 52 |         SocketAddr::from_pathname(path.clone()).unwrap(),
 53 |         sm_config.config().clone(),
 54 |     )
 55 |     .await
 56 |     .unwrap();
 57 |     let start = Instant::now();
 58 | 
 59 |     tokio_scoped::scope(|s| {
 60 |         s.spawn(async move {
 61 |             let mut stream = server.accept().await.unwrap().unwrap();
 62 |             must_read(&mut stream, size).await;
 63 |             stream.recv_buf().release_previous_read();
 64 |             must_write(&mut stream, size).await;
 65 |         });
 66 |         s.spawn(async move {
 67 |             let client = SessionManager::new(sm_config, SocketAddr::from_pathname(path).unwrap())
 68 |                 .await
 69 |                 .unwrap();
 70 |             let mut stream = client.get_stream().unwrap();
 71 |             must_write(&mut stream, size).await;
 72 |             must_read(&mut stream, size).await;
 73 |             stream.release_read_and_reuse();
 74 |             stream.close().await.unwrap();
 75 |         });
 76 |     });
 77 | 
 78 |     let elapsed: Duration = start.elapsed();
 79 |     println!("elapsed: {:?}", elapsed);
 80 | }
 81 | 
 82 | fn benchmark_config() -> SessionManagerConfig {
 83 |     let mut c = SessionManagerConfig::new();
 84 |     c.config_mut().queue_cap = 65536;
 85 |     c.config_mut().connection_write_timeout = std::time::Duration::from_secs(1);
 86 |     c.config_mut().share_memory_buffer_cap = 256 << 20;
 87 |     c.config_mut().mem_map_type = MemMapType::MemMapTypeMemFd;
 88 |     c
 89 | }
 90 | 
 91 | async fn must_write(s: &mut Stream, size: u32) {
 92 |     write_empty_buffer(s.send_buf(), size);
 93 |     loop {
 94 |         match s.flush(false).await {
 95 |             Err(e) => match e {
 96 |                 Error::QueueFull => {
 97 |                     tokio::time::sleep(std::time::Duration::from_micros(1)).await;
 98 |                     continue;
 99 |                 }
100 |                 _ => {
101 |                     panic!("must write err:{}", e);
102 |                 }
103 |             },
104 |             Ok(_) => {
105 |                 return;
106 |             }
107 |         }
108 |     }
109 | }
110 | 
111 | fn write_empty_buffer(l: &mut LinkedBuffer, size: u32) {
112 |     if size == 0 {
113 |         return;
114 |     }
115 |     let mut wrote = 0;
116 |     loop {
117 |         if l.slice_list().write_slice.is_none() {
118 |             l.alloc(size - wrote);
119 |             l.slice_list_mut().write_slice = l.slice_list().front_slice;
120 |         }
121 |         wrote += write_empty_slice(l.slice_list().write_mut().unwrap(), size - wrote);
122 |         if wrote < size {
123 |             if l.slice_list().write().unwrap().next().is_none() {
124 |                 l.alloc(size - wrote);
125 |             }
126 |             l.slice_list_mut().write_slice = l.slice_list().write().unwrap().next_slice;
127 |         } else {
128 |             break;
129 |         }
130 |     }
131 |     *l.len_mut() += size as usize;
132 | }
133 | 
134 | fn write_empty_slice(slice: &mut BufferSlice, size: u32) -> u32 {
135 |     slice.write_index += size as usize;
136 |     if slice.write_index > slice.cap as usize {
137 |         let wrote = slice.cap - (slice.write_index as u32 - size);
138 |         slice.write_index = slice.cap as usize;
139 |         return wrote;
140 |     }
141 |     size
142 | }
143 | 
144 | async fn must_read(s: &mut Stream, size: u32) -> bool {
145 |     match s.discard(size as usize).await {
146 |         Err(e) => match e {
147 |             Error::StreamClosed | Error::EndOfStream => false,
148 |             _ => {
149 |                 panic!("must read err:{}", e);
150 |             }
151 |         },
152 |         Ok(_) => true,
153 |     }
154 | }
155 | 


--------------------------------------------------------------------------------