├── .gitattributes ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .rustfmt.toml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── ci └── run_rustfmt.sh ├── examples ├── Cargo.toml ├── README.md ├── examples_gen │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── protos │ ├── box_it │ │ └── basic.proto │ ├── custom_type │ │ └── custom.proto │ ├── linked_list │ │ └── node.proto │ ├── multi │ │ ├── team │ │ │ └── svc.proto │ │ └── user │ │ │ └── svc.proto │ ├── non_optional │ │ └── basic.proto │ ├── nullable_field │ │ └── basic.proto │ ├── serde │ │ └── basic.proto │ ├── sso │ │ └── city.proto │ ├── user │ │ └── user.proto │ └── zero_copy │ │ └── basic.proto └── src │ ├── basic │ └── main.rs │ ├── box_it │ └── main.rs │ ├── custom_type │ └── main.rs │ ├── linked_list │ └── main.rs │ ├── multi │ └── main.rs │ ├── non_optional │ └── main.rs │ ├── nullable_field │ └── main.rs │ ├── serde │ └── main.rs │ ├── sso │ └── main.rs │ └── zero_copy │ └── main.rs ├── pb-jelly-gen ├── Cargo.toml ├── LICENSE ├── README.md ├── proto │ └── rust │ │ └── extensions.proto ├── regen_gen_protos.sh └── src │ ├── bin │ └── protoc-gen-jellyrust.rs │ ├── codegen.rs │ ├── generate.rs │ ├── lib.rs │ └── protos.rs ├── pb-jelly ├── Cargo.toml ├── LICENSE ├── README.md └── src │ ├── base_types.rs │ ├── buffer.rs │ ├── descriptor.rs │ ├── erased.rs │ ├── extensions.rs │ ├── helpers.rs │ ├── lib.rs │ ├── reflection.rs │ ├── tests │ └── mod.rs │ ├── varint.rs │ └── wire_format.rs └── pb-test ├── .gitignore ├── Cargo.toml ├── README.md ├── bin ├── pbtest.bin ├── pbtest2_version1enum.bin ├── pbtest2_version1oneof.bin ├── pbtest2_version1oneof_none_null.bin ├── pbtest2_version2enum.bin ├── pbtest2_version2oneof.bin ├── pbtest2_version2oneof_none_null.bin ├── pbtest3.bin ├── pbtest3_default_oneof.bin ├── pbtest3_err_if_default.bin ├── pbtest3_err_if_default_non_default.bin ├── pbtest3_missing_oneof.bin ├── pbtest3_repeated_err_if_default.bin ├── pbtest3_repeated_err_if_default_non_default.bin ├── pbtest3_version1.bin ├── pbtest3_version1enum.bin ├── pbtest3_version1oneof.bin ├── pbtest3_version1oneof_none_null.bin ├── pbtest3_version2.bin ├── pbtest3_version2enum.bin ├── pbtest3_version2oneof.bin ├── pbtest3_version2oneof_none_null.bin ├── pbtest_version1.bin └── pbtest_version2.bin ├── data ├── cities.json └── moby_dick.txt ├── gen └── pb-jelly │ ├── proto_google │ ├── Cargo.toml.expected │ └── src │ │ ├── lib.rs.expected │ │ └── protobuf │ │ ├── empty.rs.expected │ │ └── mod.rs.expected │ ├── proto_nopackage │ ├── Cargo.toml.expected │ └── src │ │ ├── lib.rs.expected │ │ └── proto_nopackage.rs.expected │ └── proto_pbtest │ ├── Cargo.toml.expected │ └── src │ ├── bench.rs.expected │ ├── extensions.rs.expected │ ├── lib.rs.expected │ ├── mod │ ├── mod.rs.expected │ └── struct.rs.expected │ ├── pbtest2.rs.expected │ ├── pbtest3.rs.expected │ └── servicepb.rs.expected ├── pb_test_gen ├── Cargo.toml └── src │ └── main.rs ├── proto └── packages │ ├── google │ └── protobuf │ │ └── empty.proto │ ├── nopackage.proto │ └── pbtest │ ├── bench.proto │ ├── extensions.proto │ ├── mod │ └── struct.proto │ ├── pbtest2.proto │ ├── pbtest3.proto │ └── servicepb.proto └── src ├── bench.rs ├── gen ├── README.md ├── mod.rs └── rust_protobuf │ ├── bench.rs │ ├── extensions.rs │ └── mod.rs ├── lib.rs ├── pbtest.rs └── verify_generated_files.rs /.gitattributes: -------------------------------------------------------------------------------- 1 | * text eol=lf 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | env: 4 | PROTOBUF_VER: 28.3 5 | 6 | # Run on git push, PR, or manually from the Actions tab 7 | on: [push, pull_request, workflow_dispatch, merge_group] 8 | 9 | jobs: 10 | rustfmt: 11 | name: Check formatting with rustfmt 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - run: rustup update nightly 16 | - run: rustup component add rustfmt --toolchain nightly 17 | - name: Run rustfmt 18 | run: rustup run nightly ci/run_rustfmt.sh 19 | 20 | pb-jelly-unit: 21 | name: pb-jelly unit tests 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v2 25 | - run: rustup update nightly 26 | - name: Run unit tests 27 | run: | 28 | cd pb-jelly 29 | rustup run nightly cargo test 30 | 31 | pb-jelly-gen-unit: 32 | name: pb-jelly-gen unit tests 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/checkout@v2 36 | - run: rustup update nightly 37 | - name: Run unit tests 38 | run: | 39 | cd pb-jelly-gen 40 | rustup run nightly cargo test 41 | 42 | pbtest: 43 | strategy: 44 | matrix: 45 | plat: [ubuntu-latest, macos-latest, windows-latest] 46 | name: pbtest integration test (${{matrix.plat}}) 47 | runs-on: ${{matrix.plat}} 48 | steps: 49 | - uses: actions/checkout@v2 50 | - run: rustup update nightly 51 | - name: Install Protoc 52 | uses: arduino/setup-protoc@v3 53 | with: 54 | version: ${{env.PROTOBUF_VER}} 55 | repo-token: ${{ secrets.GITHUB_TOKEN }} 56 | - name: Run integration test 57 | env: 58 | VALIDATE: 1 59 | run: | 60 | cd pb-test/pb_test_gen 61 | rustup run nightly cargo run 62 | cd .. 63 | rustup run nightly cargo test 64 | 65 | examples: 66 | name: examples 67 | runs-on: ubuntu-latest 68 | steps: 69 | - uses: actions/checkout@v2 70 | - run: rustup update nightly 71 | - name: Install Protoc 72 | uses: arduino/setup-protoc@v3 73 | with: 74 | version: ${{env.PROTOBUF_VER}} 75 | repo-token: ${{ secrets.GITHUB_TOKEN }} 76 | - name: Run tests 77 | run: | 78 | cd examples/examples_gen 79 | rustup run nightly cargo run 80 | cd .. 81 | rustup run nightly cargo test 82 | 83 | benchmarks: 84 | name: benchmarks 85 | runs-on: ubuntu-latest 86 | steps: 87 | - uses: actions/checkout@v2 88 | - run: rustup update nightly 89 | - name: Install Protoc 90 | uses: arduino/setup-protoc@v3 91 | with: 92 | version: ${{env.PROTOBUF_VER}} 93 | repo-token: ${{ secrets.GITHUB_TOKEN }} 94 | - name: Run benchmark 95 | run: | 96 | cd pb-test/pb_test_gen 97 | rustup run nightly cargo run --features=bench_prost,bench_rust_protobuf 98 | cd .. 99 | rustup run nightly cargo bench bench --features=bench_prost,bench_rust_protobuf 100 | 101 | clippy_gen: 102 | name: clippy gen 103 | runs-on: ubuntu-latest 104 | steps: 105 | - uses: actions/checkout@v2 106 | - run: rustup update nightly 107 | - run: rustup component add clippy --toolchain nightly 108 | - name: Install Protoc 109 | uses: arduino/setup-protoc@v3 110 | with: 111 | version: ${{env.PROTOBUF_VER}} 112 | repo-token: ${{ secrets.GITHUB_TOKEN }} 113 | - name: Run clippy 114 | env: 115 | RUSTFLAGS: "-D warnings" 116 | run: | 117 | cd pb-test/pb_test_gen 118 | rustup run nightly cargo run 119 | cd .. 120 | rustup run nightly cargo clippy -p proto_pbtest 121 | 122 | gen_gen: 123 | name: Generate pb-jelly-gen/src/protos.rs 124 | runs-on: ubuntu-latest 125 | steps: 126 | - uses: actions/checkout@v2 127 | - run: rustup update nightly 128 | - run: rustup component add clippy --toolchain nightly 129 | - name: Install Protoc 130 | uses: arduino/setup-protoc@v3 131 | with: 132 | version: ${{env.PROTOBUF_VER}} 133 | repo-token: ${{ secrets.GITHUB_TOKEN }} 134 | - name: Generate protos 135 | run: | 136 | cd pb-jelly-gen 137 | rustup run nightly bash regen_gen_protos.sh 138 | git diff --exit-code 139 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | .*.swp 4 | generated/ 5 | **/gen/**/*.rs 6 | **/gen/**/*.toml 7 | .DS_Store 8 | __pycache__/ 9 | *~ 10 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | comment_width = 120 2 | max_width = 120 3 | edition = "2018" 4 | 5 | # Disabled while rustfmt chokes on non-code code blocks in comments 6 | wrap_comments = true 7 | 8 | # Some lines are just over 120 chars... deal with it. 9 | error_on_line_overflow = false 10 | 11 | # Always format imports like this: 12 | # use foo::{ 13 | # Bar, 14 | # Baz, 15 | # }; 16 | imports_indent = "Block" 17 | imports_layout = "Vertical" 18 | reorder_imports = true 19 | 20 | match_block_trailing_comma = true 21 | 22 | # We prefer it already so just do it here 23 | use_try_shorthand = true 24 | use_field_init_shorthand = true 25 | 26 | imports_granularity = "Module" 27 | group_imports = "StdExternalCrate" 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Unreleased 2 | ... everything has been released! 3 | 4 | # 0.0.17 5 | ### Nov 20, 2024 6 | * Add read-only support for proto2 extension fields (#163, #170) 7 | * Remove Python dependency by rewriting pb-jelly-gen in Rust (#164) 8 | * Make pb-jelly-gen Rust API optional (#166) 9 | * This is a breaking change: if you were using pb-jelly-gen as a library, turn on the `generate` feature flag 10 | * Remove serde impls for Fixed/Signed types (#168) 11 | * Various optimizations around the generated code (#173, #175) 12 | 13 | # 0.0.16 14 | ### Jan 18, 2024 15 | * Replace ctype=CORD with rust.blob extension field (#160) 16 | 17 | # 0.0.15 18 | ### Dec 14, 2023 19 | * Improve handling of recursive types (#157) 20 | * In particular, recursive types now implement `Eq`/`Hash`/etc. if possible. 21 | * Reduce the amount of generated code (#158) 22 | 23 | # 0.0.14 24 | ### Oct 27, 2023 25 | * Implement box_it for oneof fields (#150) 26 | * Use r# syntax for keywords instead of appending _ (#153) 27 | * This is a breaking change. Fields with names like `type` used to be generated as `type_`. 28 | Now, you should refer to them using a raw identifier: `r#type`. 29 | 30 | # 0.0.13 31 | ### Oct 12, 2023 32 | * Add `rustdoc::` prefix to `#[allow(broken_intra_doc_links)]` (#148) 33 | 34 | # 0.0.12 35 | ### May 9, 2023 36 | * Add a `(rust.sso)` option to opt into using `compact_str` for strings (#141) 37 | * Support `optional` fields in proto3 (#143) 38 | * Optimize varint encoding (#144) 39 | * Pass through UTF-8 decode errors when deserializing strings 40 | 41 | # 0.0.11 42 | ### December 15, 2021 43 | * Avoid unnecessary `return` statements in some generated code. (#136) 44 | * Avoid extra logic for oneofs with only 1 possible value. (#133) 45 | 46 | # 0.0.10 47 | ### November 30, 2021 48 | * Simplify the zerocopy implementation. (#127) 49 | * `PbBuffer` has been reworked to untie it from `PbBufferReader`. 50 | * `copy_from_reader` replaces `from_reader` and allows a `PbBuffer` to be constructed, by copying, from any `Buf`. Implementations can still opt out by returning `Err`. 51 | * `copy_to_writer` replaces `into_reader`. Callers that were using `into_reader` to call `Message::deserialize` should instead construct their desired `PbBufferReader` directly (e.g. `Cursor`). 52 | * `PbBufferReader::as_buffer` is renamed to `read_buffer`, and provided implementations fall back to copying `Lazy` fields by default (instead of returning an error). 53 | * Implement basic support for the field reflection API. (#121) 54 | * `MessageDescriptor` has been reworked to be a `struct` rather than a `trait`, and is returned from `Message::descriptor` rather than implemented on a `Message`. 55 | * `MessageDescriptor` has been augmented to return information about the fields and oneofs in the `Message`. 56 | * A `Reflection` trait has been added to provide more dynamic access to proto fields. 57 | * This is automatically generated by `pb-jelly-gen` for all generated protos, but any manually implemented `Message` that is used in a generated `Message` will also need to manually implement `Reflection`. 58 | * Example usage can be seen in `pb-test/src/pbtest.rs` in `all_fields_reflection3`. 59 | 60 | # 0.0.9 61 | ### November 2, 2021 62 | * Simplify Option -> &str conversion in pb-jelly-gen (clippy warning) 63 | 64 | # 0.0.8 65 | ### October 25, 2021 66 | * Move `(gogoproto.nullable)` option to `(rust.nullable_field)`, removing the dependency on gogoproto 67 | * Support running with any version of protoc (by dynamically generating `extensions_pb2.py` in a venv) 68 | * Use github CI for tests, rustfmt, black, mypy --strict 69 | * Support windows for codegen 70 | * Rename master branch to main 71 | * Use a setup.cfg to install protoc-gen-rust to avoid need for --plugin flag. Simplifies manual usage process, especially on windows (no need for codegen.bat) 72 | * Remove need for requirements.txt - by using setup.cfg `install_requires` 73 | 74 | # 0.0.7 75 | ### May 5, 2021 76 | * Add `(rust.closed_enum)` option to only generate closed enums 77 | 78 | # 0.0.6 79 | ### April 25, 2021 80 | * Bump `bytes` to `1.0` 81 | * Bump `byteorder` to `1.4` 82 | 83 | # 0.0.5 84 | ### November 20, 2020 85 | * Add Windows support to `pb-jelly-gen` 86 | * Add Windows support to our CI 87 | * Allows proto files at the root of proto path 88 | 89 | #### Bugs 90 | * Fixed a bug around non-optional boxed messages 91 | 92 | # 0.0.4 93 | ### October 21, 2020 94 | * Drop python2 support (remove six and inline type annotations) 95 | * Only generate crate level attributes in lib.rs (vs every module) 96 | * Better error message if python-protobuf version is too low for codegen plugin 97 | * Bump protobuf in requirements to 3.13.0 98 | * Requires upgrading to protobuf 3.13.0 99 | 100 | #### Bugs 101 | * Fixed issue where sometimes in codegen field type was incorrect when using `err_if_default_or_unknown` 102 | 103 | # 0.0.3 104 | ### September 21, 2020 105 | * Forgot to bump the version of `pb-jelly` that the codegen script uses 106 | 107 | # v0.0.2 108 | ### September 19, 2020 109 | * Use the `license` field instead of the `license-file` field in the Cargo.toml of `pb-jelly` and `pb-jelly-gen`. 110 | * Note: The License is still the same, the update is purely for better metadata from [crates.io](https://crates.io/crates/pb-jelly) 111 | * Warn on `rust_2018_idioms` closes [#45](https://github.com/dropbox/pb-jelly/issues/45) 112 | * A few changes related solely to re-integrating `pb-jelly` into the Dropbox codebase. 113 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2020 Dropbox, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `pb-jelly` 2 |

by

3 | 4 | 5 | `pb-jelly` is a [protobuf](https://developers.google.com/protocol-buffers) code generation framework for the [Rust language](https://www.rust-lang.org/) developed at Dropbox. 6 | 7 | 8 | ### History 9 | 10 | This implementation was initially written in 2016 to satisfy the need of shuffling large amount 11 | of bytes in [Dropbox's Storage System (Magic Pocket)](https://dropbox.tech/infrastructure/inside-the-magic-pocket). 12 | Previously, we were using [`rust-protobuf`](https://github.com/stepancheg/rust-protobuf) (and therefore generated APIs are exactly 13 | the same to make migration easy) but serializing Rust structs to proto messages, and then serializing them again in 14 | our RPC layer, meant multiple copies (and same thing in reverse on parsing stack). Taking control of this 15 | implementation and integrating it in our RPC stack end-to-end helped avoid these extra copies. 16 | 17 | Over the years, the implementation has grown and matured and is currently used in several parts of Dropbox, including 18 | our [Sync Engine](https://dropbox.tech/infrastructure/rewriting-the-heart-of-our-sync-engine), and the aforementioned [Magic Pocket](https://dropbox.tech/infrastructure/inside-the-magic-pocket). 19 | 20 | Other implementations exist in the Rust ecosystem (e.g. [`prost`](https://github.com/danburkert/prost) and [`rust-protobuf`](https://github.com/stepancheg/rust-protobuf)), we wanted to share ours as well. 21 | 22 |
23 | 24 | [![Crates.io](https://img.shields.io/crates/v/pb-jelly)](https://crates.io/crates/pb-jelly) [![Documentation](https://docs.rs/pb-jelly/badge.svg)](https://docs.rs/pb-jelly) [![Crates.io](https://img.shields.io/crates/l/pb-jelly)](LICENSE) [![Build Status](https://github.com/dropbox/pb-jelly/workflows/CI/badge.svg)](https://github.com/dropbox/pb-jelly/actions?query=branch%3Amain) 25 | 26 | 27 | ## Features 28 | - Functional "Rust-minded" proto extensions, e.g. `[(rust.box_it)=true]` 29 | - Scalable - Generates separate crates per module, with option for crate-per-directory 30 | - Autogenerates `Cargo.toml` 31 | - Support for [`Serde`](https://serde.rs/) (not compliant with the JSON protobuf specification) 32 | - Zero-copy deserialization with [`Bytes`](https://docs.rs/bytes/0.5.6/bytes/) via a proto extension `[(rust.zero_copy)=true]` 33 | - Automatically boxes messages if it finds a recursive message definition 34 | - Retains comments on proto fields 35 | - Supports `proto2` and `proto3` syntaxes 36 | 37 |
38 | 39 | ### Extensions 40 | 41 | | Extension | Description | Type | Example | 42 | |:---------------------------------------:|:----------------------------------------------------------------------------------------------------------:|-------|:-------:| 43 | | `(rust.zero_copy)=true` | Generates field type of `Lazy` for proto `bytes` fields to support zero-copy deserialization | Field | [`zero_copy`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 44 | | `(rust.box_it)=true` | Generates a `Box` field type | Field | [`box_it`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 45 | | `(rust.type)="type"` | Generates a custom field type | Field | [`custom_type`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 46 | | `(rust.preserve_unrecognized)=true` | Preserves unrecognized proto fields into an `_unrecognized` struct field | Field | `TODO` | 47 | | `(rust.nullable_field)=false` | Generates non-nullable fields types | Field | `TODO` | 48 | | `(rust.nullable)=false` | Generates oneofs as non-nullable (fail on deserialization) | Oneof | [`non_optional`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 49 | | `(rust.err_if_default_or_unknown)=true` | Generates enums as non-zeroable (fail on deserialization) | Enum | [`non_optional`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 50 | | `(rust.closed_enum)=true` | Generates only a "closed" enum which will fail deserialization for unknown values, but is easier to work with in Rust | Enum | `TODO` | 51 | | `(rust.serde_derive)=true` | Generates serde serializable/deserializable messages | File | [`serde`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) | 52 | 53 |
54 | 55 | ## Using `pb-jelly` in your project 56 | Multiple crates, multiple languages, my oh my! 57 | 58 | ### Essential Crates 59 | There are only two crates you'll need: `pb-jelly` and `pb-jelly-gen`.
60 | 61 | ##### `pb-jelly` 62 | Contains all of the important traits and structs that power our generated code, e.g. `Message` and `Lazy`. Include this as a dependency, e.g. 63 | ``` 64 | [dependencies] 65 | pb-jelly = "0.0.17" 66 | ``` 67 | 68 | ##### `pb-jelly-gen` 69 | 70 | A framework for generating Rust structs and implementations for `.proto` files. 71 | In order to use pb-jelly, you need to add the pb-jelly-gen as a plugin to your protoc invocation. 72 | 73 | We added some code here to handle the protoc invocation if you choose to use it. 74 | You'll need to add a generation crate (see `examples_gen` for an example) 75 | Include `pb-jelly-gen` as a dependency of your generation crate, and `cargo run` to invoke protoc for you. 76 | ``` 77 | [dependencies] 78 | pb-jelly-gen = "0.0.17" 79 | ``` 80 | 81 | Eventually, we hope to eliminate the need for a generation crate, and simply have generation occur 82 | inside a build.rs with `pb-jelly-gen` as a build dependency. However https://github.com/rust-lang/cargo/issues/8709 83 | must be resolved first. 84 | 85 | Note that you can always invoke protoc on your own (for example if you are already doing so to generate for multiple languages) 86 | with `--rust_out=codegen.py` as a plugin for rust. 87 | 88 | ### Generating Rust Code 89 | 1. Install `protoc`, the protobuf compiler. 90 | - See [the upstream project](https://github.com/protocolbuffers/protobuf). Precompiled binaries can be found at their [releases page](https://github.com/protocolbuffers/protobuf/releases). 91 | - On macOS, `protoc` can be installed via Homebrew: `brew install protobuf`. 92 | 93 | #### To generate with pb-jelly-gen 94 | 3. Create an inner (build-step) crate which depends on pb-jelly-gen. [Example](https://github.com/dropbox/pb-jelly/tree/main/examples/examples_gen) 95 | 4. `cargo run` in the directory of the inner generation crate 96 | 97 | #### To generate manually with protoc 98 | 1. `cargo build` in `pb-jelly-gen` 99 | 2. `protoc --plugin=protoc-gen-jellyrust=pb-jelly-gen/target/debug/protoc-gen-jellyrust --jellyrust_out=generated/ input.proto` 100 | 101 | ## Example 102 | 103 | Take a look at the [`examples`](https://github.com/dropbox/pb-jelly/tree/main/examples/src) crate to see how we leverage `pb-jelly-gen` and `build.rs` to get started using protobufs in Rust! 104 | 105 |
106 | 107 | #### Non-essential Crates 108 | - `pb-test` contains integration tests and benchmarks. You don't need to worry about this one unless you want to contribute to this repository! 109 | - `examples` contains some examples to help you get started 110 | 111 |
112 | 113 | ### A Note On Scalability 📝 114 | We mention "scalabilty" as a feature, what does that mean? We take an opinionated stance that every module should be a crate, as opposed to generating Rust files 1:1 with proto files. We take this stance because `rustc` is parallel *across* crates, but not yet totally parallel *within* a crate. When we had all of our generated Rust code in a single crate, it was often that single crate that took the longest to compile. The solution to these long compile times, was creating many crates! 115 | 116 |
117 | 118 | ### The Name 🌠 119 | 120 | pb-jelly is a shoutout to the jellyfish known for its [highly efficient locomotion](https://en.wikipedia.org/wiki/Jellyfish). 121 | This library is capable of highly efficient locomotion of deserialized data. Also a shoutout to ability of the jellyfish 122 | to have [substantial increases in population](https://en.wikipedia.org/wiki/Jellyfish_bloom). This library handles generating 123 | a very large number of proto modules with complex dependencies, by generating to multiple crates. 124 | 125 | We also like [the popular sandwich](https://en.wikipedia.org/wiki/Peanut_butter_and_jelly_sandwich). 126 | 127 | # Contributing 128 | 129 | First, contributions are __greatly__ appreciated and highly encouraged. For legal reasons all outside 130 | contributors must agree to [Dropbox's CLA](https://opensource.dropbox.com/cla/). Thank you for 131 | your understanding. 132 | 133 | 134 |
135 | 136 | --- 137 | 138 | # Upcoming 139 | Some of the features here require additional tooling to be useful, which are not yet public. 140 | - Spec.toml is a stripped down templated Cargo.toml - which you can script convert into 141 | Cargo.toml in order to get consistent dependency versions in a multi-crate project. 142 | Currently, the script to convert Spec.toml -> Cargo.toml isn't yet available 143 | 144 | Closed structs with public fields 145 | - Adding fields to a proto file will lead to compiler errors. This can be a benefit in that it allows the 146 | compiler to identify all callsites that may need to be visited. However, it can make updating protos with 147 | many callsites a bit tedious. We opted to go this route to make it easier to add a new field and update 148 | all callsites with assistance from the compiler. 149 | 150 | Service Generation 151 | - Generating stubs for gPRC clients and servers 152 | 153 | # Running the `pbtest` unit tests 154 | 155 | 1. Clone Repo. 156 | 2. Install Dependencies / Testing Dependencies. Use the appropriate package manager for your system. 157 | - protoc - part of Google's [protobuf tools](https://github.com/protocolbuffers/protobuf/) 158 | - macos: `brew install protobuf` 159 | - Linux (Fedora/CentOS/RHEL): `dnf install protobuf protobuf-devel` 160 | - Linux (Ubuntu): `apt install protobuf-compiler` 161 | 3. **pb-jelly** currently uses an experimental test framework that requires a nightly build of rust. 162 | - `rustup default nightly` 163 | 4. `cd pb-test` 164 | 5. `( cd pb_test_gen ; cargo run ) ; cargo test` 165 | 166 | 167 | ## Contributors 168 | 169 | ### Dropboxers [incl former] 170 | - [@nipunn1313](https://github.com/nipunn1313) 171 | - [@rajatgoel](https://github.com/rajatgoel) 172 | - [@ParkMyCar](https://github.com/ParkMyCar) 173 | - [@rbtying](https://github.com/rbtying) 174 | - [@goffrie](https://github.com/goffrie) 175 | - [@euroelessar](https://github.com/euroelessar) 176 | - [@bradenaw](https://github.com/bradenaw) 177 | - [@glaysche2](https://github.com/glaysche2) 178 | - [@jiayixu](https://github.com/jiayixu) 179 | - [@dyv](https://github.com/dyv) 180 | - [@joshuawarner32](https://github.com/joshuawarner32) 181 | - [@peterlvilim](https://github.com/peterlvilim) 182 | - [@ddeville](https://github.com/ddeville) 183 | - [@isho](https://github.com/isho) 184 | - [@benjaminp](https://github.com/benjaminp) 185 | - [@grahamking](https://github.com/grahamking) 186 | - [@cyang1](https://github.com/cyang1) 187 | 188 | ### Non-Dropbox 189 | - [@RSMuthu](https://github.com/RSMuthu) 190 | 191 | ## Similar Projects 192 | [`rust-protobuf`](https://github.com/stepancheg/rust-protobuf) - Rust implementation of Google protocol buffers
193 | [`prost`](https://github.com/danburkert/prost) - PROST! a Protocol Buffers implementation for the Rust Language
194 | [`quick-protobuf`](https://github.com/tafia/quick-protobuf) - A rust implementation of protobuf parser
195 | [`serde-protobuf`](https://github.com/dflemstr/serde-protobuf)
196 | [`protokit`](https://github.com/semtexzv/protokit) 197 | -------------------------------------------------------------------------------- /ci/run_rustfmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | cargo --version 4 | cargo fmt --version 5 | for d in examples pb-jelly pb-jelly-gen pb-test; do 6 | pushd $d 7 | cargo fmt -- --check 8 | popd 9 | done 10 | -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "examples" 3 | version = "0.1.0" 4 | authors = ["Parker Timmerman "] 5 | edition = "2018" 6 | publish = false 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | bytes = "1.0" 12 | compact_str = "0.5" 13 | pb-jelly = "0.0.17" 14 | proto_box_it = { path = "gen/rust/proto/proto_box_it" } 15 | proto_custom_type = { path = "gen/rust/proto/proto_custom_type" } 16 | proto_linked_list = { path = "gen/rust/proto/proto_linked_list" } 17 | proto_multi = { path = "gen/rust/proto/proto_multi" } 18 | proto_non_optional = { path = "gen/rust/proto/proto_non_optional" } 19 | proto_nullable_field = { path = "gen/rust/proto/proto_nullable_field" } 20 | proto_serde = { path = "gen/rust/proto/proto_serde" } 21 | proto_sso = { path = "gen/rust/proto/proto_sso" } 22 | proto_user = { path = "gen/rust/proto/proto_user" } 23 | proto_zero_copy = { path = "gen/rust/proto/proto_zero_copy"} 24 | serde_json = "1" 25 | 26 | 27 | [[bin]] 28 | name = "basic" 29 | path = "src/basic/main.rs" 30 | 31 | [[bin]] 32 | name = "box_it" 33 | path = "src/box_it/main.rs" 34 | 35 | [[bin]] 36 | name = "custom_type" 37 | path = "src/custom_type/main.rs" 38 | 39 | [[bin]] 40 | name = "linked_list" 41 | path = "src/linked_list/main.rs" 42 | 43 | [[bin]] 44 | name = "multi" 45 | path = "src/multi/main.rs" 46 | 47 | [[bin]] 48 | name = "non_optional" 49 | path = "src/non_optional/main.rs" 50 | 51 | [[bin]] 52 | name = "nullable_field" 53 | path = "src/nullable_field/main.rs" 54 | 55 | [[bin]] 56 | name = "serde" 57 | path = "src/serde/main.rs" 58 | 59 | [[bin]] 60 | name = "sso" 61 | path = "src/sso/main.rs" 62 | 63 | [[bin]] 64 | name = "zero_copy" 65 | path = "src/zero_copy/main.rs" 66 | 67 | [patch.crates-io] 68 | pb-jelly = { path = "../pb-jelly" } 69 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # examples 2 | 3 | In order to run examples, first you must generate protos. 4 | ``` 5 | cd examples_gen 6 | cargo run 7 | ``` 8 | 9 | ### basic 10 | How to use `protobuf-rs` in the most basic sense. Creating, serializing, and de-serializing a message. 11 | 12 | ### box_it 13 | Using the `(rust.box_it)` extension to box up fields of your proto message. 14 | 15 | ### custom_type 16 | Defining custom Rust types for a field. 17 | 18 | ### linked_list 19 | Self referential messages automatically [`Box`]("https://doc.rust-lang.org/std/boxed/struct.Box.html) the recursive type. 20 | 21 | ### multi 22 | How directory structure of protos effects the module structure of the generated Rust crate. 23 | 24 | ### non_optional 25 | Making non-optional `enum` fields, and `oneof`s. 26 | 27 | ### serde 28 | All messages defined in a given `.proto` file derive [`Serialize`]("https://docs.rs/serde/1.0.114/serde/trait.Serialize.html") and [`Deserialize`]("https://docs.rs/serde/1.0.114/serde/trait.Deserialize.html"). 29 | 30 | ### zero_copy 31 | Fields of type `bytes` in proto messages map to a Rust type of `pb::Lazy` to enable zero-copy deserialization 32 | -------------------------------------------------------------------------------- /examples/examples_gen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "examples_gen" 3 | version = "0.1.0" 4 | authors = ["Parker Timmerman "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | #pb-jelly-gen = "0.0.17" # If copying this example - use this 10 | pb-jelly-gen = { path = "../../pb-jelly-gen" } 11 | 12 | [patch.crates-io] 13 | pb-jelly = { path = "../../pb-jelly" } 14 | -------------------------------------------------------------------------------- /examples/examples_gen/src/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly_gen::GenProtos; 2 | 3 | fn main() -> std::io::Result<()> { 4 | GenProtos::builder() 5 | .out_path("../gen/rust/proto") 6 | .src_path("../protos") 7 | .include_path("../includes") 8 | .cleanup_out_path(true) 9 | .gen_protos() 10 | .expect("Failed to generate protos"); 11 | 12 | Ok(()) 13 | } 14 | -------------------------------------------------------------------------------- /examples/protos/box_it/basic.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package basic; 3 | 4 | import "rust/extensions.proto"; 5 | 6 | message BoxedMessage { 7 | string name = 1; 8 | } 9 | 10 | message OuterMessage { 11 | BoxedMessage optional_msg = 1 [(rust.box_it)=true]; 12 | BoxedMessage msg = 2 [(rust.box_it)=true, (rust.nullable_field)=false]; 13 | string other = 3; 14 | } 15 | -------------------------------------------------------------------------------- /examples/protos/custom_type/custom.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package custom; 3 | 4 | import "rust/extensions.proto"; 5 | 6 | message UserId { 7 | // `rust.type` is more or less a string replacement 8 | bytes id = 1 [(rust.type)="[u8; 8]"]; 9 | 10 | // Note: If your bytes has a defined size, using `rust.type` could be advantageous because 11 | // your generated code will include a statically sized array, as opposed to a dynamically 12 | // sized `Vec` that `bytes` typically maps to. 13 | } 14 | 15 | message UserRequest { 16 | UserId user_id = 1; 17 | } -------------------------------------------------------------------------------- /examples/protos/linked_list/node.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package linked_list; 3 | 4 | message Node { 5 | string content = 1; 6 | // This self-referential node will automatically be put in a Box 7 | Node next = 2; 8 | } 9 | -------------------------------------------------------------------------------- /examples/protos/multi/team/svc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package team; 3 | 4 | message TeamRequest { 5 | uint64 team_id = 1; 6 | } 7 | -------------------------------------------------------------------------------- /examples/protos/multi/user/svc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package user; 3 | 4 | message UserRequest { 5 | uint64 user_id = 1; 6 | } 7 | -------------------------------------------------------------------------------- /examples/protos/non_optional/basic.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package basic; 3 | 4 | // rust/extensions.proto is included by default 5 | import "rust/extensions.proto"; 6 | 7 | enum City { 8 | // In the generated Rust code, `enum City` won't have `INVALID` as a variant. 9 | option (rust.err_if_default_or_unknown) = true; 10 | INVALID = 0; 11 | NEW_YORK = 1; 12 | SAN_FRANCISCO = 2; 13 | } 14 | 15 | message PlaneTicket { 16 | string passenger = 1; 17 | City city = 2; 18 | oneof airport { 19 | // This makes the `airport` field required 20 | option (rust.nullable) = false; 21 | string name = 3; 22 | uint32 id = 4; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/protos/nullable_field/basic.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package basic; 3 | 4 | import "rust/extensions.proto"; 5 | 6 | message Inner { int32 x = 1; } 7 | message PlainMessage { 8 | Inner field1 = 1; 9 | Inner field2 = 2; 10 | Inner field3 = 3; 11 | } 12 | message NonNullableMessage { 13 | Inner field1_nonnullable = 1 [(rust.nullable_field)=false]; 14 | Inner field2 = 2 [(rust.nullable_field)=true]; 15 | Inner field3 = 3; // Messages default to nullable 16 | } 17 | -------------------------------------------------------------------------------- /examples/protos/serde/basic.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package basic; 3 | 4 | // rust/extensions.proto is included by default 5 | import "rust/extensions.proto"; 6 | 7 | option (rust.serde_derive) = true; 8 | 9 | message NewYorkCity { 10 | uint64 num_windows = 1; 11 | string neighborhood = 2 [(rust.sso)=true]; 12 | } 13 | -------------------------------------------------------------------------------- /examples/protos/sso/city.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package city; 3 | 4 | // rust/extensions.proto is included by default 5 | import "rust/extensions.proto"; 6 | 7 | message City { 8 | string name = 1 [(rust.sso)=true]; 9 | double latitude = 2; 10 | double longitude = 3; 11 | string population = 4 [(rust.sso)=true]; 12 | string rank = 5 [(rust.sso)=true]; 13 | string state = 6 [(rust.sso)=true]; 14 | } 15 | -------------------------------------------------------------------------------- /examples/protos/user/user.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package user; 3 | 4 | message HelloUser { 5 | string name = 1; 6 | } 7 | -------------------------------------------------------------------------------- /examples/protos/zero_copy/basic.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package basic; 3 | 4 | // rust/extensions.proto is included by default 5 | import "rust/extensions.proto"; 6 | 7 | message BytesMessage { 8 | optional bytes data = 1 [(rust.zero_copy)=true]; 9 | optional string name = 2; 10 | } 11 | -------------------------------------------------------------------------------- /examples/src/basic/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | use proto_user::user::HelloUser; 3 | 4 | fn main() -> std::io::Result<()> { 5 | // Say hello! 6 | let msg = HelloUser { 7 | name: "Parker".to_string(), 8 | }; 9 | 10 | // Serialize our message to bytes 11 | let se_msg = msg.serialize_to_vec(); 12 | println!("Bytes 0x{:X?}", se_msg); 13 | 14 | // De-serialize our message back to a Rust struct 15 | let de_msg: HelloUser = Message::deserialize_from_slice(&se_msg[..])?; 16 | 17 | // Pretty print! 18 | println!("Message: {:#?}", de_msg); 19 | 20 | // Assert our two messages are the same 21 | assert_eq!(msg, de_msg); 22 | 23 | Ok(()) 24 | } 25 | -------------------------------------------------------------------------------- /examples/src/box_it/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | use proto_box_it::basic::{ 3 | BoxedMessage, 4 | OuterMessage, 5 | }; 6 | 7 | fn main() -> std::io::Result<()> { 8 | // Create our messages 9 | let maybe_msg = BoxedMessage { 10 | name: "Paper".to_owned(), 11 | }; 12 | let msg = BoxedMessage { 13 | name: "Passwords".to_owned(), 14 | }; 15 | let msg = OuterMessage { 16 | optional_msg: Some(Box::new(maybe_msg)), 17 | msg: Box::new(msg), 18 | other: "Dropbox".to_owned(), 19 | }; 20 | 21 | // Serialize our message 22 | let se_msg = msg.serialize_to_vec(); 23 | 24 | // Deserialize our message 25 | let de_msg: OuterMessage = Message::deserialize_from_slice(&se_msg[..])?; 26 | 27 | // Grab our inner box 28 | let de_box_msg: Box = de_msg.optional_msg.unwrap(); 29 | 30 | // Print our message! 31 | println!("{} {}", de_msg.other, de_box_msg.name); 32 | 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /examples/src/custom_type/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | use proto_custom_type::custom::{ 3 | UserId, 4 | UserRequest, 5 | }; 6 | 7 | fn main() -> std::io::Result<()> { 8 | let req = UserRequest { 9 | user_id: Some(UserId { 10 | id: [0, 1, 2, 3, 4, 5, 6, 7], 11 | }), 12 | }; 13 | let bytes = req.serialize_to_vec(); 14 | let req_de: UserRequest = Message::deserialize_from_slice(&bytes[..])?; 15 | 16 | // Our custom type! 17 | let user_id: [u8; 8] = req_de.user_id.unwrap().id; 18 | 19 | println!("Our 8 byte UserId: {:X?}", user_id); 20 | 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /examples/src/linked_list/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | use proto_linked_list::node::Node; 3 | 4 | fn main() -> std::io::Result<()> { 5 | // Creat our head node 6 | let mut head = Node { 7 | content: "Hello".to_owned(), 8 | next: None, 9 | }; 10 | // Create a second node 11 | let next_node = Node { 12 | content: "World".to_owned(), 13 | next: None, 14 | }; 15 | // Set next_node to be head.next 16 | head.next.replace(Box::new(next_node)); 17 | 18 | // Serialize to bytes 19 | let bytes = head.serialize_to_vec(); 20 | 21 | // Deserialize 22 | let list: Node = Message::deserialize_from_slice(&bytes[..])?; 23 | 24 | // Pretty print! 25 | println!("{:#?}", list); 26 | 27 | Ok(()) 28 | } 29 | -------------------------------------------------------------------------------- /examples/src/multi/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | // Under the multi folder we have two folders /team and /user 3 | use proto_multi::{ 4 | // generates a team module 5 | team::svc::TeamRequest, 6 | // generates a user module 7 | user::svc::UserRequest, 8 | }; 9 | 10 | fn main() { 11 | // What we do with these messages is largely irrelevant, we're just trying to show how proto 12 | // folder structure translates to Rust module structure 13 | 14 | let user_req = UserRequest { user_id: 1 }; 15 | let team_req = TeamRequest { team_id: 1 }; 16 | 17 | let user_bytes = user_req.serialize_to_vec(); 18 | let team_bytes = team_req.serialize_to_vec(); 19 | 20 | println!("Byte content is the same, but their meaning is different!"); 21 | println!("User: {:X?}", user_bytes); 22 | println!("Team: {:X?}", team_bytes); 23 | } 24 | -------------------------------------------------------------------------------- /examples/src/non_optional/main.rs: -------------------------------------------------------------------------------- 1 | use proto_non_optional::basic::{ 2 | City, 3 | PlaneTicket, 4 | PlaneTicket_Airport, 5 | }; 6 | 7 | fn main() { 8 | // Make our ticket, which has required fields! 9 | let ticket = PlaneTicket { 10 | passenger: "Parker".to_owned(), 11 | city: City::NEW_YORK, 12 | airport: PlaneTicket_Airport::Id(1), 13 | }; 14 | 15 | println!("Our ticket: {:#?}", ticket); 16 | } 17 | -------------------------------------------------------------------------------- /examples/src/nullable_field/main.rs: -------------------------------------------------------------------------------- 1 | use pb_jelly::Message; 2 | use proto_nullable_field::basic::{ 3 | Inner, 4 | NonNullableMessage, 5 | PlainMessage, 6 | }; 7 | 8 | /// Shows comparison between a plain message (regular fields) 9 | /// and non-nullable fields via field option rust.nullable 10 | /// 11 | /// With [(rust.nullable)=false], pb-jelly will generate 12 | /// structs fields MessageType rather than Option 13 | /// 14 | /// From protos/nullable_field/basic.proto 15 | /// message NonNullableMessage { 16 | /// Inner field1_nonnullable = 1 [(rust.nullable_field)=false]; 17 | /// Inner field2 = 2 [(rust.nullable_field)=true]; 18 | /// Inner field3 = 3; // Messages default to nullable 19 | /// } 20 | /// 21 | /// This is beneficial when you know that the message field 22 | /// semantically must be set - as you get simpler generated code. 23 | /// 24 | /// However, this loses expressiveness, as 25 | /// None and MessageType::default(), though they can be represented 26 | /// distinguishably on the wire, will be interpreted equivalently 27 | /// - as shown in the example. 28 | fn main() { 29 | // When all fields are set (non null), they operate equivalently. 30 | let plain1 = PlainMessage { 31 | field1: Some(Inner { x: 1 }), 32 | field2: Some(Inner { x: 1 }), 33 | field3: Some(Inner { x: 1 }), 34 | }; 35 | let nn1 = NonNullableMessage { 36 | // [(rust.nullable_field)=false] 37 | field1_nonnullable: Inner { x: 1 }, 38 | // [(rust.nullable_field)=true] 39 | field2: Some(Inner { x: 1 }), 40 | field3: Some(Inner { x: 1 }), 41 | }; 42 | assert_eq!(plain1.serialize_to_vec(), nn1.serialize_to_vec()); 43 | 44 | // When fields are None, there is difference in operation 45 | let plain2 = PlainMessage { 46 | field1: None, 47 | field2: None, 48 | field3: None, 49 | }; 50 | let plain2_serialized = plain2.serialize_to_vec(); 51 | let nn2 = NonNullableMessage { 52 | field1_nonnullable: Inner { x: 0 }, 53 | field2: None, 54 | field3: None, 55 | }; 56 | let nn2_serialized = nn2.serialize_to_vec(); 57 | 58 | // Serialization is not equivalent - default value and nil are distinguishable 59 | assert_eq!(plain2_serialized, vec![]); 60 | assert_eq!(nn2_serialized, vec![10, 0]); 61 | 62 | // Both wire formats deserialize equivalently for NonNullable message 63 | // despite being different on the wire 64 | assert_eq!( 65 | NonNullableMessage::deserialize_from_slice(&plain2_serialized).unwrap(), 66 | NonNullableMessage::deserialize_from_slice(&nn2_serialized).unwrap(), 67 | ); 68 | 69 | // However, they deserialize differently for Plain message - field1 being 70 | // None vs Some(Inner {x: 0}) 71 | assert_ne!( 72 | PlainMessage::deserialize_from_slice(&plain2_serialized).unwrap(), 73 | PlainMessage::deserialize_from_slice(&nn2_serialized).unwrap(), 74 | ); 75 | } 76 | -------------------------------------------------------------------------------- /examples/src/serde/main.rs: -------------------------------------------------------------------------------- 1 | use proto_serde::basic::NewYorkCity; 2 | use serde_json; 3 | 4 | fn main() -> std::io::Result<()> { 5 | // Some data formatted in JSON 6 | let json_str = r#" 7 | { 8 | "num_windows": 42, 9 | "neighborhood": "West Village" 10 | }"#; 11 | 12 | // Parse the data from our JSON formatted string, into our struct 13 | let msg: NewYorkCity = serde_json::from_str(json_str).unwrap(); 14 | 15 | assert_eq!(msg.num_windows, 42); 16 | assert_eq!(msg.neighborhood, "West Village"); 17 | 18 | println!("{:#?}", msg); 19 | 20 | Ok(()) 21 | } 22 | -------------------------------------------------------------------------------- /examples/src/sso/main.rs: -------------------------------------------------------------------------------- 1 | use compact_str::ToCompactString; 2 | use pb_jelly::Message; 3 | use proto_sso::city::City; 4 | 5 | fn main() -> std::io::Result<()> { 6 | let msg = City { 7 | name: "New York City".to_compact_string(), 8 | latitude: 40.7127837, 9 | longitude: -74.0059413, 10 | population: "8405837".to_compact_string(), 11 | rank: "1".to_compact_string(), 12 | state: "New York".to_compact_string(), 13 | }; 14 | 15 | // Serialize our message to bytes 16 | let bytes = msg.serialize_to_vec(); 17 | println!("Bytes 0x{:X?}", bytes); 18 | 19 | // De-serialize our message back to a Rust struct 20 | let de_msg: City = Message::deserialize_from_slice(&bytes[..])?; 21 | 22 | // Pretty print! 23 | println!("Message: {:#?}", de_msg); 24 | 25 | // Assert our two messages are the same 26 | assert_eq!(msg, de_msg); 27 | 28 | Ok(()) 29 | } 30 | -------------------------------------------------------------------------------- /examples/src/zero_copy/main.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use bytes::Bytes; 4 | use pb_jelly::{ 5 | Lazy, 6 | Message, 7 | }; 8 | use proto_zero_copy::basic::BytesMessage; 9 | 10 | fn main() -> std::io::Result<()> { 11 | // Create 1kb of Data 12 | let data = Lazy::new(Bytes::from(vec![42 as u8; 1 * 1024])); 13 | 14 | // Create our Proto Struct 15 | let mut proto = BytesMessage::default(); 16 | proto.set_data(data); 17 | proto.set_name("Parker".to_owned()); 18 | 19 | // Serialize our proto 20 | let ser_bytes: Vec = proto.serialize_to_vec(); 21 | 22 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 23 | 24 | // To achieve zero copy deserialization, we need to our serialized bytes to be in a container 25 | // with a reader that implements the `pb::PbBufferReader` trait, such as `Cursor`. 26 | // 27 | // Ideally your serialized bytes would already be in a `Bytes` struct, e.g. some network request you're about to 28 | // handle. 29 | let request = Bytes::from(ser_bytes); 30 | let mut reader = Cursor::new(request.clone()); 31 | 32 | // Deserialize our proto 33 | let mut de_proto = BytesMessage::default(); 34 | de_proto.deserialize(&mut reader)?; 35 | // Grab the bytes from the Lazy field 36 | let inner_bytes: Bytes = de_proto.data.unwrap().into_buffer(); 37 | // Because of the coordination between the `Cursor` reader and the `Lazy` field, 38 | // this field contains a reference to the original data, rather than a copy! 39 | assert!(request.as_ptr_range().contains(&inner_bytes.as_ptr())); 40 | 41 | // Print our fields! 42 | println!("Name: {}", de_proto.name.unwrap()); 43 | println!("{:#X}", &inner_bytes); 44 | 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /pb-jelly-gen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pb-jelly-gen" 3 | description = "A protobuf binding generation framework for the Rust language developed at Dropbox" 4 | version = "0.0.17" 5 | authors = ["Rajat Goel ", "Nipunn Koorapati ", "Parker Timmerman "] 6 | edition = "2018" 7 | license = "Apache-2.0" 8 | homepage = "https://github.com/dropbox/pb-jelly" 9 | repository = "https://github.com/dropbox/pb-jelly/tree/main/pb-jelly-gen" 10 | readme = "README.md" 11 | keywords = ["google", "protobuf", "proto", "dropbox"] 12 | categories = ["encoding", "parsing", "web-programming"] 13 | 14 | include = ["src/**/*.rs", "README.md", "LICENSE", "proto"] 15 | 16 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 17 | 18 | [features] 19 | default = ["generate"] 20 | generate = ["walkdir", "tempfile"] 21 | 22 | [dependencies] 23 | tempfile = { version = "3.1.0", optional = true } 24 | walkdir = { version = "2", optional = true } 25 | 26 | pb-jelly = { version = "0.0.17" } 27 | lazy_static = "1.4.0" 28 | indexmap = "2.0.2" 29 | 30 | # Override pb-jelly dependency for generated crates as well 31 | [patch.crates-io] 32 | pb-jelly = { path = "../pb-jelly" } 33 | -------------------------------------------------------------------------------- /pb-jelly-gen/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2020 Dropbox, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /pb-jelly-gen/README.md: -------------------------------------------------------------------------------- 1 | # `pb-jelly-gen` 2 | ###### It's working! It's working! - Anakin Skywalker 3 | [![Crates.io](https://img.shields.io/crates/v/pb-jelly-gen)](https://crates.io/crates/pb-jelly-gen) [![Documentation](https://docs.rs/pb-jelly-gen/badge.svg)](https://docs.rs/pb-jelly-gen) [![Crates.io](https://img.shields.io/crates/l/pb-jelly-gen)](LICENSE) 4 | 5 | This crate provides a tool to generate [`Rust`](https://www.rust-lang.org/) code from `.proto` files. 6 | 7 | ### How To Use 8 | 9 | You'll need the protobuf compiler which you can get by: 10 | 1. Running `brew install protobuf` or... 11 | 2. Download or build from source [`protobuf`](https://github.com/protocolbuffers/protobuf) 12 | 13 | #### As a plugin for protoc 14 | 15 | A binary is included that can be passed directly to `protoc`: 16 | 17 | ``` 18 | % cargo build --bin protoc-gen-jellyrust 19 | % protoc --plugin=protoc-gen-jellyrust=target/debug/protoc-gen-jellyrust --jellyrust_out=out foo/bar.proto... 20 | ``` 21 | 22 | #### As a library 23 | 24 | Add this crate as a dependency in your `Cargo.toml` and then call `gen_protos`: 25 | 26 | ##### `Cargo.toml` 27 | ``` 28 | [dependencies] 29 | pb-jelly-gen = "0.0.17" 30 | ``` 31 | 32 | ##### `main.rs` 33 | ``` 34 | use pb_jelly_gen::gen_protos; 35 | 36 | fn main() { 37 | // Replace `./protos` with a path to your proto files. 38 | gen_protos(vec!["./protos"]).unwrap() 39 | } 40 | ``` 41 | -------------------------------------------------------------------------------- /pb-jelly-gen/proto/rust/extensions.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers for Rust with Gadgets 2 | // 3 | // Note: While proto3 still supports extensions for custom fields (https://github.com/protocolbuffers/protobuf/issues/1460) 4 | // it does not support default values. Unfortunately the OneOfOptions::nullable field has a default value of true, while 5 | // in proto3 booleans have a default value of false. We can't migrate this file to proto3 without deprecating nullable 6 | // and changing its semantics. 7 | 8 | syntax = "proto2"; 9 | package rust; 10 | 11 | import "google/protobuf/descriptor.proto"; 12 | 13 | extend google.protobuf.FieldOptions { 14 | // Generate this field in a Box as opposed to inline Option 15 | optional bool box_it = 50000; 16 | // Generates a `Lazy` 17 | optional bool grpc_slices = 50003; 18 | // Generates a `Lazy` 19 | optional bool blob = 50010; 20 | // Use a different Rust type which implements `pb::Message` to represent the field. 21 | // All paths must be fully qualified, as in `::my_crate::full::path::to::type`. 22 | // This only works with proto3. 23 | optional string type = 50004; 24 | 25 | // Generate this `bytes` field using a Lazy to enable zero-copy deserialization. 26 | optional bool zero_copy = 50007; 27 | // Generate this `string` field using a type that supports a small string optimization. 28 | optional bool sso = 50009; 29 | 30 | // If false, make this field's Rust type non-Optional. If the field is 31 | // missing on the wire during deserialization, it will remain as 32 | // Default::default(). 33 | // 34 | // It doesn't make sense to specify this option for a repeating field, as a 35 | // missing field always deserializes as an empty Vec. 36 | // In proto3, this option is also ignored for non-`optional` primitive types, 37 | // which are already non-nullable. 38 | // 39 | // Beware that Default may not make sense for all message types. In 40 | // particular, fields using `OneofOptions.nullable=false` or 41 | // `EnumOptions.err_if_default_or_unknown=true` will simply default to their 42 | // first variant, and will _not_ trigger a deserialization error. That 43 | // behaviour may change in the future (TODO). 44 | optional bool nullable_field = 50008 [default=true]; 45 | } 46 | 47 | extend google.protobuf.EnumOptions { 48 | // Setting this to true on an enum means the generated enum won't even have a value for the 49 | // 0-value, and any message that would've parsed to having the value be 0 fail instead. 50 | // 51 | // If an enum field doesn't appear in the wire format of proto3, the 0 value is assumed. So if 52 | // an enum field is added to a message in use between a client and server, and the client hasn't 53 | // been recompiled, then all received messages on the server side will get the 0 value. As such, 54 | // it's common in cases when there's not an obvious (and safe) default value to make the 0 value 55 | // an explicit unknown/invalid value. In those cases, they become cumbersome to use in Rust 56 | // because match statements will always require a branch for the unknown/invalid case, but it's 57 | // more desirable to just fail at parse time. If the client has updated *past* the server, it may 58 | // send a value that the server does not know how to handle. We *also* fail this at parse time. 59 | optional bool err_if_default_or_unknown = 50002; 60 | 61 | // Setting this to true means that an enum's variants are considered exhaustive. 62 | // A Rust `enum` will be generated, rather than a wrapper around `i32`. This 63 | // makes matching against the enum simpler as there is no need to match 64 | // unknown variants. Instead, deserialization will fail if an unknown 65 | // variant is found over the wire. That makes adding or removing variants 66 | // potentially unsafe when it comes to version skew. 67 | // 68 | // This option differs from `err_if_default_or_unknown` because default 69 | // values are still allowed. The two options are incompatible, as 70 | // `err_if_default_or_unknown` is strictly stronger. 71 | optional bool closed_enum = 50008; 72 | } 73 | 74 | extend google.protobuf.MessageOptions { 75 | // Setting this to true adds an extra field to the deserialized message, which includes 76 | // a serialized representation of unrecognized fields. 77 | // Eg. 78 | // MyMessage { 79 | // field: u32, 80 | // _unrecognized: Vec, 81 | // } 82 | optional bool preserve_unrecognized = 50006; 83 | } 84 | 85 | extend google.protobuf.OneofOptions { 86 | // If false, this oneof must have a field set. Parse error if no variant (or unrecognized 87 | // variant) is set. 88 | optional bool nullable = 50001 [default=true]; 89 | } 90 | 91 | extend google.protobuf.FileOptions { 92 | optional bool serde_derive = 50005 [default=false]; 93 | } 94 | -------------------------------------------------------------------------------- /pb-jelly-gen/regen_gen_protos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cargo build --no-default-features && \ 3 | protoc --plugin=protoc-gen-jellyrust=target/debug/protoc-gen-jellyrust -Iproto rust/extensions.proto google/protobuf/{compiler/plugin,descriptor}.proto --jellyrust_out=single_file:src 4 | -------------------------------------------------------------------------------- /pb-jelly-gen/src/bin/protoc-gen-jellyrust.rs: -------------------------------------------------------------------------------- 1 | //! A codegen plugin for use with `protoc --jellyrust_out=...` 2 | 3 | use std::io::{ 4 | self, 5 | Read, 6 | Write, 7 | }; 8 | 9 | use pb_jelly::Message; 10 | use pb_jelly_gen::codegen::generate_code; 11 | use pb_jelly_gen::protos::google::protobuf::compiler::plugin; 12 | 13 | fn main() -> io::Result<()> { 14 | // Read request message from stdin 15 | let mut data = Vec::new(); 16 | io::stdin().read_to_end(&mut data)?; 17 | 18 | // Parse request 19 | let request = plugin::CodeGeneratorRequest::deserialize_from_slice(&data)?; 20 | 21 | // Generate code 22 | let response = generate_code(&request); 23 | 24 | // Serialise response message 25 | let output = response.serialize_to_vec(); 26 | // Write to stdout 27 | io::stdout().write_all(&output)?; 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /pb-jelly-gen/src/generate.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::fs; 3 | use std::path::{ 4 | self, 5 | Path, 6 | PathBuf, 7 | }; 8 | use std::process::Command; 9 | 10 | use pb_jelly::Message; 11 | use walkdir::WalkDir; 12 | 13 | use crate::codegen; 14 | use crate::protos::google::protobuf::compiler::plugin::CodeGeneratorRequest; 15 | use crate::protos::google::protobuf::descriptor::FileDescriptorSet; 16 | 17 | /// A "no frills" way to generate Rust bindings for your proto files. `src_paths` is a list of 18 | /// paths to your `.proto` files, or the directories that contain them. Generated code it outputted 19 | /// to `/gen`. 20 | pub fn gen_protos>(src_paths: Vec

) -> Result<(), Box> { 21 | GenProtos::builder().src_paths(src_paths).gen_protos() 22 | } 23 | 24 | /// A builder struct to configure the way your protos are generated, create one with `GenProtos::builder()` 25 | #[must_use] 26 | pub struct GenProtos { 27 | gen_path: PathBuf, 28 | src_paths: Vec, 29 | include_paths: Vec, 30 | include_extensions: bool, 31 | cleanup_out_path: bool, 32 | } 33 | 34 | impl std::default::Default for GenProtos { 35 | fn default() -> Self { 36 | let gen_path = 37 | get_cargo_manifest_path().expect("couldn't get `CARGO_MANIFEST_DIR` when building default GenProtos"); 38 | let gen_path = gen_path.join(PathBuf::from("./gen")); 39 | 40 | let src_paths = vec![]; 41 | let include_paths = vec![]; 42 | let include_extensions = true; 43 | let cleanup_out_path = false; 44 | 45 | GenProtos { 46 | gen_path, 47 | src_paths, 48 | include_paths, 49 | include_extensions, 50 | cleanup_out_path, 51 | } 52 | } 53 | } 54 | 55 | // Public functions 56 | impl GenProtos { 57 | /// Create a default builder 58 | pub fn builder() -> GenProtos { 59 | GenProtos::default() 60 | } 61 | 62 | /// Set the output path for the generated code. This should be relative to the current crate's 63 | /// manifest. 64 | /// 65 | /// Defaults to the `/gen` 66 | pub fn out_path>(mut self, path: P) -> GenProtos { 67 | let manifest_path = get_cargo_manifest_path().expect("out_path"); 68 | self.gen_path = manifest_path.join(path); 69 | self 70 | } 71 | 72 | /// Set the output path for the generate code. This will be treated as an absolute path. 73 | pub fn abs_out_path>(mut self, path: P) -> GenProtos { 74 | self.gen_path = path.as_ref().to_owned(); 75 | self 76 | } 77 | 78 | /// Add a path to a `.proto` file, or a directory containing your proto files. 79 | pub fn src_path>(mut self, path: P) -> GenProtos { 80 | self.src_paths.push(path.as_ref().to_owned()); 81 | self 82 | } 83 | 84 | /// Add a list of paths to `.proto` files, or to directories containing your proto files. 85 | pub fn src_paths, I: IntoIterator>(mut self, paths: I) -> GenProtos { 86 | self.src_paths.extend(paths.into_iter().map(|p| p.as_ref().to_owned())); 87 | self 88 | } 89 | 90 | /// A path to a protobuf file, or a directory containing protobuf files, that get included in 91 | /// the proto compilation. Rust bindings will *not* be generated for these files, but the proto 92 | /// compiler will look at included paths for proto dependencies. 93 | pub fn include_path>(mut self, path: P) -> GenProtos { 94 | self.include_paths.push(path.as_ref().to_owned()); 95 | self 96 | } 97 | 98 | /// Paths to a protobuf files, or directories containing protobuf files, that get included in 99 | /// the proto compilation. Rust bindings will *not* be generated for these files, but the proto 100 | /// compiler will look at included paths for proto dependencies. 101 | pub fn include_paths, I: IntoIterator>(mut self, paths: I) -> GenProtos { 102 | self.include_paths 103 | .extend(paths.into_iter().map(|p| p.as_ref().to_owned())); 104 | self 105 | } 106 | 107 | /// Include `rust/extensions.proto` in the proto compilation. 108 | /// 109 | /// Defaults to true 110 | pub fn include_extensions(mut self, include: bool) -> GenProtos { 111 | self.include_extensions = include; 112 | self 113 | } 114 | 115 | /// If true, before proto compilation, will delete whatever exists at `out_path` and create a 116 | /// directory at that location. 117 | pub fn cleanup_out_path(mut self, cleanup: bool) -> GenProtos { 118 | self.cleanup_out_path = cleanup; 119 | self 120 | } 121 | 122 | /// Consumes the builder and generates Rust bindings to your proto files. 123 | pub fn gen_protos(self) -> Result<(), Box> { 124 | // TODO: change expect()s to propagate errors. 125 | 126 | // Clean up root generated directory 127 | if self.cleanup_out_path && self.gen_path.exists() && self.gen_path.is_dir() { 128 | fs::remove_dir_all(&self.gen_path).expect("Failed to clean"); 129 | } 130 | 131 | let temp_dir = tempfile::Builder::new() 132 | .prefix("codegen") 133 | .tempdir() 134 | .expect("Failed to create temp dir"); 135 | 136 | // Construct protoc command line 137 | let mut protoc_cmd = Command::new("protoc"); 138 | 139 | // Directories that contain protos 140 | for path in &self.src_paths { 141 | protoc_cmd.arg("-I"); 142 | protoc_cmd.arg(path); 143 | } 144 | 145 | // If we want to include our `extensions.proto` file for Rust extentions 146 | if self.include_extensions { 147 | fs::create_dir_all(temp_dir.path().join("rust")).expect("failed to create rust/"); 148 | fs::write(temp_dir.path().join("rust").join("extensions.proto"), EXTENSIONS_PROTO) 149 | .expect("failed to create rust/extensions.proto"); 150 | protoc_cmd.arg("-I"); 151 | protoc_cmd.arg(temp_dir.path()); 152 | } 153 | 154 | // Include any protos from our include paths 155 | for path in &self.include_paths { 156 | protoc_cmd.arg("-I"); 157 | protoc_cmd.arg(path); 158 | } 159 | 160 | // Ideally we'd just invoke protoc with our plugin, 161 | // but without artifact dependencies in Cargo it's hard to depend on a binary Rust target. 162 | // Instead we'll invoke the guts of the plugin manually. 163 | let file_descriptor_set_path = temp_dir.path().join("file_descriptor_set.pb"); 164 | protoc_cmd.arg("-o").arg(&file_descriptor_set_path); 165 | protoc_cmd.arg("--include_imports"); 166 | protoc_cmd.arg("--include_source_info"); 167 | 168 | // Get paths of our Protos 169 | let proto_paths: Vec = self 170 | .src_paths 171 | .iter() 172 | .flat_map(|path| { 173 | WalkDir::new(path) 174 | .into_iter() 175 | .filter_map(Result::ok) 176 | .filter(|file| file.path().extension().unwrap_or_default() == "proto") 177 | .map(move |file| { 178 | let relative_path = file 179 | .path() 180 | .strip_prefix(path) 181 | .expect("Walked file didn't have root as a prefix"); 182 | // Convert all paths into Unix-style, relative paths 183 | relative_path 184 | .to_str() 185 | .unwrap_or_else(|| panic!("File path is not UTF-8: {}", file.path().display())) 186 | .replace(path::MAIN_SEPARATOR, "/") 187 | }) 188 | }) 189 | .collect(); 190 | 191 | // Set each proto file as an argument 192 | protoc_cmd.args(&proto_paths); 193 | 194 | let protoc_status = protoc_cmd.status().expect("something went wrong in running protoc"); 195 | 196 | if !protoc_status.success() { 197 | return Err(format!("protoc exited with status {protoc_status}").into()); 198 | } 199 | 200 | let file_descriptor_set = FileDescriptorSet::deserialize_from_slice( 201 | &fs::read(file_descriptor_set_path).expect("Failed to read protoc output"), 202 | ) 203 | .expect("Failed to deserialize FileDescriptorSet"); 204 | 205 | let plugin_input = CodeGeneratorRequest { 206 | file_to_generate: proto_paths, 207 | proto_file: file_descriptor_set.file, 208 | ..Default::default() 209 | }; 210 | let out = codegen::generate_code(&plugin_input); 211 | if let Some(error) = out.error { 212 | panic!("Codegen error: {}", error); 213 | } 214 | for file in out.file { 215 | let path = self.gen_path.join(file.get_name()); 216 | fs::create_dir_all(path.parent().expect("generated path should have parent")) 217 | .expect("Failed to create dir"); 218 | fs::write(path, file.get_content()).expect("Failed to write output"); 219 | } 220 | Ok(()) 221 | } 222 | } 223 | 224 | const EXTENSIONS_PROTO: &str = include_str!("../proto/rust/extensions.proto"); 225 | 226 | /// Helper function to get the path of the current Cargo.toml 227 | /// 228 | /// Get the environment value of `CARGO_MANIFEST_DIR` and converts it into a `PathBuf` 229 | #[doc(hidden)] 230 | fn get_cargo_manifest_path() -> std::io::Result { 231 | let path_str = std::env::var("CARGO_MANIFEST_DIR").map_err(|_| std::io::ErrorKind::NotFound)?; 232 | Ok(PathBuf::from(path_str)) 233 | } 234 | -------------------------------------------------------------------------------- /pb-jelly-gen/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! `pb-jelly-gen` generates Rust bindings for `proto` files. It's intended to be used with [`pb-jelly`](https://github.com/dropbox/pb-jelly). 2 | //! 3 | //! ## Examples 4 | //! Complete examples can be found in the [`examples`](https://github.com/dropbox/pb-jelly/tree/main/examples) crate, 5 | //! or the [`pb-test`](https://github.com/dropbox/pb-jelly/tree/main/pb-test) crate of the [`pb-jelly`](https://github.com/dropbox/pb-jelly) workspace. 6 | //! 7 | //! ## In a nutshell 🥜 8 | //! You can include `pb-jelly-gen` in your Cargo project, by including it as a `[build-dependency]` in your `Cargo.toml` 9 | //! ```toml 10 | //! [build-dependencies] 11 | //! pb-jelly-gen = "0.0.17" 12 | //! ``` 13 | //! 14 | //! Then from a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script, use either the `GenProtos` builder struct, 15 | //! or the `gen_protos` convience function to specify where your protos live, and where the generated code should be 16 | //! put. 17 | //! ```no_run 18 | //! use pb_jelly_gen::GenProtos; 19 | //! 20 | //! fn main() -> Result<(), Box> { 21 | //! GenProtos::builder() 22 | //! // output path for our generated code 23 | //! .out_path("./gen") 24 | //! // directory where our protos live 25 | //! .src_path("./protos") 26 | //! // delete and recreate the `out_path` directory every time 27 | //! .cleanup_out_path(true) 28 | //! .gen_protos()?; 29 | //! 30 | //! Ok(()) 31 | //! } 32 | //! ``` 33 | 34 | pub mod codegen; 35 | /// Don't depend on this module, it's only public so that `protoc-gen-jellyrust` can use it 36 | #[doc(hidden)] 37 | #[rustfmt::skip] 38 | pub mod protos; 39 | 40 | #[cfg(feature = "generate")] 41 | mod generate; 42 | #[cfg(feature = "generate")] 43 | pub use crate::generate::*; 44 | -------------------------------------------------------------------------------- /pb-jelly/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pb-jelly" 3 | description = "A protobuf runtime for the Rust language developed at Dropbox" 4 | version = "0.0.17" 5 | authors = ["Rajat Goel ", "Nipunn Koorapati ", "Parker Timmerman "] 6 | edition = "2018" 7 | license = "Apache-2.0" 8 | homepage = "https://github.com/dropbox/pb-jelly" 9 | repository = "https://github.com/dropbox/pb-jelly/tree/main/pb-jelly" 10 | readme = "README.md" 11 | keywords = ["google", "protobuf", "proto", "dropbox"] 12 | categories = ["encoding", "parsing", "web-programming"] 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | byteorder = "1.4" 18 | bytes = "1.0" 19 | compact_str = { version = "0.5", features = ["bytes"] } 20 | -------------------------------------------------------------------------------- /pb-jelly/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2020 Dropbox, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /pb-jelly/README.md: -------------------------------------------------------------------------------- 1 | # `pb-jelly` 2 | ###### With great power, comes great responsibility - Peter Parker 3 | 4 | [![Crates.io](https://img.shields.io/crates/v/pb-jelly)](https://crates.io/crates/pb-jelly) [![Documentation](https://docs.rs/pb-jelly/badge.svg)](https://docs.rs/pb-jelly) [![Crates.io](https://img.shields.io/crates/l/pb-jelly)](LICENSE) 5 | 6 | This crates provides the necessary trait implementations to power code generated with [`pb-jelly-gen`](https://github.com/dropbox/pb-jelly/tree/main/pb-jelly-gen). You should 7 | include this crate as a dependency in your `Cargo.toml`. 8 | 9 | 10 | ##### `Cargo.toml` 11 | ``` 12 | [dependencies] 13 | pb-jelly = "0.0.17" 14 | ``` 15 | 16 | Then in the general case, all you'll need to use in your code is the `Message` trait this crate defines, e.g. 17 | ``` 18 | use pb_jelly::Message; 19 | ``` 20 | 21 | More complete examples can be found in the [`examples`](https://github.com/dropbox/pb-jelly/tree/main/examples) crate, or 22 | the [`pb-test`](https://github.com/dropbox/pb-jelly/tree/main/pb-test) crate itself. 23 | -------------------------------------------------------------------------------- /pb-jelly/src/buffer.rs: -------------------------------------------------------------------------------- 1 | //! This module exposes two fundamental concepts: 2 | //! 3 | //! [PbBufferReader] 4 | //! A [PbBufferReader] is something which a [Message] can be deserialized from. In the common case, 5 | //! this means that the relevant bytes are copied out of the underlying store and copied into an 6 | //! appropriate struct which implements [Message]. 7 | //! 8 | //! [PbBufferWriter] 9 | //! A [PbBufferWriter] is something which a [Message] can be serialized to. In the common case, this 10 | //! means that the relevant bytes are copied out of the concrete struct and into the underlying 11 | //! data store. 12 | //! 13 | //! # Zerocopy serialization and deserialization and `Lazy` 14 | //! 15 | //! There are cases where we want to minimize the number of times we copy the data contained within 16 | //! a message. Especially on resource-constrained hardware (mostly MP OSDs), we want to avoid the cost 17 | //! of copying large buffers during serialization and deserialization. 18 | //! 19 | //! To support this, messages may contain zerocopy fields using the [`Lazy`] type. 20 | //! `pb-jelly-gen` may generate these using the `blob`, `grpc_slices`, or `zero_copy` options; they 21 | //! use different underlying types, which must implement [PbBuffer], but they all behave similarly. 22 | //! 23 | //! [PbBufferReader] and a [PbBufferWriter] have the opportunity to recognize [Lazy] fields. 24 | //! At deserialization time, if [PbBufferReader] is used with a compatible [Lazy] field, instead of 25 | //! allocating, it may simply store a reference to its underlying input buffer in the [Lazy]. 26 | //! Similarly, at serialization time, a [PbBufferWriter] used with a compatible [Lazy] may copy a 27 | //! reference to the [Lazy] field into its output buffer, rather than copying its content. 28 | //! 29 | //! Request (bytes on the wire) 30 | //! | 31 | //! v 32 | //! RPC Framework (with an underlying allocator, e.g. blob::Blob or grpc::Slice) 33 | //! | 34 | //! v 35 | //! BR: [PbBufferReader] deserializes the struct using RPC framework's allocator. 36 | //! | 37 | //! v 38 | //! Request (concrete struct containing a [Lazy] field) 39 | //! | 40 | //! v 41 | //! RPC handler (doesn't modify the [Lazy] field) 42 | //! | 43 | //! v 44 | //! Response (concrete struct containing a [Lazy] field) 45 | //! | 46 | //! v 47 | //! BW: [PbBufferWriter] serializes the struct using RPC framework's allocator. 48 | //! | 49 | //! v 50 | //! RPC Framework (with an underlying allocator, e.g. blob::Blob or grpc::Slice) 51 | //! | 52 | //! v 53 | //! Response (bytes on the wire). 54 | //! 55 | //! 56 | //! In the status quo, the behavior is as follows: 57 | //! 58 | //! `blob_pb::WrappedBlob` and `blob_pb::VecSlice` allow zero-copy deserialization -> serialization, 59 | //! provided that their respective [PbBufferWriter]s are used. 60 | //! Converting from `blob_pb::WrappedBlob` to a `blob_pb::VecSlice` is zero-copy. 61 | //! Converting from a `blob_pb::VecSlice` to a `blob_pb::Blob` requires a single copy. 62 | 63 | use std::any::Any; 64 | use std::fmt::{ 65 | self, 66 | Debug, 67 | }; 68 | use std::io::{ 69 | Cursor, 70 | Result, 71 | Write, 72 | }; 73 | 74 | use bytes::{ 75 | Buf, 76 | Bytes, 77 | }; 78 | 79 | use super::{ 80 | Message, 81 | Reflection, 82 | }; 83 | 84 | /// A stand-in trait for any backing buffer store. 85 | /// `PbBuffer`s are expected to own references to the data they reference, and should be cheap 86 | /// (constant-time) to clone. 87 | #[allow(clippy::len_without_is_empty)] 88 | pub trait PbBuffer: Any + Sized { 89 | /// Returns the length of the data contained in this buffer. 90 | fn len(&self) -> usize; 91 | /// Fallback method to read the contents of `self`. This method is expected to write exactly 92 | /// `self.len()` bytes into `writer`, or fail with an error. 93 | /// 94 | /// This method is used to write `Lazy` fields to incompatible [`PbBufferWriter`]s. 95 | fn copy_to_writer(&self, writer: &mut W) -> Result<()>; 96 | /// Fallback method to create an instance of this `PbBuffer`. 97 | /// 98 | /// This method is used to read `Lazy` fields from incompatible [`PbBufferReader`]s. 99 | fn copy_from_reader(reader: &mut B) -> Result; 100 | } 101 | 102 | /// If `B1` and `B2` are the same type, returns a function to cast `B1 -> B2`; otherwise None. 103 | /// Used to implement [PbBuffer] casting. 104 | pub fn type_is() -> Option B2> { 105 | let f: fn(B1) -> B1 = |x| x; 106 | // If B1 = B2, then this cast should succeed! 107 | (&f as &dyn Any).downcast_ref:: B2>().copied() 108 | } 109 | 110 | /// All concrete types which are used for deserialization should implement 111 | /// [PbBufferReader], which includes functions to convert to and from [PbBuffer]. 112 | pub trait PbBufferReader: Buf { 113 | /// Attempt to read into a compatible [PbBuffer], avoiding a copy if possible. 114 | /// The implementation should dispatch on the type `B`. If unsupported, 115 | /// the reader may fall back to [PbBuffer::copy_from_reader]. 116 | fn read_buffer(&mut self) -> Result { 117 | B::copy_from_reader(self) 118 | } 119 | 120 | /// Advance the interal cursor by `at`, and return a [PbBufferReader] corresponding to the 121 | /// traversed indices (i.e. self.position..self.position + at). 122 | fn split(&mut self, at: usize) -> Self; 123 | } 124 | 125 | /// All concrete types used for serialization should implement [PbBufferWriter] in order to support 126 | /// serializing [Lazy] fields without copies. 127 | pub trait PbBufferWriter: Write { 128 | /// Attempt to write a zerocopy buffer into `self`. If `B` is not zero-copy-supported 129 | /// by the [PbBufferWriter], this may read/copy the bytes out from `buf`. 130 | fn write_buffer(&mut self, buf: &B) -> Result<()>; 131 | } 132 | 133 | /// A wrapper around a [PbBuffer], which implements [Message]. 134 | #[derive(Clone, PartialEq)] 135 | pub struct Lazy { 136 | // TODO: Make this not an `Option` by giving `VecSlice` a cheap `Default` impl 137 | contents: Option, 138 | } 139 | 140 | impl Default for Lazy { 141 | fn default() -> Self { 142 | Self { contents: None } 143 | } 144 | } 145 | 146 | impl Lazy { 147 | pub fn new(r: B) -> Self { 148 | Self { contents: Some(r) } 149 | } 150 | 151 | pub fn into_buffer(self) -> B 152 | where 153 | B: Default, 154 | { 155 | self.contents.unwrap_or_default() 156 | } 157 | } 158 | 159 | impl Debug for Lazy { 160 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 161 | f.debug_struct("Lazy") 162 | .field("contents", &self.contents.as_ref().map(|_| "_")) 163 | .finish() 164 | } 165 | } 166 | 167 | impl Message for Lazy { 168 | fn compute_size(&self) -> usize { 169 | self.contents.as_ref().map(PbBuffer::len).unwrap_or(0) 170 | } 171 | 172 | fn compute_grpc_slices_size(&self) -> usize { 173 | self.contents.as_ref().map(PbBuffer::len).unwrap_or(0) 174 | } 175 | 176 | fn serialize(&self, w: &mut W) -> Result<()> { 177 | if let Some(ref contents) = self.contents { 178 | w.write_buffer(contents)?; 179 | } 180 | Ok(()) 181 | } 182 | 183 | fn deserialize(&mut self, r: &mut R) -> Result<()> { 184 | self.contents = Some(r.read_buffer()?); 185 | Ok(()) 186 | } 187 | } 188 | 189 | impl Reflection for Lazy {} 190 | 191 | impl<'a> PbBufferReader for Cursor<&'a [u8]> { 192 | fn split(&mut self, at: usize) -> Self { 193 | let pos = self.position() as usize; 194 | self.advance(at); 195 | let new_slice = &self.get_ref()[pos..pos + at]; 196 | Self::new(new_slice) 197 | } 198 | } 199 | 200 | impl PbBuffer for Bytes { 201 | #[inline] 202 | fn len(&self) -> usize { 203 | self.len() 204 | } 205 | 206 | fn copy_to_writer(&self, writer: &mut W) -> Result<()> { 207 | writer.write_all(&self) 208 | } 209 | 210 | fn copy_from_reader(reader: &mut B) -> Result { 211 | let len = reader.remaining(); 212 | Ok(reader.copy_to_bytes(len)) 213 | } 214 | } 215 | 216 | impl PbBufferReader for Cursor { 217 | fn read_buffer(&mut self) -> Result { 218 | if let Some(cast) = type_is::() { 219 | let bytes = self.get_ref().slice((self.position() as usize)..); 220 | Ok(cast(bytes)) 221 | } else { 222 | B::copy_from_reader(self) 223 | } 224 | } 225 | 226 | #[inline] 227 | fn split(&mut self, at: usize) -> Self { 228 | let pos = self.position() as usize; 229 | self.advance(at); 230 | let new_slice = self.get_ref().slice(pos..(pos + at)); 231 | Self::new(new_slice) 232 | } 233 | } 234 | 235 | impl<'a> PbBufferWriter for Cursor<&'a mut Vec> { 236 | /// Note: this implementation freely copies the data out of `buf`. 237 | #[inline] 238 | fn write_buffer(&mut self, buf: &B) -> Result<()> { 239 | buf.copy_to_writer(self) 240 | } 241 | } 242 | 243 | impl<'a> PbBufferWriter for Cursor<&'a mut [u8]> { 244 | /// Note: this implementation freely copies the data out of `buf`. 245 | #[inline] 246 | fn write_buffer(&mut self, buf: &B) -> Result<()> { 247 | buf.copy_to_writer(self) 248 | } 249 | } 250 | 251 | /// A wrapper around a [Write] which copies all [Lazy] data into the underlying [Write]r. 252 | pub struct CopyWriter<'a, W: Write> { 253 | pub writer: &'a mut W, 254 | } 255 | 256 | impl<'a, W: Write + 'a> Write for CopyWriter<'a, W> { 257 | #[inline] 258 | fn write(&mut self, buf: &[u8]) -> Result { 259 | self.writer.write(buf) 260 | } 261 | 262 | #[inline] 263 | fn flush(&mut self) -> Result<()> { 264 | self.writer.flush() 265 | } 266 | } 267 | 268 | impl<'a, W: Write + 'a> PbBufferWriter for CopyWriter<'a, W> { 269 | /// Note: this implementation freely copies the data out of `buf`. 270 | #[inline] 271 | fn write_buffer(&mut self, buf: &B) -> Result<()> { 272 | buf.copy_to_writer(self.writer) 273 | } 274 | } 275 | 276 | #[test] 277 | fn test_lazy_bytes_deserialize() { 278 | let mut lazy = Lazy::::default(); 279 | let bytes = Bytes::from_static(b"asdfasdf"); 280 | lazy.deserialize(&mut Cursor::new(bytes.clone())) 281 | .expect("failed to deserialize"); 282 | let deserialized = lazy.into_buffer(); 283 | assert_eq!(deserialized, bytes, "The entire buffer should be copied"); 284 | assert_eq!( 285 | deserialized.as_ptr(), 286 | bytes.as_ptr(), 287 | "The Bytes instance should be cloned" 288 | ) 289 | } 290 | -------------------------------------------------------------------------------- /pb-jelly/src/descriptor.rs: -------------------------------------------------------------------------------- 1 | use crate::wire_format; 2 | 3 | /// Trait implemented by all the messages defined in proto files. 4 | /// Provides rudimentary support for message descriptor, mostly as a way to get 5 | /// the type name for a given message rather than trying to implement the full 6 | /// reflection API. For more info, see: 7 | /// 8 | /// 9 | pub struct MessageDescriptor { 10 | /// The name of the message type, not including its scope. 11 | pub name: &'static str, 12 | /// The fully-qualified name of the message type, scope delimited by periods. 13 | /// For example, message type "Foo" which is declared in package "bar" has full name "bar.Foo". 14 | /// If a type "Baz" is nested within Foo, Baz's full_name is "bar.Foo_Baz". To get only the 15 | /// part that comes after the last '.', use `message_name`. 16 | pub full_name: &'static str, 17 | /// Information about the fields in this message. 18 | pub fields: &'static [FieldDescriptor], 19 | /// Information about the oneofs in this message. 20 | pub oneofs: &'static [OneofDescriptor], 21 | } 22 | 23 | /// Describes a field within a message. 24 | /// Provides rudimentary support for the proto field reflection API, with the intention of being 25 | /// able to serialize or introspect fields individually without having knowledge of the structure 26 | /// of the specific message itself. For more info, see: 27 | /// 28 | /// 29 | pub struct FieldDescriptor { 30 | /// The name of this field, not including its scope. 31 | pub name: &'static str, 32 | /// The fully-qualified name of this field, scope delimited by periods. 33 | pub full_name: &'static str, 34 | /// The index of this field, which has values from 0 inclusive to n exclusive, where n is the 35 | /// number of fields in this message. 36 | pub index: u32, 37 | /// The number assigned to the field in the proto declaration. 38 | pub number: u32, 39 | pub typ: wire_format::Type, 40 | pub label: Label, 41 | /// If this field is part of a oneof group, this is an index into the `oneofs` of the parent 42 | /// message. Otherwise None. 43 | pub oneof_index: Option, 44 | } 45 | 46 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] 47 | pub enum Label { 48 | Optional = 1, 49 | Required = 2, 50 | Repeated = 3, 51 | } 52 | 53 | /// Describes a oneof. 54 | pub struct OneofDescriptor { 55 | pub name: &'static str, 56 | } 57 | 58 | impl MessageDescriptor { 59 | /// Gets a field by name. 60 | pub fn get_field(&self, name: &str) -> Option<&FieldDescriptor> { 61 | self.fields.iter().find(|f| f.name == name) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pb-jelly/src/erased.rs: -------------------------------------------------------------------------------- 1 | //! Blanket-implemented erased typings for the types used throughout the serialization portion of 2 | //! this crate, to allow for implementation of the proto message field reflection API. 3 | use std::io::{ 4 | Cursor, 5 | Result, 6 | }; 7 | 8 | use crate::descriptor::MessageDescriptor; 9 | use crate::Message as ConcreteMessage; 10 | 11 | pub trait Message { 12 | fn erased_descriptor(&self) -> Option; 13 | fn erased_compute_size(&self) -> usize; 14 | fn erased_compute_grpc_slices_size(&self) -> usize; 15 | fn erased_serialize(&self) -> Vec; 16 | fn erased_deserialize(&mut self, r: &[u8]) -> Result<()>; 17 | } 18 | 19 | impl Message for T 20 | where 21 | T: ConcreteMessage, 22 | { 23 | fn erased_descriptor(&self) -> Option { 24 | self.descriptor() 25 | } 26 | 27 | fn erased_compute_size(&self) -> usize { 28 | self.compute_size() 29 | } 30 | 31 | fn erased_compute_grpc_slices_size(&self) -> usize { 32 | self.compute_grpc_slices_size() 33 | } 34 | 35 | fn erased_serialize(&self) -> Vec { 36 | self.serialize_to_vec() 37 | } 38 | 39 | fn erased_deserialize(&mut self, r: &[u8]) -> Result<()> { 40 | self.deserialize(&mut Cursor::new(r)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pb-jelly/src/extensions.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::marker::PhantomData; 3 | 4 | use crate::{ 5 | ensure_wire_format, 6 | varint, 7 | wire_format, 8 | Message, 9 | PbBufferReader, 10 | Unrecognized, 11 | }; 12 | 13 | /// Indicates that a message type has extension ranges defined. 14 | /// See for details. 15 | pub trait Extensible: Message { 16 | /// Attempts to read the given extension field from `self`. 17 | /// 18 | /// If the field was not present, or any value failed to deserialize, returns the 19 | /// default value for the extension field. This is either the declared default if 20 | /// specified, or the empty value. 21 | fn get_extension>(&self, extension: E) -> E::Value { 22 | extension.get_or_default(self) 23 | } 24 | 25 | /// Attempts to read the given extension field from `self`. 26 | /// 27 | /// Returns `Err(_)` if the field was found but could not be deserialized as the declared field type. 28 | fn get_extension_opt>(&self, extension: E) -> io::Result { 29 | extension.get(self) 30 | } 31 | 32 | /// Returns a reference to the `_extensions` field. 33 | /// This is intended to be implemented by generated code and isn't very useful for users of pb-jelly, 34 | /// so it's doc(hidden). 35 | #[doc(hidden)] 36 | fn _extensions(&self) -> &Unrecognized; 37 | } 38 | 39 | /// Abstracts over [SingularExtension]/[RepeatedExtension]. 40 | pub trait Extension { 41 | type Extendee: Extensible; 42 | type MaybeValue; 43 | type Value; 44 | fn get(&self, m: &Self::Extendee) -> io::Result; 45 | fn get_or_default(&self, m: &Self::Extendee) -> Self::Value; 46 | } 47 | 48 | /// An extension field. See for details. 49 | pub struct SingularExtension { 50 | pub field_number: u32, 51 | pub wire_format: wire_format::Type, 52 | pub name: &'static str, 53 | pub default: fn() -> U, 54 | _phantom: PhantomData U>, 55 | } 56 | 57 | impl SingularExtension { 58 | pub const fn new( 59 | field_number: u32, 60 | wire_format: wire_format::Type, 61 | name: &'static str, 62 | default: fn() -> U, 63 | ) -> Self { 64 | Self { 65 | field_number, 66 | wire_format, 67 | name, 68 | default, 69 | _phantom: PhantomData, 70 | } 71 | } 72 | } 73 | 74 | impl Copy for SingularExtension {} 75 | impl Clone for SingularExtension { 76 | fn clone(&self) -> Self { 77 | *self 78 | } 79 | } 80 | 81 | impl Extension for SingularExtension { 82 | type Extendee = T; 83 | type MaybeValue = Option; 84 | type Value = U; 85 | 86 | fn get(&self, m: &Self::Extendee) -> io::Result> { 87 | Ok(match m._extensions().get_singular_field(self.field_number) { 88 | Some((field, wire_format)) => { 89 | let mut buf = io::Cursor::new(field); 90 | ensure_wire_format(wire_format, self.wire_format, self.name, self.field_number)?; 91 | if wire_format == wire_format::Type::LengthDelimited { 92 | // we don't actually need this since we already have the length of `field` 93 | varint::read(&mut buf)?; 94 | } 95 | let mut msg = U::default(); 96 | msg.deserialize(&mut buf)?; 97 | Some(msg) 98 | }, 99 | None => None, 100 | }) 101 | } 102 | 103 | fn get_or_default(&self, m: &Self::Extendee) -> Self::Value { 104 | self.get(m).ok().flatten().unwrap_or_else(self.default) 105 | } 106 | } 107 | 108 | /// A `repeated` extension field. See for details. 109 | pub struct RepeatedExtension { 110 | pub field_number: u32, 111 | pub wire_format: wire_format::Type, 112 | pub name: &'static str, 113 | _phantom: PhantomData U>, 114 | } 115 | 116 | impl RepeatedExtension { 117 | pub const fn new(field_number: u32, wire_format: wire_format::Type, name: &'static str) -> Self { 118 | Self { 119 | field_number, 120 | wire_format, 121 | name, 122 | _phantom: PhantomData, 123 | } 124 | } 125 | } 126 | 127 | impl Copy for RepeatedExtension {} 128 | impl Clone for RepeatedExtension { 129 | fn clone(&self) -> Self { 130 | *self 131 | } 132 | } 133 | 134 | impl Extension for RepeatedExtension { 135 | type Extendee = T; 136 | type MaybeValue = Vec; 137 | type Value = Vec; 138 | 139 | fn get(&self, m: &Self::Extendee) -> io::Result> { 140 | let mut result = vec![]; 141 | let mut buf = io::Cursor::new(m._extensions().get_fields(self.field_number)); 142 | while let Some((_field_number, wire_format)) = wire_format::read(&mut buf)? { 143 | ensure_wire_format(wire_format, self.wire_format, self.name, self.field_number)?; 144 | let mut msg = U::default(); 145 | if wire_format == wire_format::Type::LengthDelimited { 146 | let length = varint::read(&mut buf)?.expect("corrupted Unrecognized"); 147 | msg.deserialize(&mut PbBufferReader::split(&mut buf, length as usize))?; 148 | } else { 149 | // we rely on the fact that the appropriate `Message` impls for i32/Fixed32/etc. only read the prefix of 150 | // `buf`. this is a little dirty 151 | msg.deserialize(&mut buf)?; 152 | } 153 | result.push(msg); 154 | } 155 | Ok(result) 156 | } 157 | 158 | fn get_or_default(&self, m: &Self::Extendee) -> Self::Value { 159 | self.get(m).unwrap_or_default() 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /pb-jelly/src/helpers.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use crate::{ 4 | ensure_split, 5 | ensure_wire_format, 6 | varint, 7 | wire_format, 8 | Message, 9 | PbBufferReader, 10 | PbBufferWriter, 11 | }; 12 | 13 | pub fn deserialize_packed( 14 | buf: &mut B, 15 | typ: wire_format::Type, 16 | expected_wire_format: wire_format::Type, 17 | msg_name: &'static str, 18 | field_number: u32, 19 | out: &mut Vec, 20 | ) -> io::Result<()> { 21 | match typ { 22 | wire_format::Type::LengthDelimited => { 23 | let len = varint::ensure_read(buf)?; 24 | let mut vals = ensure_split(buf, len as usize)?; 25 | while vals.has_remaining() { 26 | let mut val: T = Default::default(); 27 | val.deserialize(&mut vals)?; 28 | out.push(val); 29 | } 30 | }, 31 | _ => { 32 | ensure_wire_format(typ, expected_wire_format, msg_name, field_number)?; 33 | let mut val: T = Default::default(); 34 | val.deserialize(buf)?; 35 | out.push(val); 36 | }, 37 | } 38 | Ok(()) 39 | } 40 | 41 | pub fn deserialize_length_delimited( 42 | buf: &mut B, 43 | typ: wire_format::Type, 44 | msg_name: &'static str, 45 | field_number: u32, 46 | ) -> io::Result { 47 | ensure_wire_format(typ, wire_format::Type::LengthDelimited, msg_name, field_number)?; 48 | let len = varint::ensure_read(buf)?; 49 | let mut next = ensure_split(buf, len as usize)?; 50 | let mut val: T = Default::default(); 51 | val.deserialize(&mut next)?; 52 | Ok(val) 53 | } 54 | 55 | pub fn deserialize_known_length( 56 | buf: &mut B, 57 | typ: wire_format::Type, 58 | expected_wire_format: wire_format::Type, 59 | msg_name: &'static str, 60 | field_number: u32, 61 | ) -> io::Result { 62 | ensure_wire_format(typ, expected_wire_format, msg_name, field_number)?; 63 | let mut val: T = Default::default(); 64 | val.deserialize(buf)?; 65 | Ok(val) 66 | } 67 | 68 | pub fn serialize_field( 69 | w: &mut W, 70 | val: &T, 71 | field_number: u32, 72 | wire_format: wire_format::Type, 73 | ) -> io::Result<()> { 74 | wire_format::write(field_number, wire_format, w)?; 75 | if let wire_format::Type::LengthDelimited = wire_format { 76 | let l = val.compute_size(); 77 | varint::write(l as u64, w)?; 78 | } 79 | val.serialize(w) 80 | } 81 | 82 | pub fn serialize_scalar( 83 | w: &mut W, 84 | val: &T, 85 | field_number: u32, 86 | wire_format: wire_format::Type, 87 | ) -> io::Result<()> { 88 | if *val != T::default() { 89 | serialize_field(w, val, field_number, wire_format)?; 90 | } 91 | Ok(()) 92 | } 93 | 94 | pub fn compute_size_field(val: &T, field_number: u32, wire_format: wire_format::Type) -> usize { 95 | let mut size = wire_format::serialized_length(field_number); 96 | let l = val.compute_size(); 97 | if let wire_format::Type::LengthDelimited = wire_format { 98 | size += varint::serialized_length(l as u64); 99 | } 100 | size + l 101 | } 102 | 103 | pub fn compute_size_scalar(val: &T, field_number: u32, wire_format: wire_format::Type) -> usize { 104 | if *val != T::default() { 105 | compute_size_field(val, field_number, wire_format) 106 | } else { 107 | 0 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /pb-jelly/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![warn(rust_2018_idioms)] 2 | #![allow(clippy::cast_sign_loss)] 3 | #![allow(clippy::cast_possible_truncation)] 4 | #![allow(clippy::cast_possible_wrap)] 5 | 6 | use std::any::Any; 7 | use std::collections::BTreeMap; 8 | use std::default::Default; 9 | use std::fmt::{ 10 | self, 11 | Debug, 12 | }; 13 | use std::io::{ 14 | Cursor, 15 | Error, 16 | ErrorKind, 17 | Result, 18 | Write, 19 | }; 20 | 21 | use bytes::buf::{ 22 | Buf, 23 | BufMut, 24 | }; 25 | 26 | pub mod erased; 27 | pub mod extensions; 28 | pub mod helpers; 29 | pub mod varint; 30 | pub mod wire_format; 31 | 32 | mod buffer; 33 | pub use crate::buffer::{ 34 | type_is, 35 | CopyWriter, 36 | Lazy, 37 | PbBuffer, 38 | PbBufferReader, 39 | PbBufferWriter, 40 | }; 41 | 42 | mod base_types; 43 | pub use crate::base_types::{ 44 | ClosedProtoEnum, 45 | Fixed32, 46 | Fixed64, 47 | OpenProtoEnum, 48 | ProtoEnum, 49 | Sfixed32, 50 | Sfixed64, 51 | Signed32, 52 | Signed64, 53 | }; 54 | 55 | mod descriptor; 56 | pub use crate::descriptor::{ 57 | FieldDescriptor, 58 | Label, 59 | MessageDescriptor, 60 | OneofDescriptor, 61 | }; 62 | 63 | pub mod reflection; 64 | pub use crate::reflection::Reflection; 65 | 66 | #[cfg(test)] 67 | mod tests; 68 | 69 | /// Trait implemented by all the messages defined in proto files and base datatypes 70 | /// like string, bytes, etc. The exact details of this trait is implemented for messages 71 | /// and base types can be found at - 72 | pub trait Message: PartialEq + Default + Debug + Any { 73 | /// Returns the `MessageDescriptor` for this message, if this is not a primitive type. 74 | fn descriptor(&self) -> Option { 75 | None 76 | } 77 | 78 | /// Computes the number of bytes a message will take when serialized. This does not 79 | /// include number of bytes required for tag+wire_format or the bytes used to represent 80 | /// length of the message in case of LengthDelimited messages/types. 81 | fn compute_size(&self) -> usize; 82 | 83 | /// Computes the number of bytes in all grpc slices. 84 | /// This information is used to optimize memory allocations in zero-copy encoding. 85 | fn compute_grpc_slices_size(&self) -> usize { 86 | 0 87 | } 88 | 89 | /// Serializes the message to the writer. 90 | fn serialize(&self, w: &mut W) -> Result<()>; 91 | 92 | /// Reads the message from the blob reader, copying as necessary. 93 | fn deserialize(&mut self, r: &mut B) -> Result<()>; 94 | 95 | /// Helper method for serializing a message to a [Vec]. 96 | #[inline] 97 | fn serialize_to_vec(&self) -> Vec { 98 | let size = self.compute_size() as usize; 99 | let mut out = Vec::with_capacity(size); 100 | // We know that a Cursor> only fails on u32 overflow 101 | // https://doc.rust-lang.org/src/std/io/cursor.rs.html#295 102 | self.serialize(&mut Cursor::new(&mut out)).expect("Vec u32 overflow"); 103 | debug_assert_eq!(out.len(), size); 104 | out 105 | } 106 | 107 | /// Helper method for serializing a message to an arbitrary [Write]. 108 | /// 109 | /// If there are [Lazy] fields in the message, their contents will be copied out. 110 | #[inline] 111 | fn serialize_to_writer(&self, writer: &mut W) -> Result<()> { 112 | let mut copy_writer = CopyWriter { writer }; 113 | self.serialize(&mut copy_writer)?; 114 | Ok(()) 115 | } 116 | 117 | /// Helper method for deserializing a message from a u8 slice. 118 | /// 119 | /// This will error if there are any [Lazy] fields in the message. 120 | #[inline] 121 | fn deserialize_from_slice(slice: &[u8]) -> Result { 122 | let mut buf = Cursor::new(slice); 123 | let mut m = Self::default(); 124 | m.deserialize(&mut buf)?; 125 | Ok(m) 126 | } 127 | } 128 | 129 | pub fn ensure_wire_format( 130 | format: wire_format::Type, 131 | expected: wire_format::Type, 132 | msg_name: &str, 133 | field_number: u32, 134 | ) -> Result<()> { 135 | if format != expected { 136 | return Err(Error::new( 137 | ErrorKind::Other, 138 | format!( 139 | "expected wire_format {:?}, found {:?}, at {:?}:{:?}", 140 | expected, format, msg_name, field_number 141 | ), 142 | )); 143 | } 144 | 145 | Ok(()) 146 | } 147 | 148 | pub fn unexpected_eof() -> Error { 149 | Error::new(ErrorKind::UnexpectedEof, "unexpected EOF") 150 | } 151 | 152 | // XXX: arguably this should not impl PartialEq since we cannot canonicalize the unparsed field contents 153 | #[derive(Clone, Default, PartialEq)] 154 | pub struct Unrecognized { 155 | by_field_number: BTreeMap>, 156 | } 157 | 158 | impl fmt::Debug for Unrecognized { 159 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 160 | f.debug_map() 161 | .entries(self.by_field_number.keys().map(|k| (k, ..))) 162 | .finish() 163 | } 164 | } 165 | 166 | impl Unrecognized { 167 | pub fn new() -> Self { 168 | Self::default() 169 | } 170 | 171 | pub fn serialize(&self, unrecognized_buf: &mut impl PbBufferWriter) -> Result<()> { 172 | // Write out sorted by field number 173 | for serialized_field in self.by_field_number.values() { 174 | unrecognized_buf.write_all(&serialized_field)?; 175 | } 176 | Ok(()) 177 | } 178 | 179 | pub fn compute_size(&self) -> usize { 180 | self.by_field_number.values().map(|v| v.len()).sum() 181 | } 182 | 183 | pub fn gather(&mut self, field_number: u32, typ: wire_format::Type, buf: &mut B) -> Result<()> { 184 | let unrecognized_buf = self.by_field_number.entry(field_number).or_default(); 185 | 186 | wire_format::write(field_number, typ, unrecognized_buf)?; 187 | let advance = match typ { 188 | wire_format::Type::Varint => { 189 | if let Some(num) = varint::read(buf)? { 190 | varint::write(num, unrecognized_buf)?; 191 | } else { 192 | return Err(unexpected_eof()); 193 | }; 194 | 195 | 0 196 | }, 197 | wire_format::Type::Fixed64 => 8, 198 | wire_format::Type::Fixed32 => 4, 199 | wire_format::Type::LengthDelimited => match varint::read(buf)? { 200 | Some(n) => { 201 | varint::write(n, unrecognized_buf)?; 202 | n as usize 203 | }, 204 | None => return Err(unexpected_eof()), 205 | }, 206 | }; 207 | 208 | if buf.remaining() < advance { 209 | return Err(unexpected_eof()); 210 | } 211 | 212 | unrecognized_buf.put(buf.take(advance)); 213 | 214 | Ok(()) 215 | } 216 | 217 | pub(crate) fn get_singular_field(&self, field_number: u32) -> Option<(&[u8], wire_format::Type)> { 218 | let mut buf = Cursor::new(&self.by_field_number.get(&field_number)?[..]); 219 | let mut result = None; 220 | // It's technically legal for a singular field to occur multiple times on the wire, 221 | // so skip over all but the last instance. 222 | while let Some((_field_number, wire_format)) = 223 | wire_format::read(&mut buf).expect("self.by_field_number malformed") 224 | { 225 | result = Some((&buf.get_ref()[buf.position() as usize..], wire_format)); 226 | 227 | skip(wire_format, &mut buf).expect("self.by_field_number malformed"); 228 | } 229 | result 230 | } 231 | 232 | pub(crate) fn get_fields(&self, field_number: u32) -> &[u8] { 233 | self.by_field_number.get(&field_number).map_or(&[], Vec::as_ref) 234 | } 235 | } 236 | 237 | pub fn skip(typ: wire_format::Type, buf: &mut B) -> Result<()> { 238 | let advance = match typ { 239 | wire_format::Type::Varint => { 240 | if varint::read(buf)?.is_none() { 241 | return Err(unexpected_eof()); 242 | }; 243 | 244 | 0 245 | }, 246 | wire_format::Type::Fixed64 => 8, 247 | wire_format::Type::Fixed32 => 4, 248 | wire_format::Type::LengthDelimited => match varint::read(buf)? { 249 | Some(n) => n as usize, 250 | None => return Err(unexpected_eof()), 251 | }, 252 | }; 253 | 254 | if buf.remaining() < advance { 255 | return Err(unexpected_eof()); 256 | } 257 | 258 | buf.advance(advance); 259 | Ok(()) 260 | } 261 | 262 | pub fn ensure_split(buf: &mut B, len: usize) -> Result { 263 | if buf.remaining() < len { 264 | Err(unexpected_eof()) 265 | } else { 266 | Ok(buf.split(len)) 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /pb-jelly/src/reflection.rs: -------------------------------------------------------------------------------- 1 | use crate::erased; 2 | 3 | /// A mutable borrow of a field, or `Empty` if the field does not have a value. 4 | #[non_exhaustive] 5 | pub enum FieldMut<'a> { 6 | /// Mutable borrow of the field. 7 | Value(&'a mut dyn Reflection), 8 | // TODO: Would be nice to support this, but some more thought would need to be put into what 9 | // the API for it looks like. 10 | // Repeated(&'a mut Vec), 11 | /// An empty oneof variant. 12 | Empty, 13 | } 14 | 15 | pub trait Reflection: erased::Message { 16 | /// Returns the name of the field that is set inside a oneof group. 17 | /// 18 | /// If no field is set, returns None. 19 | /// 20 | /// # Panics 21 | /// If the name given is not the name of a oneof field in this message. 22 | fn which_one_of(&self, _oneof_name: &str) -> Option<&'static str> { 23 | unimplemented!("proto reflection is not implemented for this type") 24 | } 25 | 26 | /// Returns a mutable borrow of the given field. 27 | /// 28 | /// If the field given is part of a oneof group, and that oneof group is not currently set to 29 | /// this variant, this will mutate the current message, setting the oneof group to the variant 30 | /// given by this field name with the default value. 31 | /// 32 | /// # Panics 33 | /// If the name given is not the name of a field in this message. 34 | fn get_field_mut(&mut self, _field_name: &str) -> FieldMut<'_> { 35 | unimplemented!("proto reflection is not implemented for this type") 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /pb-jelly/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | use std::any::Any; 2 | use std::io::{ 3 | Error, 4 | ErrorKind, 5 | Result, 6 | Write, 7 | }; 8 | use std::rc::Rc; 9 | 10 | use bytes::{ 11 | Buf, 12 | BufMut, 13 | }; 14 | 15 | use super::{ 16 | ensure_split, 17 | ensure_wire_format, 18 | skip, 19 | type_is, 20 | Lazy, 21 | Message, 22 | PbBuffer, 23 | PbBufferReader, 24 | PbBufferWriter, 25 | }; 26 | use crate::{ 27 | varint, 28 | wire_format, 29 | }; 30 | 31 | /// A wrapper around a `Vec` which owns its contents. 32 | #[derive(Clone, PartialEq, Debug, Default)] 33 | struct VecReader { 34 | contents: Rc>, 35 | position: usize, 36 | end: usize, 37 | } 38 | 39 | impl AsRef<[u8]> for VecReader { 40 | fn as_ref(&self) -> &[u8] { 41 | &self.contents[self.position..self.end] 42 | } 43 | } 44 | 45 | impl PbBuffer for VecReader { 46 | fn len(&self) -> usize { 47 | self.remaining() 48 | } 49 | fn copy_to_writer(&self, writer: &mut W) -> Result<()> { 50 | writer.write_all(self.as_ref()) 51 | } 52 | fn copy_from_reader(reader: &mut B) -> Result { 53 | let mut v = vec![]; 54 | v.put(reader); 55 | Ok(v.into()) 56 | } 57 | } 58 | 59 | impl From> for VecReader { 60 | fn from(v: Vec) -> Self { 61 | VecReader { 62 | position: 0, 63 | end: v.len(), 64 | contents: Rc::new(v), 65 | } 66 | } 67 | } 68 | 69 | impl Buf for VecReader { 70 | fn remaining(&self) -> usize { 71 | debug_assert!(self.position <= self.end); 72 | self.end - self.position 73 | } 74 | 75 | fn chunk(&self) -> &[u8] { 76 | self.as_ref() 77 | } 78 | 79 | fn advance(&mut self, cnt: usize) { 80 | debug_assert!(self.position + cnt <= self.end); 81 | self.position += cnt; 82 | } 83 | } 84 | 85 | impl PbBufferReader for VecReader { 86 | fn read_buffer(&mut self) -> Result { 87 | if let Some(cast) = type_is::() { 88 | Ok(cast(self.clone())) 89 | } else { 90 | B::copy_from_reader(self) 91 | } 92 | } 93 | 94 | fn split(&mut self, at: usize) -> Self { 95 | debug_assert!(self.position + at <= self.end); 96 | 97 | let pos = self.position; 98 | self.position += at; 99 | 100 | Self { 101 | contents: Rc::clone(&self.contents), 102 | position: pos, 103 | end: self.position, 104 | } 105 | } 106 | } 107 | 108 | /// A kind of dumb zero-copy implementation in memory. 109 | #[derive(Default, Debug)] 110 | struct VecWriter { 111 | contents: Vec, 112 | } 113 | 114 | impl VecWriter { 115 | fn serialized(&self) -> Vec { 116 | let mut out = vec![]; 117 | for content in &self.contents { 118 | out.extend_from_slice(content.as_ref()); 119 | } 120 | out 121 | } 122 | } 123 | 124 | impl Write for VecWriter { 125 | fn write(&mut self, buf: &[u8]) -> Result { 126 | self.contents.push(Vec::from(buf).into()); 127 | Ok(buf.len()) 128 | } 129 | 130 | fn flush(&mut self) -> Result<()> { 131 | Ok(()) 132 | } 133 | } 134 | 135 | impl PbBufferWriter for VecWriter { 136 | /// Attempt to write non-deserialized buf into [Self] 137 | fn write_buffer(&mut self, buf: &B) -> Result<()> { 138 | if let Some(buf) = (buf as &dyn Any).downcast_ref::() { 139 | self.contents.push(buf.clone()); 140 | Ok(()) 141 | } else { 142 | Err(Error::new(ErrorKind::Other, "Can't zero-copy non-VecReader")) 143 | } 144 | } 145 | } 146 | 147 | #[derive(Debug, PartialEq, Default, Clone)] 148 | struct TestMessage { 149 | normal_data: Vec, 150 | payload: Lazy, 151 | } 152 | 153 | impl Message for TestMessage { 154 | fn compute_size(&self) -> usize { 155 | let mut size = wire_format::serialized_length(1); 156 | let l = self.normal_data.compute_size(); 157 | size += varint::serialized_length(l as u64); 158 | size += l; 159 | 160 | size += wire_format::serialized_length(2); 161 | let l = self.payload.compute_size(); 162 | size += varint::serialized_length(l as u64); 163 | size += l; 164 | 165 | size 166 | } 167 | 168 | fn compute_grpc_slices_size(&self) -> usize { 169 | self.normal_data.compute_grpc_slices_size() + self.payload.compute_grpc_slices_size() 170 | } 171 | 172 | fn serialize(&self, w: &mut W) -> Result<()> { 173 | wire_format::write(1, wire_format::Type::LengthDelimited, w)?; 174 | let l = self.normal_data.compute_size(); 175 | varint::write(l as u64, w)?; 176 | self.normal_data.serialize(w)?; 177 | 178 | wire_format::write(2, wire_format::Type::LengthDelimited, w)?; 179 | let l = self.payload.compute_size(); 180 | varint::write(l as u64, w)?; 181 | self.payload.serialize(w)?; 182 | 183 | Ok(()) 184 | } 185 | 186 | fn deserialize(&mut self, mut buf: &mut B) -> Result<()> { 187 | while let Some((field_number, typ)) = wire_format::read(&mut buf)? { 188 | match field_number { 189 | 1 => { 190 | ensure_wire_format(typ, wire_format::Type::LengthDelimited, "normal_data", 1)?; 191 | let len = varint::ensure_read(&mut buf)?; 192 | let mut next = ensure_split(buf, len as usize)?; 193 | let mut val: Vec = Default::default(); 194 | val.deserialize(&mut next)?; 195 | self.normal_data = val; 196 | }, 197 | 2 => { 198 | ensure_wire_format(typ, wire_format::Type::LengthDelimited, "payload", 2)?; 199 | let len = varint::ensure_read(&mut buf)?; 200 | let mut next = ensure_split(buf, len as usize)?; 201 | let mut val: Lazy = Default::default(); 202 | val.deserialize(&mut next)?; 203 | self.payload = val; 204 | }, 205 | _ => { 206 | skip(typ, &mut buf)?; 207 | }, 208 | } 209 | } 210 | 211 | Ok(()) 212 | } 213 | } 214 | 215 | #[test] 216 | fn test_deserialize_message_from_vec_reader() { 217 | let mut message = TestMessage::default(); 218 | message.normal_data = vec![1, 2, 3]; 219 | message.payload = Lazy::new(vec![4, 5, 6].into()); 220 | 221 | let serialized = message.serialize_to_vec(); 222 | assert_eq!(serialized.len(), message.compute_size()); 223 | assert_eq!(serialized, [10, 3, 1, 2, 3, 18, 3, 4, 5, 6]); 224 | 225 | // Deserializing the message from a slice should succeed, even though it incurs a copy. 226 | assert_eq!( 227 | TestMessage::deserialize_from_slice(&serialized).expect("deserialization failed"), 228 | message 229 | ); 230 | 231 | // Deserializing from a `VecReader` should succeed and capture a reference to the reader. 232 | let reader = VecReader::from(serialized); 233 | let mut deserialized = TestMessage::default(); 234 | deserialized.deserialize(&mut reader.clone()).unwrap(); 235 | 236 | assert_eq!(deserialized.normal_data, message.normal_data); 237 | let deserialized_payload = deserialized.payload.into_buffer(); 238 | assert!( 239 | Rc::ptr_eq(&deserialized_payload.contents, &reader.contents), 240 | "Should reference the original buffer" 241 | ); 242 | assert_eq!(deserialized_payload.as_ref(), message.payload.into_buffer().as_ref()); 243 | } 244 | 245 | #[test] 246 | fn test_serialize_message_into_vec_writer() { 247 | let payload: VecReader = vec![4, 5, 6].into(); 248 | let mut message = TestMessage::default(); 249 | message.normal_data = vec![1, 2, 3]; 250 | message.payload = Lazy::new(payload.clone()); 251 | 252 | // Serialize to a `VecWriter`. 253 | let mut out = VecWriter::default(); 254 | message.serialize(&mut out).unwrap(); 255 | 256 | // One of the segments of `out` should reference the same buffer as `payload`. 257 | assert_eq!( 258 | out.contents 259 | .iter() 260 | .filter(|buf| Rc::ptr_eq(&buf.contents, &payload.contents)) 261 | .count(), 262 | 1 263 | ); 264 | 265 | let serialized = out.serialized(); 266 | assert_eq!( 267 | TestMessage::deserialize_from_slice(&serialized).expect("deserialization failed"), 268 | message 269 | ); 270 | assert_eq!(serialized, [10, 3, 1, 2, 3, 18, 3, 4, 5, 6]); 271 | } 272 | -------------------------------------------------------------------------------- /pb-jelly/src/varint.rs: -------------------------------------------------------------------------------- 1 | use std::cmp::min; 2 | use std::io::{ 3 | self, 4 | Write, 5 | }; 6 | 7 | use bytes::Buf; 8 | 9 | #[inline] 10 | pub const fn serialized_length(mut val: u64) -> usize { 11 | let mut ans = 1; 12 | while val & !0x7Fu64 != 0 { 13 | val >>= 7; 14 | ans += 1; 15 | } 16 | 17 | ans 18 | } 19 | 20 | #[inline] 21 | pub fn write(mut val: u64, w: &mut W) -> io::Result<()> { 22 | // the maximum length in bytes of an encoded (64-bit) varint 23 | const MAX_LEN: usize = serialized_length(u64::MAX); 24 | 25 | let mut len = 0; 26 | let mut buf = [0; MAX_LEN]; 27 | loop { 28 | if val & !0x7Fu64 == 0 { 29 | buf[len] = val as u8; 30 | len += 1; 31 | break; 32 | } else { 33 | buf[len] = ((val & 0x7F) | 0x80) as u8; 34 | val >>= 7; 35 | len += 1; 36 | } 37 | } 38 | w.write_all(&buf[..len]) 39 | } 40 | 41 | #[inline] 42 | pub fn read(buf: &mut B) -> io::Result> { 43 | if buf.remaining() == 0 { 44 | return Ok(None); 45 | } 46 | 47 | // Find offset of first byte with MSB not set 48 | let idx = match buf 49 | .chunk() 50 | .iter() 51 | .enumerate() 52 | .find(|&(_, val)| *val < 0x80 as u8) 53 | .map(|(idx, _)| idx) 54 | { 55 | Some(idx) => idx, 56 | // Fallback to per-byte read if data is span across multiple slices. 57 | None => return read_slow(buf), 58 | }; 59 | 60 | let varint = { 61 | let buf = &buf.chunk()[..=idx]; 62 | 63 | let mut r: u64 = 0; 64 | for byte in buf.iter().rev() { 65 | r = (r << 7) | u64::from(byte & 0x7f); 66 | } 67 | 68 | r 69 | }; 70 | 71 | buf.advance(idx + 1); 72 | Ok(Some(varint)) 73 | } 74 | 75 | fn read_slow(buf: &mut B) -> io::Result> { 76 | let mut r: u64 = 0; 77 | for index in 0..min(10, buf.remaining()) { 78 | let byte = buf.get_u8(); 79 | r |= u64::from(byte & 0x7f) << (index * 7); 80 | if byte < 0x80 { 81 | return Ok(Some(r)); 82 | } 83 | } 84 | Err(super::unexpected_eof()) 85 | } 86 | 87 | #[inline] 88 | pub fn ensure_read(buf: &mut B) -> io::Result { 89 | match read(buf)? { 90 | Some(n) => Ok(n), 91 | None => Err(super::unexpected_eof()), 92 | } 93 | } 94 | 95 | #[test] 96 | fn test_basic() { 97 | use std::io::Cursor; 98 | 99 | use crate::Message; 100 | 101 | let from_vec = |vec| read(&mut Cursor::new(vec)).unwrap().unwrap(); 102 | 103 | let from_vec_split = |mut first_vec: Vec| { 104 | let at = first_vec.len() / 2; 105 | let second_vec = first_vec.split_off(at); 106 | read(&mut Cursor::new(first_vec).chain(Cursor::new(second_vec))) 107 | .unwrap() 108 | .unwrap() 109 | }; 110 | 111 | assert_eq!(1000.serialize_to_vec(), vec![232, 7]); 112 | assert_eq!(from_vec(vec![232, 7]), 1000); 113 | assert_eq!(from_vec_split(vec![232, 7]), 1000); 114 | 115 | let minus1 = vec![255, 255, 255, 255, 255, 255, 255, 255, 255, 1]; 116 | assert_eq!((-1 as i32).serialize_to_vec(), minus1); 117 | assert_eq!((-1 as i64).serialize_to_vec(), minus1); 118 | assert_eq!(from_vec(minus1.clone()) as i32, -1); 119 | assert_eq!(from_vec(minus1.clone()) as i64, -1); 120 | assert_eq!(from_vec_split(minus1.clone()) as i32, -1); 121 | assert_eq!(from_vec_split(minus1) as i64, -1); 122 | 123 | let minus1000 = vec![152, 248, 255, 255, 255, 255, 255, 255, 255, 1]; 124 | assert_eq!((-1000 as i32).serialize_to_vec(), minus1000); 125 | assert_eq!((-1000 as i64).serialize_to_vec(), minus1000); 126 | assert_eq!(from_vec(minus1000.clone()) as i32, -1000); 127 | assert_eq!(from_vec(minus1000.clone()) as i64, -1000); 128 | assert_eq!(from_vec_split(minus1000.clone()) as i32, -1000); 129 | assert_eq!(from_vec_split(minus1000) as i64, -1000); 130 | } 131 | -------------------------------------------------------------------------------- /pb-jelly/src/wire_format.rs: -------------------------------------------------------------------------------- 1 | use std::io::{ 2 | self, 3 | Write, 4 | }; 5 | 6 | use bytes::Buf; 7 | 8 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] 9 | pub enum Type { 10 | Varint = 0, 11 | Fixed64 = 1, 12 | LengthDelimited = 2, 13 | Fixed32 = 5, 14 | } 15 | 16 | impl Type { 17 | const fn as_u32(self) -> u32 { 18 | match self { 19 | Type::Varint => 0, 20 | Type::Fixed64 => 1, 21 | Type::LengthDelimited => 2, 22 | Type::Fixed32 => 5, 23 | } 24 | } 25 | 26 | fn from_u32(val: u32) -> io::Result { 27 | match val { 28 | 0 => Ok(Type::Varint), 29 | 1 => Ok(Type::Fixed64), 30 | 2 => Ok(Type::LengthDelimited), 31 | 5 => Ok(Type::Fixed32), 32 | v => Err(io::Error::new( 33 | io::ErrorKind::InvalidData, 34 | format!("Unexpected tag type: {}", v), 35 | )), 36 | } 37 | } 38 | } 39 | 40 | #[inline] 41 | pub const fn serialized_length(field_number: u32) -> usize { 42 | super::varint::serialized_length((field_number << 3) as u64) 43 | } 44 | 45 | #[inline] 46 | pub fn write(field_number: u32, typ: Type, w: &mut W) -> io::Result<()> { 47 | let encoded = (field_number << 3) | typ.as_u32(); 48 | super::varint::write(u64::from(encoded), w) 49 | } 50 | 51 | #[inline] 52 | pub fn read(buf: &mut B) -> io::Result> { 53 | let encoded = match super::varint::read(buf)? { 54 | Some(v) => v, 55 | None => return Ok(None), 56 | }; 57 | 58 | let field_number = (encoded >> 3) as u32; 59 | let typ = Type::from_u32((encoded & 0x7) as u32)?; 60 | Ok(Some((field_number, typ))) 61 | } 62 | -------------------------------------------------------------------------------- /pb-test/.gitignore: -------------------------------------------------------------------------------- 1 | !/src/gen/**/*.rs 2 | -------------------------------------------------------------------------------- /pb-test/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pb-test" 3 | version = "0.1.0" 4 | authors = ["Rajat Goel ", "Nipunn Koorapati ", "Parker Timmerman "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | bytes = "1.0" 11 | pb-jelly = { path = "../pb-jelly" } 12 | pretty_assertions = "0.6.1" 13 | proto_pbtest = { path = "gen/pb-jelly/proto_pbtest" } 14 | walkdir = "2.3.1" 15 | 16 | # only used when benchmarking PROST! 17 | prost = { version = "0.6", optional = true } 18 | # only used when benchmarking rust-protobuf 19 | protobuf = { version = "3.3", features = ["bytes"], optional = true } 20 | 21 | [dev-dependencies] 22 | compact_str = { version = "0.5", features = ["serde"] } 23 | serde = { version = "1", features = ["derive"] } 24 | serde_json = "1" 25 | 26 | [features] 27 | bench_prost = ["prost"] 28 | bench_rust_protobuf = ["protobuf"] 29 | 30 | # Override pb-jelly dependency for generated crates as well 31 | [patch.crates-io] 32 | pb-jelly = { path = "../pb-jelly" } 33 | -------------------------------------------------------------------------------- /pb-test/README.md: -------------------------------------------------------------------------------- 1 | # `pb-test` 2 | ###### Testing 1, 2, 3... can you hear me? 3 | 4 | This crate has integration tests to assert we properly serialize/deserialize proto messages between [`Rust`](https://www.rust-lang.org/) and [`Go`](https://golang.org/). Checked in under `/bin` are the Go serialized bytes 5 | of our test messages, we use these binary files to assert proper ser/de without having to run Go. 6 | 7 | Also in this crate are benchmarks around deserialization, utilizing zero-copy deserialization where we can. We benchmark our implementation of zero-copy versus 8 | non-zero-copy, we also benchmark against [`PROST!`](https://github.com/danburkert/prost) and [`rust-protobuf`](https://github.com/stepancheg/rust-protobuf). 9 | -------------------------------------------------------------------------------- /pb-test/bin/pbtest.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version1enum.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version1oneof.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version1oneof_none_null.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version2enum.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version2oneof.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest2_version2oneof_none_null.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_default_oneof.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3_default_oneof.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_err_if_default.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3_err_if_default.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_err_if_default_non_default.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_missing_oneof.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3_missing_oneof.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_repeated_err_if_default.bin: -------------------------------------------------------------------------------- 1 | 2 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_repeated_err_if_default_non_default.bin: -------------------------------------------------------------------------------- 1 | 2 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version1.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version1enum.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3_version1enum.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version1oneof.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version1oneof_none_null.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest3_version2.bin -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version2enum.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version2oneof.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest3_version2oneof_none_null.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /pb-test/bin/pbtest_version1.bin: -------------------------------------------------------------------------------- 1 | 2 | abc -------------------------------------------------------------------------------- /pb-test/bin/pbtest_version2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/pb-jelly/93831c69d9d2a0ba17be0ff0aa12cd1656d4cae7/pb-test/bin/pbtest_version2.bin -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_google/Cargo.toml.expected: -------------------------------------------------------------------------------- 1 | # @generated, do not edit 2 | [package] 3 | name = "proto_google" 4 | version = "0.0.1" 5 | edition = "2018" 6 | 7 | [dependencies] 8 | lazy_static = { version = "1.4.0" } 9 | pb-jelly = { version = "0.0.17" } 10 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_google/src/lib.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | 3 | #![allow(irrefutable_let_patterns)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(non_snake_case)] 6 | #![allow(non_upper_case_globals)] 7 | #![allow(unused_imports)] 8 | #![allow(unused_variables)] 9 | #![allow(irrefutable_let_patterns)] 10 | #![allow(rustdoc::broken_intra_doc_links)] 11 | 12 | // Modules are generated based on the naming conventions of protobuf, which might cause "module inception" 13 | #![allow(clippy::module_inception)] 14 | // This is all generated code, so "manually" implementing derivable impls is okay 15 | #![allow(clippy::derivable_impls)] 16 | // For enums with many variants, the matches!(...) macro isn't obviously better 17 | #![allow(clippy::match_like_matches_macro)] 18 | // Used by extension codegen 19 | #![allow(clippy::redundant_closure)] 20 | // TODO: Ideally we don't allow this 21 | #![allow(clippy::option_as_ref_deref)] 22 | // TODO: Ideally we don't allow this 23 | #![allow(clippy::match_single_binding)] 24 | 25 | 26 | pub mod protobuf; 27 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_google/src/protobuf/empty.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | /// A generic empty message that you can re-use to avoid defining duplicated 3 | /// empty messages in your APIs. A typical example is to use it as the request 4 | /// or the response type of an API method. For instance: 5 | 6 | /// service Foo { 7 | /// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); 8 | /// } 9 | 10 | /// The JSON representation for `Empty` is empty JSON object `{}`. 11 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 12 | pub struct Empty { 13 | } 14 | impl ::std::default::Default for Empty { 15 | fn default() -> Self { 16 | Empty { 17 | } 18 | } 19 | } 20 | ::lazy_static::lazy_static! { 21 | pub static ref Empty_default: Empty = Empty::default(); 22 | } 23 | impl ::pb_jelly::Message for Empty { 24 | fn descriptor(&self) -> ::std::option::Option<::pb_jelly::MessageDescriptor> { 25 | Some(::pb_jelly::MessageDescriptor { 26 | name: "Empty", 27 | full_name: "google.protobuf.Empty", 28 | fields: &[ 29 | ], 30 | oneofs: &[ 31 | ], 32 | }) 33 | } 34 | fn compute_size(&self) -> usize { 35 | 0usize 36 | } 37 | fn serialize(&self, w: &mut W) -> ::std::io::Result<()> { 38 | Ok(()) 39 | } 40 | fn deserialize(&mut self, mut buf: &mut B) -> ::std::io::Result<()> { 41 | while let Some((field_number, typ)) = ::pb_jelly::wire_format::read(&mut buf)? { 42 | match field_number { 43 | _ => { 44 | ::pb_jelly::skip(typ, &mut buf)?; 45 | } 46 | } 47 | } 48 | Ok(()) 49 | } 50 | } 51 | impl ::pb_jelly::Reflection for Empty { 52 | fn which_one_of(&self, oneof_name: &str) -> ::std::option::Option<&'static str> { 53 | match oneof_name { 54 | _ => { 55 | panic!("unknown oneof name given"); 56 | } 57 | } 58 | } 59 | fn get_field_mut(&mut self, field_name: &str) -> ::pb_jelly::reflection::FieldMut<'_> { 60 | match field_name { 61 | _ => { 62 | panic!("unknown field name given") 63 | } 64 | } 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_google/src/protobuf/mod.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | 3 | pub mod empty; 4 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_nopackage/Cargo.toml.expected: -------------------------------------------------------------------------------- 1 | # @generated, do not edit 2 | [package] 3 | name = "proto_nopackage" 4 | version = "0.0.1" 5 | edition = "2018" 6 | 7 | [dependencies] 8 | lazy_static = { version = "1.4.0" } 9 | pb-jelly = { version = "0.0.17" } 10 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_nopackage/src/lib.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | 3 | #![allow(irrefutable_let_patterns)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(non_snake_case)] 6 | #![allow(non_upper_case_globals)] 7 | #![allow(unused_imports)] 8 | #![allow(unused_variables)] 9 | #![allow(irrefutable_let_patterns)] 10 | #![allow(rustdoc::broken_intra_doc_links)] 11 | 12 | // Modules are generated based on the naming conventions of protobuf, which might cause "module inception" 13 | #![allow(clippy::module_inception)] 14 | // This is all generated code, so "manually" implementing derivable impls is okay 15 | #![allow(clippy::derivable_impls)] 16 | // For enums with many variants, the matches!(...) macro isn't obviously better 17 | #![allow(clippy::match_like_matches_macro)] 18 | // Used by extension codegen 19 | #![allow(clippy::redundant_closure)] 20 | // TODO: Ideally we don't allow this 21 | #![allow(clippy::option_as_ref_deref)] 22 | // TODO: Ideally we don't allow this 23 | #![allow(clippy::match_single_binding)] 24 | 25 | 26 | pub mod proto_nopackage; 27 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_nopackage/src/proto_nopackage.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 3 | pub struct NoPackage { 4 | pub field: ::std::string::String, 5 | } 6 | impl ::std::default::Default for NoPackage { 7 | fn default() -> Self { 8 | NoPackage { 9 | field: ::std::default::Default::default(), 10 | } 11 | } 12 | } 13 | ::lazy_static::lazy_static! { 14 | pub static ref NoPackage_default: NoPackage = NoPackage::default(); 15 | } 16 | impl ::pb_jelly::Message for NoPackage { 17 | fn descriptor(&self) -> ::std::option::Option<::pb_jelly::MessageDescriptor> { 18 | Some(::pb_jelly::MessageDescriptor { 19 | name: "NoPackage", 20 | full_name: "NoPackage", 21 | fields: &[ 22 | ::pb_jelly::FieldDescriptor { 23 | name: "field", 24 | full_name: "NoPackage.field", 25 | index: 0, 26 | number: 1, 27 | typ: ::pb_jelly::wire_format::Type::LengthDelimited, 28 | label: ::pb_jelly::Label::Optional, 29 | oneof_index: None, 30 | }, 31 | ], 32 | oneofs: &[ 33 | ], 34 | }) 35 | } 36 | fn compute_size(&self) -> usize { 37 | let mut size = 0usize; 38 | size += ::pb_jelly::helpers::compute_size_scalar::<::std::string::String>(&self.field, 1, ::pb_jelly::wire_format::Type::LengthDelimited); 39 | size 40 | } 41 | fn serialize(&self, w: &mut W) -> ::std::io::Result<()> { 42 | ::pb_jelly::helpers::serialize_scalar::(w, &self.field, 1, ::pb_jelly::wire_format::Type::LengthDelimited)?; 43 | Ok(()) 44 | } 45 | fn deserialize(&mut self, mut buf: &mut B) -> ::std::io::Result<()> { 46 | while let Some((field_number, typ)) = ::pb_jelly::wire_format::read(&mut buf)? { 47 | match field_number { 48 | 1 => { 49 | let val = ::pb_jelly::helpers::deserialize_length_delimited::(buf, typ, "NoPackage", 1)?; 50 | self.field = val; 51 | } 52 | _ => { 53 | ::pb_jelly::skip(typ, &mut buf)?; 54 | } 55 | } 56 | } 57 | Ok(()) 58 | } 59 | } 60 | impl ::pb_jelly::Reflection for NoPackage { 61 | fn which_one_of(&self, oneof_name: &str) -> ::std::option::Option<&'static str> { 62 | match oneof_name { 63 | _ => { 64 | panic!("unknown oneof name given"); 65 | } 66 | } 67 | } 68 | fn get_field_mut(&mut self, field_name: &str) -> ::pb_jelly::reflection::FieldMut<'_> { 69 | match field_name { 70 | "field" => { 71 | ::pb_jelly::reflection::FieldMut::Value(&mut self.field) 72 | } 73 | _ => { 74 | panic!("unknown field name given") 75 | } 76 | } 77 | } 78 | } 79 | 80 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_pbtest/Cargo.toml.expected: -------------------------------------------------------------------------------- 1 | # @generated, do not edit 2 | [package] 3 | name = "proto_pbtest" 4 | version = "0.0.1" 5 | edition = "2018" 6 | 7 | [dependencies] 8 | bytes = { version = "1.0" } 9 | compact_str = { features=["bytes"], version = "0.5" } 10 | lazy_static = { version = "1.4.0" } 11 | pb-jelly = { version = "0.0.17" } 12 | proto_google = { path = "../proto_google" } 13 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_pbtest/src/lib.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | 3 | #![allow(irrefutable_let_patterns)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(non_snake_case)] 6 | #![allow(non_upper_case_globals)] 7 | #![allow(unused_imports)] 8 | #![allow(unused_variables)] 9 | #![allow(irrefutable_let_patterns)] 10 | #![allow(rustdoc::broken_intra_doc_links)] 11 | 12 | // Modules are generated based on the naming conventions of protobuf, which might cause "module inception" 13 | #![allow(clippy::module_inception)] 14 | // This is all generated code, so "manually" implementing derivable impls is okay 15 | #![allow(clippy::derivable_impls)] 16 | // For enums with many variants, the matches!(...) macro isn't obviously better 17 | #![allow(clippy::match_like_matches_macro)] 18 | // Used by extension codegen 19 | #![allow(clippy::redundant_closure)] 20 | // TODO: Ideally we don't allow this 21 | #![allow(clippy::option_as_ref_deref)] 22 | // TODO: Ideally we don't allow this 23 | #![allow(clippy::match_single_binding)] 24 | 25 | 26 | pub mod bench; 27 | pub mod extensions; 28 | pub mod r#mod; 29 | pub mod pbtest2; 30 | pub mod pbtest3; 31 | pub mod servicepb; 32 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_pbtest/src/mod/mod.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | 3 | pub mod r#struct; 4 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_pbtest/src/mod/struct.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 3 | pub struct Message { 4 | } 5 | impl ::std::default::Default for Message { 6 | fn default() -> Self { 7 | Message { 8 | } 9 | } 10 | } 11 | ::lazy_static::lazy_static! { 12 | pub static ref Message_default: Message = Message::default(); 13 | } 14 | impl ::pb_jelly::Message for Message { 15 | fn descriptor(&self) -> ::std::option::Option<::pb_jelly::MessageDescriptor> { 16 | Some(::pb_jelly::MessageDescriptor { 17 | name: "Message", 18 | full_name: "pbtest.mod.Message", 19 | fields: &[ 20 | ], 21 | oneofs: &[ 22 | ], 23 | }) 24 | } 25 | fn compute_size(&self) -> usize { 26 | 0usize 27 | } 28 | fn serialize(&self, w: &mut W) -> ::std::io::Result<()> { 29 | Ok(()) 30 | } 31 | fn deserialize(&mut self, mut buf: &mut B) -> ::std::io::Result<()> { 32 | while let Some((field_number, typ)) = ::pb_jelly::wire_format::read(&mut buf)? { 33 | match field_number { 34 | _ => { 35 | ::pb_jelly::skip(typ, &mut buf)?; 36 | } 37 | } 38 | } 39 | Ok(()) 40 | } 41 | } 42 | impl ::pb_jelly::Reflection for Message { 43 | fn which_one_of(&self, oneof_name: &str) -> ::std::option::Option<&'static str> { 44 | match oneof_name { 45 | _ => { 46 | panic!("unknown oneof name given"); 47 | } 48 | } 49 | } 50 | fn get_field_mut(&mut self, field_name: &str) -> ::pb_jelly::reflection::FieldMut<'_> { 51 | match field_name { 52 | _ => { 53 | panic!("unknown field name given") 54 | } 55 | } 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /pb-test/gen/pb-jelly/proto_pbtest/src/servicepb.rs.expected: -------------------------------------------------------------------------------- 1 | // @generated, do not edit 2 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 3 | pub struct InpMessage { 4 | pub i: i32, 5 | } 6 | impl ::std::default::Default for InpMessage { 7 | fn default() -> Self { 8 | InpMessage { 9 | i: ::std::default::Default::default(), 10 | } 11 | } 12 | } 13 | ::lazy_static::lazy_static! { 14 | pub static ref InpMessage_default: InpMessage = InpMessage::default(); 15 | } 16 | impl ::pb_jelly::Message for InpMessage { 17 | fn descriptor(&self) -> ::std::option::Option<::pb_jelly::MessageDescriptor> { 18 | Some(::pb_jelly::MessageDescriptor { 19 | name: "InpMessage", 20 | full_name: "pbtest.InpMessage", 21 | fields: &[ 22 | ::pb_jelly::FieldDescriptor { 23 | name: "i", 24 | full_name: "pbtest.InpMessage.i", 25 | index: 0, 26 | number: 1, 27 | typ: ::pb_jelly::wire_format::Type::Varint, 28 | label: ::pb_jelly::Label::Optional, 29 | oneof_index: None, 30 | }, 31 | ], 32 | oneofs: &[ 33 | ], 34 | }) 35 | } 36 | fn compute_size(&self) -> usize { 37 | let mut size = 0usize; 38 | size += ::pb_jelly::helpers::compute_size_scalar::(&self.i, 1, ::pb_jelly::wire_format::Type::Varint); 39 | size 40 | } 41 | fn serialize(&self, w: &mut W) -> ::std::io::Result<()> { 42 | ::pb_jelly::helpers::serialize_scalar::(w, &self.i, 1, ::pb_jelly::wire_format::Type::Varint)?; 43 | Ok(()) 44 | } 45 | fn deserialize(&mut self, mut buf: &mut B) -> ::std::io::Result<()> { 46 | while let Some((field_number, typ)) = ::pb_jelly::wire_format::read(&mut buf)? { 47 | match field_number { 48 | 1 => { 49 | let val = ::pb_jelly::helpers::deserialize_known_length::(buf, typ, ::pb_jelly::wire_format::Type::Varint, "InpMessage", 1)?; 50 | self.i = val; 51 | } 52 | _ => { 53 | ::pb_jelly::skip(typ, &mut buf)?; 54 | } 55 | } 56 | } 57 | Ok(()) 58 | } 59 | } 60 | impl ::pb_jelly::Reflection for InpMessage { 61 | fn which_one_of(&self, oneof_name: &str) -> ::std::option::Option<&'static str> { 62 | match oneof_name { 63 | _ => { 64 | panic!("unknown oneof name given"); 65 | } 66 | } 67 | } 68 | fn get_field_mut(&mut self, field_name: &str) -> ::pb_jelly::reflection::FieldMut<'_> { 69 | match field_name { 70 | "i" => { 71 | ::pb_jelly::reflection::FieldMut::Value(&mut self.i) 72 | } 73 | _ => { 74 | panic!("unknown field name given") 75 | } 76 | } 77 | } 78 | } 79 | 80 | #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] 81 | pub struct OutMessage { 82 | pub o: i32, 83 | } 84 | impl ::std::default::Default for OutMessage { 85 | fn default() -> Self { 86 | OutMessage { 87 | o: ::std::default::Default::default(), 88 | } 89 | } 90 | } 91 | ::lazy_static::lazy_static! { 92 | pub static ref OutMessage_default: OutMessage = OutMessage::default(); 93 | } 94 | impl ::pb_jelly::Message for OutMessage { 95 | fn descriptor(&self) -> ::std::option::Option<::pb_jelly::MessageDescriptor> { 96 | Some(::pb_jelly::MessageDescriptor { 97 | name: "OutMessage", 98 | full_name: "pbtest.OutMessage", 99 | fields: &[ 100 | ::pb_jelly::FieldDescriptor { 101 | name: "o", 102 | full_name: "pbtest.OutMessage.o", 103 | index: 0, 104 | number: 1, 105 | typ: ::pb_jelly::wire_format::Type::Varint, 106 | label: ::pb_jelly::Label::Optional, 107 | oneof_index: None, 108 | }, 109 | ], 110 | oneofs: &[ 111 | ], 112 | }) 113 | } 114 | fn compute_size(&self) -> usize { 115 | let mut size = 0usize; 116 | size += ::pb_jelly::helpers::compute_size_scalar::(&self.o, 1, ::pb_jelly::wire_format::Type::Varint); 117 | size 118 | } 119 | fn serialize(&self, w: &mut W) -> ::std::io::Result<()> { 120 | ::pb_jelly::helpers::serialize_scalar::(w, &self.o, 1, ::pb_jelly::wire_format::Type::Varint)?; 121 | Ok(()) 122 | } 123 | fn deserialize(&mut self, mut buf: &mut B) -> ::std::io::Result<()> { 124 | while let Some((field_number, typ)) = ::pb_jelly::wire_format::read(&mut buf)? { 125 | match field_number { 126 | 1 => { 127 | let val = ::pb_jelly::helpers::deserialize_known_length::(buf, typ, ::pb_jelly::wire_format::Type::Varint, "OutMessage", 1)?; 128 | self.o = val; 129 | } 130 | _ => { 131 | ::pb_jelly::skip(typ, &mut buf)?; 132 | } 133 | } 134 | } 135 | Ok(()) 136 | } 137 | } 138 | impl ::pb_jelly::Reflection for OutMessage { 139 | fn which_one_of(&self, oneof_name: &str) -> ::std::option::Option<&'static str> { 140 | match oneof_name { 141 | _ => { 142 | panic!("unknown oneof name given"); 143 | } 144 | } 145 | } 146 | fn get_field_mut(&mut self, field_name: &str) -> ::pb_jelly::reflection::FieldMut<'_> { 147 | match field_name { 148 | "o" => { 149 | ::pb_jelly::reflection::FieldMut::Value(&mut self.o) 150 | } 151 | _ => { 152 | panic!("unknown field name given") 153 | } 154 | } 155 | } 156 | } 157 | 158 | -------------------------------------------------------------------------------- /pb-test/pb_test_gen/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "examples_gen" 3 | version = "0.1.0" 4 | authors = ["Parker Timmerman "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [dependencies] 9 | #pb-jelly-gen = "0.0.17" # If copying this example - use this 10 | pb-jelly-gen = { path = "../../pb-jelly-gen" } 11 | 12 | # only used when benchmarking PROST! 13 | prost-build = { version = "0.6", optional = true } 14 | # only used when benchmarking rust-protobuf 15 | protobuf-codegen = { version = "3.3", optional = true } 16 | 17 | [features] 18 | bench_prost = ["prost-build"] 19 | bench_rust_protobuf = ["protobuf-codegen"] 20 | 21 | # Override pb-jelly dependency for generated crates as well 22 | [patch.crates-io] 23 | pb-jelly = { path = "../../pb-jelly" } 24 | -------------------------------------------------------------------------------- /pb-test/pb_test_gen/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | env, 3 | fs, 4 | }; 5 | 6 | use pb_jelly_gen::GenProtos; 7 | #[cfg(feature = "bench_prost")] 8 | use prost_build; 9 | #[cfg(feature = "bench_rust_protobuf")] 10 | use protobuf_codegen::Customize; 11 | 12 | fn main() -> std::io::Result<()> { 13 | // Tell Cargo only re-run our build script if something in protos changes 14 | // println!("cargo:rerun-if-changed=protos"); 15 | 16 | // Generate protobuf-rust bindings 17 | GenProtos::builder() 18 | .out_path("../gen/pb-jelly") 19 | .src_path("../proto/packages") 20 | .gen_protos() 21 | .expect("Failed to generate protos"); 22 | 23 | // compile the protos we use for bench marking, if we want to benchmark against PROST! 24 | if cfg!(feature = "bench_prost") { 25 | let mut crate_path = env!("CARGO_MANIFEST_DIR").to_owned(); 26 | crate_path.push_str("/../gen/prost"); 27 | fs::create_dir_all(&crate_path)?; 28 | env::set_var("OUT_DIR", crate_path); 29 | 30 | #[cfg(feature = "bench_prost")] 31 | prost_build::compile_protos( 32 | &["../proto/packages/pbtest/bench.proto"], 33 | &["../proto/packages", "../../pb-jelly-gen/proto"], 34 | ) 35 | .unwrap(); 36 | } 37 | 38 | // compile the protos we use for bench marking, if we want to benchmark against rust_protobuf 39 | if cfg!(feature = "bench_rust_protobuf") { 40 | let mut crate_path = env!("CARGO_MANIFEST_DIR").to_owned(); 41 | crate_path.push_str("/../src/gen/rust_protobuf"); 42 | fs::create_dir_all(&crate_path)?; 43 | 44 | #[cfg(feature = "bench_rust_protobuf")] 45 | protobuf_codegen::Codegen::new() 46 | .out_dir("../src/gen/rust_protobuf") 47 | .inputs(&[ 48 | "../proto/packages/pbtest/bench.proto", 49 | "../../pb-jelly-gen/proto/rust/extensions.proto", 50 | ]) 51 | .include("../../pb-jelly-gen/proto") 52 | .include("../proto/packages") 53 | .customize(Customize::default().tokio_bytes(true)) 54 | .run() 55 | .expect("failed to generate rust_protobuf protos!"); 56 | } 57 | 58 | Ok(()) 59 | } 60 | -------------------------------------------------------------------------------- /pb-test/proto/packages/google/protobuf/empty.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option go_package = "github.com/golang/protobuf/ptypes/empty"; 37 | option java_package = "com.google.protobuf"; 38 | option java_outer_classname = "EmptyProto"; 39 | option java_multiple_files = true; 40 | option objc_class_prefix = "GPB"; 41 | option cc_enable_arenas = true; 42 | 43 | // A generic empty message that you can re-use to avoid defining duplicated 44 | // empty messages in your APIs. A typical example is to use it as the request 45 | // or the response type of an API method. For instance: 46 | // 47 | // service Foo { 48 | // rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); 49 | // } 50 | // 51 | // The JSON representation for `Empty` is empty JSON object `{}`. 52 | message Empty {} 53 | -------------------------------------------------------------------------------- /pb-test/proto/packages/nopackage.proto: -------------------------------------------------------------------------------- 1 | syntax="proto3"; 2 | 3 | message NoPackage { 4 | string field = 1; 5 | } 6 | -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/bench.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pbtest; 3 | 4 | // rust/extensions.proto is included by default 5 | import "rust/extensions.proto"; 6 | 7 | message BytesData { 8 | // Using the `zero_copy` option, the `bytes` type maps to `pb::Lazy` 9 | optional bytes data = 1 [(rust.zero_copy)=true]; 10 | } 11 | 12 | message VecData { 13 | // By default, the `bytes` type maps to Vec 14 | optional bytes data = 1; 15 | } 16 | 17 | message StringMessage { 18 | optional string data = 1; 19 | } 20 | 21 | message StringMessageSSO { 22 | optional string data = 1 [(rust.sso)=true]; 23 | } 24 | 25 | message Cities { 26 | repeated City cities = 1; 27 | } 28 | 29 | message City { 30 | required string city = 1; 31 | required string growth_from_2000_to_2013 = 2; 32 | required double latitude = 3; 33 | required double longitude = 4; 34 | required string population = 5; 35 | required string rank = 6; 36 | required string state = 7; 37 | } 38 | 39 | message CitiesSSO { 40 | repeated CitySSO cities = 1; 41 | } 42 | 43 | message CitySSO { 44 | required string city = 1 [(rust.sso)=true]; 45 | required string growth_from_2000_to_2013 = 2 [(rust.sso)=true]; 46 | required double latitude = 3; 47 | required double longitude = 4; 48 | required string population = 5 [(rust.sso)=true]; 49 | required string rank = 6 [(rust.sso)=true]; 50 | required string state = 7 [(rust.sso)=true]; 51 | } 52 | -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/extensions.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pbtest; 3 | 4 | import "pbtest/pbtest3.proto"; 5 | 6 | message Msg { 7 | optional int32 base_field = 250; 8 | extensions 100 to 200; 9 | extensions 300 to max; 10 | } 11 | 12 | extend Msg { 13 | optional int32 singular_primitive = 101; 14 | optional int32 singular_primitive_with_default = 102 [default=123]; 15 | optional ForeignMessage3 singular_message = 301; 16 | repeated int32 repeated_primitive = 300; 17 | repeated ForeignMessage3 repeated_message = 200; 18 | } 19 | 20 | message FakeMsg { 21 | optional int32 base_field = 250; 22 | 23 | optional int32 singular_primitive = 101; 24 | optional int32 singular_primitive_with_default = 102; 25 | optional ForeignMessage3 singular_message = 301; 26 | repeated int32 repeated_primitive = 300; 27 | repeated ForeignMessage3 repeated_message = 200; 28 | } 29 | -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/mod/struct.proto: -------------------------------------------------------------------------------- 1 | syntax="proto3"; 2 | 3 | package pbtest.mod; 4 | 5 | message Message {} -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/pbtest2.proto: -------------------------------------------------------------------------------- 1 | syntax="proto2"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | import "rust/extensions.proto"; 5 | 6 | package pbtest; 7 | 8 | // Test message names which collide with rust primitive names 9 | message Option {} 10 | message Vec {} 11 | message Default {} 12 | message String {} 13 | 14 | message Version0OneOfNoneNullable { 15 | oneof test_oneof { 16 | option (rust.nullable) = false; 17 | string string_one_of = 1; 18 | } 19 | } 20 | 21 | message Version1OneOfNoneNullable { 22 | oneof test_oneof { 23 | option (rust.nullable) = false; 24 | string string_one_of = 1; 25 | string string_two_of = 2; 26 | } 27 | } 28 | 29 | message Version2OneOfNoneNullable { 30 | oneof test_oneof { 31 | option (rust.nullable) = false; 32 | string string_one_of = 1; 33 | string string_two_of = 2; 34 | int32 int_one_Of = 3; 35 | } 36 | } 37 | 38 | message Version1Enum { 39 | enum TestEnum { 40 | ENUM0 = 0; 41 | } 42 | optional TestEnum enum_field = 1; 43 | } 44 | 45 | message Version2Enum { 46 | enum TestEnum { 47 | ENUM0 = 0; 48 | ENUM1 = 1; 49 | } 50 | optional TestEnum enum_field = 1; 51 | } 52 | 53 | message Version1OneOf { 54 | oneof test_oneof { 55 | string string_one_of = 1; 56 | } 57 | } 58 | 59 | message Version2OneOf { 60 | oneof test_oneof { 61 | string string_one_of = 1; 62 | int32 int_one_of = 2; 63 | } 64 | } 65 | 66 | message Version1 { 67 | required string required_string = 1; 68 | } 69 | 70 | message Version2 { 71 | required string required_string = 1; 72 | optional int32 optional_int32 = 2; 73 | optional int64 optional_int64 = 3; 74 | optional uint32 optional_uint32 = 4; 75 | optional uint64 optional_uint64 = 5; 76 | optional fixed64 optional_fixed64 = 6; 77 | optional fixed32 optional_fixed32 = 7; 78 | optional sfixed64 optional_sfixed64 = 8; 79 | optional sfixed32 optional_sfixed32 = 9; 80 | optional double optional_double = 10; 81 | optional bool optional_bool = 11; 82 | optional string optional_string = 12; 83 | optional bytes optional_bytes = 13; 84 | optional float optional_float = 14; 85 | } 86 | 87 | message ForeignMessage { 88 | optional int32 c = 1; 89 | } 90 | 91 | enum ForeignEnum { 92 | FOREIGN_FOO = 0; 93 | FOREIGN_BAR = 1; 94 | FOREIGN_BAZ = 2; 95 | } 96 | 97 | message TestMessage { 98 | enum NestedEnum { 99 | FOO = 0; 100 | BAR = 1; 101 | BAZ = 2; 102 | NEG = -1; // Intentionally negative. 103 | } 104 | 105 | optional int32 optional_int32 = 1; 106 | optional int64 optional_int64 = 2; 107 | optional uint32 optional_uint32 = 3; 108 | optional uint64 optional_uint64 = 4; 109 | optional fixed64 optional_fixed64 = 8; 110 | optional fixed32 optional_fixed32 = 9; 111 | optional sfixed64 optional_sfixed64 = 10; 112 | optional sfixed32 optional_sfixed32 = 11; 113 | optional double optional_double = 12; 114 | optional bool optional_bool = 13; 115 | optional string optional_string = 14; 116 | optional bytes optional_bytes = 15; 117 | optional float optional_float = 16; 118 | 119 | optional ForeignMessage optional_foreign_message = 19; 120 | 121 | optional NestedEnum optional_nested_enum = 21; 122 | optional ForeignEnum optional_foreign_enum = 22; 123 | 124 | repeated int32 repeated_int32 = 31; 125 | repeated int64 repeated_int64 = 32; 126 | repeated uint32 repeated_uint32 = 33; 127 | repeated uint64 repeated_uint64 = 34; 128 | repeated fixed64 repeated_fixed64 = 38; 129 | repeated fixed32 repeated_fixed32 = 39; 130 | repeated sfixed64 repeated_sfixed64 = 40; 131 | repeated sfixed32 repeated_sfixed32 = 41; 132 | repeated double repeated_double = 42; 133 | repeated bool repeated_bool = 43; 134 | repeated string repeated_string = 44; 135 | repeated bytes repeated_bytes = 45; 136 | repeated float repeated_float = 46; 137 | 138 | repeated ForeignMessage repeated_foreign_message = 49; 139 | 140 | repeated NestedEnum repeated_nested_enum = 51; 141 | repeated ForeignEnum repeated_foreign_enum = 52; 142 | 143 | optional ForeignMessage optional_foreign_message_boxed = 53 [(rust.box_it)=true]; 144 | optional ForeignMessage optional_foreign_message_nonnullable = 54 [(rust.nullable_field) = false]; 145 | 146 | oneof oneof_int { 147 | int32 int1 = 57; 148 | ForeignMessage foreign1 = 58; 149 | } 150 | oneof oneof_foreign { 151 | int32 int2 = 59; 152 | ForeignMessage foreign2 = 60; 153 | } 154 | oneof oneof_zero { 155 | int32 int3 = 61; 156 | ForeignMessage foreign3 = 62; 157 | } 158 | oneof oneof_null { 159 | int32 int4 = 63; 160 | ForeignMessage foreign4 = 64; 161 | } 162 | oneof oneof_unset { 163 | int32 int5 = 65; 164 | ForeignMessage foreign5 = 66; 165 | } 166 | oneof oneof_primitives { 167 | int32 int6 = 67; 168 | bool bool6 = 68; 169 | } 170 | // Generate a rust enum like this: 171 | // Enum { a, b, c(int32) } 172 | oneof oneof_empty_field { 173 | option (rust.nullable) = false; 174 | google.protobuf.Empty a = 70; 175 | google.protobuf.Empty b = 71; 176 | int32 c = 72; 177 | } 178 | 179 | // Use some rust reserved keywords 180 | optional bool type = 73; 181 | oneof mod { 182 | int32 loop = 74; 183 | int32 unsafe = 75; 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/pbtest3.proto: -------------------------------------------------------------------------------- 1 | syntax="proto3"; 2 | 3 | import "google/protobuf/empty.proto"; 4 | import "pbtest/pbtest2.proto"; 5 | import "pbtest/mod/struct.proto"; 6 | import "rust/extensions.proto"; 7 | 8 | package pbtest; 9 | 10 | message ForeignMessage3 { 11 | int32 c = 1; 12 | } 13 | 14 | enum ForeignEnum3 { 15 | FOREIGN3_FOO = 0; 16 | FOREIGN3_BAR = 1; 17 | FOREIGN3_BAZ = 2; 18 | } 19 | 20 | message Version31OneOfNoneNullable { 21 | oneof test_oneof { 22 | option (rust.nullable) = false; 23 | string string_one_of = 1; 24 | string string_two_of = 2; 25 | } 26 | } 27 | 28 | message Version32OneOfNoneNullable { 29 | oneof test_oneof { 30 | option (rust.nullable) = false; 31 | string string_one_of = 1; 32 | string string_two_of = 2; 33 | int32 int_one_Of = 3; 34 | } 35 | } 36 | 37 | message Version31Enum { 38 | enum TestEnum { 39 | ENUM0 = 0; 40 | } 41 | TestEnum enum_field = 1; 42 | } 43 | 44 | message Version32Enum { 45 | enum TestEnum { 46 | ENUM0 = 0; 47 | ENUM1 = 1; 48 | } 49 | TestEnum enum_field = 1; 50 | } 51 | 52 | message Version31OneOf { 53 | oneof test_oneof { 54 | string string_one_of = 1; 55 | } 56 | } 57 | 58 | message Version32OneOf { 59 | oneof test_oneof { 60 | string string_one_of = 1; 61 | int32 int_one_of = 2; 62 | } 63 | } 64 | 65 | message Version31 { 66 | string optional_string1 = 1; 67 | } 68 | 69 | message Version31SSO { 70 | string optional_string1 = 1 [(rust.sso)=true]; 71 | } 72 | 73 | message Version32 { 74 | string optional_string1 = 1; 75 | int32 optional_int32 = 2; 76 | int64 optional_int64 = 3; 77 | uint32 optional_uint32 = 4; 78 | uint64 optional_uint64 = 5; 79 | fixed64 optional_fixed64 = 6; 80 | fixed32 optional_fixed32 = 7; 81 | sfixed64 optional_sfixed64 = 8; 82 | sfixed32 optional_sfixed32 = 9; 83 | double optional_double = 10; 84 | bool optional_bool = 11; 85 | string optional_string = 12; 86 | bytes optional_bytes = 13; 87 | float optional_float = 14; 88 | } 89 | 90 | message TestMessage3 { 91 | enum NestedEnum3 { 92 | FOO = 0; 93 | BAR = 1; 94 | BAZ = 2; 95 | NEG = -1; // Intentionally negative. 96 | } 97 | 98 | int32 optional_int32 = 1; 99 | int64 optional_int64 = 2; 100 | uint32 optional_uint32 = 3; 101 | uint64 optional_uint64 = 4; 102 | fixed64 optional_fixed64 = 8; 103 | fixed32 optional_fixed32 = 9; 104 | sfixed64 optional_sfixed64 = 10; 105 | sfixed32 optional_sfixed32 = 11; 106 | double optional_double = 12; 107 | bool optional_bool = 13; 108 | string optional_string = 14; 109 | bytes optional_bytes = 15; 110 | float optional_float = 16; 111 | 112 | ForeignMessage3 optional_foreign_message = 19; 113 | 114 | NestedEnum3 optional_nested_enum = 21; 115 | ForeignEnum3 optional_foreign_enum = 22; 116 | 117 | repeated int32 repeated_int32 = 31; 118 | repeated int64 repeated_int64 = 32; 119 | repeated uint32 repeated_uint32 = 33; 120 | repeated uint64 repeated_uint64 = 34; 121 | repeated fixed64 repeated_fixed64 = 38; 122 | repeated fixed32 repeated_fixed32 = 39; 123 | repeated sfixed64 repeated_sfixed64 = 40; 124 | repeated sfixed32 repeated_sfixed32 = 41; 125 | repeated double repeated_double = 42; 126 | repeated bool repeated_bool = 43; 127 | repeated string repeated_string = 44; 128 | repeated bytes repeated_bytes = 45; 129 | repeated float repeated_float = 46; 130 | 131 | repeated ForeignMessage3 repeated_foreign_message = 49; 132 | 133 | repeated NestedEnum3 repeated_nested_enum = 51; 134 | repeated ForeignEnum3 repeated_foreign_enum = 52; 135 | 136 | // Message compatibility with proto2. 137 | // proto2 enum are not compatible, but messages are. 138 | ForeignMessage proto2_msg = 53; 139 | ForeignMessage proto2_msg_empty = 54; 140 | ForeignMessage proto2_msg_missing = 55; 141 | ForeignMessage3 optional_foreign_message_boxed = 56 [(rust.box_it)=true]; 142 | ForeignMessage3 optional_foreign_message_nonnullable = 69 [(rust.nullable_field)=false]; 143 | 144 | oneof oneof_int { 145 | int32 int1 = 57; 146 | ForeignMessage3 foreign1 = 58; 147 | } 148 | oneof oneof_foreign { 149 | int32 int2 = 59; 150 | ForeignMessage3 foreign2 = 60; 151 | } 152 | oneof oneof_zero { 153 | int32 int3 = 61; 154 | ForeignMessage3 foreign3 = 62; 155 | } 156 | oneof oneof_null { 157 | int32 int4 = 63; 158 | ForeignMessage3 foreign4 = 64; 159 | } 160 | oneof oneof_unset { 161 | int32 int5 = 65; 162 | ForeignMessage3 foreign5 = 66; 163 | } 164 | oneof oneof_primitives { 165 | int32 int6 = 67; 166 | bool bool6 = 68; 167 | } 168 | 169 | // Generate a rust enum like this: 170 | // Enum { a, b, c(int32) } 171 | oneof oneof_empty_field { 172 | option (rust.nullable) = false; 173 | google.protobuf.Empty a = 70; 174 | google.protobuf.Empty b = 71; 175 | int32 c = 72; 176 | } 177 | 178 | message NestedMessage { 179 | message File { 180 | repeated bytes blocklist = 1; 181 | uint32 size = 2; 182 | } 183 | message Dir {} 184 | enum Enum { 185 | ENUM_VARIANT_ONE = 0; 186 | } 187 | enum NonNullableEnum { 188 | option (rust.err_if_default_or_unknown) = true; 189 | NON_NULLABLE_VARIANT_UNKNOWN = 0; 190 | NON_NULLABLE_VARIANT_ONE = 1; 191 | } 192 | 193 | oneof nested_oneof { 194 | option (rust.nullable) = false; 195 | File f = 3; 196 | Dir d = 4; 197 | Enum e = 5; 198 | NonNullableEnum n = 6; 199 | } 200 | } 201 | 202 | NestedMessage nested = 73 [(rust.nullable_field)=false]; 203 | NestedMessage nested_nullable = 74; 204 | repeated NestedMessage nested_repeated = 75 [(rust.nullable_field)=false]; 205 | 206 | bytes fixed_length = 76 [(rust.type)="[u8; 4]"]; 207 | repeated bytes fixed_length_repeated = 77 [(rust.type)="[u8; 4]"]; 208 | bytes zero_or_fixed_length = 78 [(rust.type)="Option<[u8; 4]>"]; 209 | repeated bytes zero_or_fixed_length_repeated = 79 [(rust.type)="Option<[u8; 4]>"]; 210 | } 211 | 212 | message TestBoxedNonnullable { 213 | ForeignMessage3 field = 1 [(rust.box_it)=true, (rust.nullable_field)=false]; 214 | } 215 | 216 | message TestMessage3NonNullableOneof { 217 | oneof non_nullable_oneof { 218 | option (rust.nullable) = false; 219 | 220 | int32 a = 1; 221 | int32 b = 2; 222 | } 223 | 224 | uint64 other_field = 3; 225 | } 226 | 227 | message TestMessage3ErrIfDefaultEnum { 228 | enum ErrIfDefaultEnum { 229 | option (rust.err_if_default_or_unknown) = true; 230 | 231 | UNKNOWN_INVALID_VALUE = 0; 232 | THE_OTHER_ONE = 1; 233 | } 234 | ErrIfDefaultEnum field = 1; 235 | } 236 | 237 | message TestMessage3ErrIfDefaultEnumOneof { 238 | enum ErrIfDefaultEnum { 239 | option (rust.err_if_default_or_unknown) = true; 240 | 241 | UNKNOWN_INVALID_VALUE = 0; 242 | THE_OTHER_ONE = 1; 243 | } 244 | oneof maybe { 245 | ErrIfDefaultEnum something = 1; 246 | } 247 | google.protobuf.Empty nothing = 2; 248 | } 249 | 250 | message TestMessage3RepeatedErrIfDefaultEnum { 251 | repeated TestMessage3ErrIfDefaultEnum.ErrIfDefaultEnum field = 1; 252 | } 253 | 254 | message TestMessage3ClosedEnum { 255 | enum ClosedEnum { 256 | option (rust.closed_enum) = true; 257 | 258 | DEFAULT = 0; 259 | ONE = 1; 260 | } 261 | ClosedEnum value = 1; 262 | } 263 | 264 | message TestMessage3ClosedEnum2 { 265 | enum ClosedEnum { 266 | option (rust.closed_enum) = true; 267 | 268 | DEFAULT = 0; 269 | ONE = 1; 270 | TWO = 2; 271 | } 272 | ClosedEnum value = 1; 273 | } 274 | 275 | message TestMessage3NonOptionalBoxedMessage { 276 | message InnerMessage { 277 | string name = 1; 278 | } 279 | 280 | InnerMessage msg = 1 [(rust.box_it)=true, (rust.nullable_field)=false]; 281 | } 282 | 283 | message TestPreserveUnrecognized1 { 284 | option (rust.preserve_unrecognized) = true; 285 | string string1 = 1; 286 | } 287 | 288 | message TestPreserveUnrecognized2 { 289 | option (rust.preserve_unrecognized) = true; 290 | string a_string1 = 1; 291 | int32 a_int32 = 2; 292 | int64 a_int64 = 3; 293 | uint32 a_uint32 = 4; 294 | uint64 a_uint64 = 5; 295 | fixed64 a_fixed64 = 6; 296 | fixed32 a_fixed32 = 7; 297 | sfixed64 a_sfixed64 = 8; 298 | sfixed32 a_sfixed32 = 9; 299 | double a_double = 10; 300 | bool a_bool = 11; 301 | string a_string = 12; 302 | bytes a_bytes = 13; 303 | float a_float = 14; 304 | } 305 | 306 | message TestPreserveUnrecognizedEmpty { 307 | option (rust.preserve_unrecognized) = true; 308 | } 309 | 310 | message TestSmallString { 311 | string compact = 1 [(rust.sso)=true]; 312 | } 313 | 314 | message TestBoxedSmallString { 315 | string compact = 1 [(rust.sso)=true, (rust.box_it)=true]; 316 | } 317 | 318 | message TestNonOptionalSmallString { 319 | string compact = 1 [(rust.sso)=true, (rust.box_it)=true, (rust.nullable_field)=false]; 320 | } 321 | 322 | message TestSmallStringPreserveUnrecognized { 323 | option (rust.preserve_unrecognized) = true; 324 | 325 | string compact = 1; 326 | } 327 | 328 | message TestProto3Optional { 329 | // optional doesn't mean anything here, messages are already optional 330 | optional ForeignMessage3 a_message = 1; 331 | optional int32 a_int32 = 2; 332 | optional int64 a_int64 = 3; 333 | optional uint32 a_uint32 = 4; 334 | optional uint64 a_uint64 = 5; 335 | optional fixed64 a_fixed64 = 6; 336 | optional fixed32 a_fixed32 = 7; 337 | optional sfixed64 a_sfixed64 = 8; 338 | optional sfixed32 a_sfixed32 = 9; 339 | optional double a_double = 10; 340 | optional bool a_bool = 11; 341 | optional string a_string = 12; 342 | optional bytes a_bytes = 13; 343 | optional float a_float = 14; 344 | // make sure the synthetic oneofs don't interfere with real oneofs 345 | oneof real_oneof_1 { 346 | string real_oneof_1_1 = 15; 347 | string real_oneof_1_2 = 16; 348 | } 349 | oneof real_oneof_2 { 350 | option (rust.nullable) = false; 351 | string real_oneof_2_1 = 17; 352 | string real_oneof_2_2 = 18; 353 | } 354 | } 355 | 356 | message TestProto3Zerocopy { 357 | bytes data1 = 1 [(rust.zero_copy) = true]; 358 | bytes data2 = 2 [(rust.zero_copy) = true]; 359 | } 360 | 361 | message TestProto3ContainsZerocopy { 362 | TestProto3Zerocopy inner = 1; 363 | } 364 | 365 | message RecursiveOneof { 366 | oneof oneof_field { 367 | // This field should be boxed automatically. 368 | RecursiveOneof field = 1; 369 | google.protobuf.Empty empty = 2; 370 | // Boxing should override the empty-oneof-field special case. 371 | google.protobuf.Empty boxed_empty = 3 [(rust.box_it) = true]; 372 | ForeignMessage3 not_boxed = 4; 373 | ForeignMessage3 boxed = 5 [(rust.box_it) = true]; 374 | } 375 | } 376 | 377 | message MentionsKeywordPath { 378 | pbtest.mod.Message message = 1; 379 | } 380 | 381 | message NonNullableOneofKeyword { 382 | oneof async { 383 | option (rust.nullable) = false; 384 | int64 a = 1; 385 | } 386 | } 387 | 388 | message NonNullableEnumKeyword { 389 | TestMessage3.NestedMessage.NonNullableEnum enum = 1; 390 | } 391 | 392 | message MutuallyRecursiveA { 393 | MutuallyRecursiveB inner = 1; 394 | } 395 | 396 | message MutuallyRecursiveB { 397 | MutuallyRecursiveA inner = 1; 398 | } 399 | 400 | message MutuallyRecursiveWithRepeatedA { 401 | repeated MutuallyRecursiveWithRepeatedB inner = 1; 402 | } 403 | 404 | message MutuallyRecursiveWithRepeatedB { 405 | // This field should *not* be boxed because `repeated` on the other message 406 | // should suffice to break the cycle. 407 | MutuallyRecursiveWithRepeatedA inner = 1; 408 | } 409 | 410 | message MutuallyRecursiveWithBoxedA { 411 | MutuallyRecursiveWithBoxedB inner = 1 [(rust.box_it) = true]; 412 | } 413 | 414 | message MutuallyRecursiveWithBoxedB { 415 | // This field should not be boxed. 416 | MutuallyRecursiveWithBoxedA inner = 1; 417 | } 418 | -------------------------------------------------------------------------------- /pb-test/proto/packages/pbtest/servicepb.proto: -------------------------------------------------------------------------------- 1 | syntax="proto3"; 2 | 3 | package pbtest; 4 | 5 | message InpMessage { 6 | int32 i = 1; 7 | } 8 | 9 | message OutMessage { 10 | int32 o = 1; 11 | } 12 | 13 | service TestService { 14 | rpc TestRPC(InpMessage) returns (OutMessage) {} 15 | } 16 | -------------------------------------------------------------------------------- /pb-test/src/bench.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod benches { 3 | use std::hint::black_box; 4 | use std::io::Cursor; 5 | 6 | use bytes::Bytes; 7 | use compact_str::CompactString; 8 | use pb_jelly::{ 9 | Lazy, 10 | Message, 11 | }; 12 | use proto_pbtest::bench::{ 13 | BytesData, 14 | Cities, 15 | CitiesSSO, 16 | City, 17 | CitySSO, 18 | StringMessage, 19 | StringMessageSSO, 20 | VecData, 21 | }; 22 | use serde::Deserialize; 23 | use test::Bencher; 24 | 25 | #[bench] 26 | fn bench_deserialize_zero_copy_bytes(b: &mut Bencher) { 27 | // Generate 4MB of data 28 | let data = Lazy::new(Bytes::from(vec![42 as u8; 4 * 1024 * 1024])); 29 | 30 | // Create our proto struct 31 | let mut proto = BytesData::default(); 32 | proto.set_data(data); 33 | 34 | // Serialize the proto 35 | let ser_bytes: Vec = proto.serialize_to_vec(); 36 | 37 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 38 | 39 | // Read our serialized bytes into a Bytes 40 | let bytes_buf = Bytes::from(ser_bytes); 41 | 42 | b.iter(|| { 43 | // Deserialize our proto 44 | let mut de_proto = BytesData::default(); 45 | de_proto.deserialize(&mut Cursor::new(bytes_buf.clone())).unwrap(); 46 | assert!(de_proto.has_data()); 47 | de_proto 48 | }); 49 | } 50 | 51 | #[bench] 52 | fn bench_deserialize_vec_bytes(b: &mut Bencher) { 53 | // Generate 4MB of data 54 | let data = vec![42 as u8; 4 * 1024 * 1024]; 55 | 56 | // Create our proto struct 57 | let mut proto = VecData::default(); 58 | proto.set_data(data); 59 | 60 | // Serialize the proto 61 | let ser_bytes: Vec = proto.serialize_to_vec(); 62 | 63 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 64 | 65 | // Read our serialized bytes into a Bytes 66 | let bytes_buf = Bytes::from(ser_bytes); 67 | 68 | b.iter(|| { 69 | // Deserialize our proto 70 | let mut de_proto = VecData::default(); 71 | de_proto.deserialize(&mut Cursor::new(bytes_buf.clone())).unwrap(); 72 | assert!(de_proto.has_data()); 73 | de_proto 74 | }); 75 | } 76 | 77 | #[bench] 78 | fn bench_deserialize_string(b: &mut Bencher) { 79 | let data = String::from(include_str!("../data/moby_dick.txt")); 80 | 81 | let mut proto = StringMessage::default(); 82 | proto.set_data(data); 83 | 84 | let bytes: Vec = proto.serialize_to_vec(); 85 | let bytes = Bytes::from(bytes); 86 | 87 | b.iter(|| { 88 | // Deserialize our proto 89 | let mut de_proto = StringMessage::default(); 90 | de_proto.deserialize(&mut Cursor::new(bytes.clone())).unwrap(); 91 | black_box(de_proto) 92 | }); 93 | } 94 | 95 | #[bench] 96 | fn bench_deserialize_string_sso(b: &mut Bencher) { 97 | let data = CompactString::from(include_str!("../data/moby_dick.txt")); 98 | 99 | let mut proto = StringMessageSSO::default(); 100 | proto.set_data(data); 101 | 102 | let bytes: Vec = proto.serialize_to_vec(); 103 | let bytes = Bytes::from(bytes); 104 | 105 | b.iter(|| { 106 | // Deserialize our proto 107 | let mut de_proto = StringMessageSSO::default(); 108 | de_proto.deserialize(&mut Cursor::new(bytes.clone())).unwrap(); 109 | black_box(de_proto) 110 | }); 111 | } 112 | 113 | #[bench] 114 | fn bench_deserialize_small_string(b: &mut Bencher) { 115 | let data = String::from("IMG_1234.png"); 116 | 117 | let mut proto = StringMessage::default(); 118 | proto.set_data(data); 119 | 120 | let bytes: Vec = proto.serialize_to_vec(); 121 | let bytes = Bytes::from(bytes); 122 | 123 | b.iter(|| { 124 | // Deserialize our proto 125 | let mut de_proto = StringMessage::default(); 126 | de_proto.deserialize(&mut Cursor::new(bytes.clone())).unwrap(); 127 | black_box(de_proto) 128 | }); 129 | } 130 | 131 | #[bench] 132 | fn bench_deserialize_small_string_sso(b: &mut Bencher) { 133 | let data = CompactString::from("IMG_1234.png"); 134 | 135 | let mut proto = StringMessageSSO::default(); 136 | proto.set_data(data); 137 | 138 | let bytes: Vec = proto.serialize_to_vec(); 139 | let bytes = Bytes::from(bytes); 140 | 141 | b.iter(|| { 142 | // Deserialize our proto 143 | let mut de_proto = StringMessageSSO::default(); 144 | de_proto.deserialize(&mut Cursor::new(bytes.clone())).unwrap(); 145 | black_box(de_proto) 146 | }); 147 | } 148 | 149 | #[derive(Debug, Deserialize)] 150 | struct JsonCity { 151 | city: CompactString, 152 | growth_from_2000_to_2013: CompactString, 153 | latitude: f64, 154 | longitude: f64, 155 | population: CompactString, 156 | rank: CompactString, 157 | state: CompactString, 158 | } 159 | 160 | #[bench] 161 | fn bench_deserialize_many_strings(b: &mut Bencher) { 162 | let json = String::from(include_str!("../data/cities.json")); 163 | let json_cities: Vec = serde_json::from_str(&json).expect("failed to parse cities.json"); 164 | 165 | let cities: Vec = json_cities 166 | .into_iter() 167 | .map(|city| City { 168 | city: Some(city.city.to_string()), 169 | growth_from_2000_to_2013: Some(city.growth_from_2000_to_2013.to_string()), 170 | latitude: Some(city.latitude), 171 | longitude: Some(city.longitude), 172 | population: Some(city.population.to_string()), 173 | rank: Some(city.rank.to_string()), 174 | state: Some(city.state.to_string()), 175 | }) 176 | .collect(); 177 | let proto = Cities { cities }; 178 | 179 | let bytes = proto.serialize_to_vec(); 180 | 181 | b.iter(|| { 182 | // Deserialize our proto 183 | let de_proto: Cities = Message::deserialize_from_slice(&bytes[..]).unwrap(); 184 | black_box(de_proto) 185 | }); 186 | } 187 | 188 | #[bench] 189 | fn bench_deserialize_many_strings_sso(b: &mut Bencher) { 190 | let json = String::from(include_str!("../data/cities.json")); 191 | let json_cities: Vec = serde_json::from_str(&json).expect("failed to parse cities.json"); 192 | 193 | let cities: Vec = json_cities 194 | .into_iter() 195 | .map(|city| CitySSO { 196 | city: Some(city.city), 197 | growth_from_2000_to_2013: Some(city.growth_from_2000_to_2013), 198 | latitude: Some(city.latitude), 199 | longitude: Some(city.longitude), 200 | population: Some(city.population), 201 | rank: Some(city.rank), 202 | state: Some(city.state), 203 | }) 204 | .collect(); 205 | let proto = CitiesSSO { cities }; 206 | 207 | let bytes = proto.serialize_to_vec(); 208 | 209 | b.iter(|| { 210 | // Deserialize our proto 211 | let de_proto: CitiesSSO = Message::deserialize_from_slice(&bytes[..]).unwrap(); 212 | black_box(de_proto) 213 | }); 214 | } 215 | } 216 | 217 | #[cfg(all(test, feature = "bench_prost"))] 218 | mod prost { 219 | use prost::bytes::Bytes; 220 | use prost::Message; 221 | use test::Bencher; 222 | 223 | mod gen { 224 | include!(concat!(env!("CARGO_MANIFEST_DIR"), "/gen/prost/pbtest.rs")); 225 | } 226 | 227 | #[bench] 228 | fn bench_deserialize_prost_bytes(b: &mut Bencher) { 229 | // Generate 4MB of data 230 | let data = vec![42 as u8; 4 * 1024 * 1024]; 231 | 232 | // Create our proto struct 233 | let mut proto = gen::BytesData::default(); 234 | proto.data = Some(data); 235 | 236 | // Serialize the proto 237 | let csz = proto.encoded_len(); 238 | let mut ser_bytes = Vec::with_capacity(csz); 239 | proto.encode(&mut ser_bytes).expect("failed to encode PROST proto!"); 240 | 241 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 242 | 243 | // Read our serialized bytes into a Bytes struct, this implements bytes::Buf 244 | let bytes_buf = Bytes::from(ser_bytes); 245 | 246 | b.iter(|| { 247 | // Deserialize our proto 248 | let de_proto = gen::BytesData::decode(bytes_buf.clone()).expect("failed to decode PROST proto!"); 249 | assert!(de_proto.data.is_some()); 250 | }); 251 | } 252 | 253 | #[bench] 254 | fn bench_deserialize_prost_string(b: &mut Bencher) { 255 | let data = String::from(include_str!("../data/moby_dick.txt")); 256 | 257 | // Create our proto struct 258 | let mut proto = gen::StringMessage::default(); 259 | proto.data = Some(data); 260 | 261 | // Serialize the proto 262 | let csz = proto.encoded_len(); 263 | let mut ser_bytes = Vec::with_capacity(csz); 264 | proto.encode(&mut ser_bytes).expect("failed to encode PROST proto!"); 265 | 266 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 267 | 268 | // Read our serialized bytes into a Bytes struct, this implements bytes::Buf 269 | let bytes_buf = Bytes::from(ser_bytes); 270 | 271 | b.iter(|| { 272 | // Deserialize our proto 273 | let de_proto = gen::StringMessage::decode(bytes_buf.clone()).expect("failed to decode PROST proto!"); 274 | assert!(de_proto.data.is_some()); 275 | }); 276 | } 277 | } 278 | 279 | #[cfg(all(test, feature = "bench_rust_protobuf"))] 280 | mod rust_protobuf { 281 | use bytes::Bytes; 282 | use protobuf::{ 283 | CodedInputStream, 284 | Message, 285 | }; 286 | use test::Bencher; 287 | 288 | use crate::gen::rust_protobuf::bench::{ 289 | BytesData, 290 | StringMessage, 291 | }; 292 | 293 | #[bench] 294 | fn bench_deserialize_rust_protobuf_bytes(b: &mut Bencher) { 295 | // Generate 4MB of data 296 | let data = Bytes::from(vec![42 as u8; 4 * 1024 * 1024]); 297 | 298 | // Create our proto struct 299 | let mut proto = BytesData::new(); 300 | proto.set_data(data); 301 | 302 | // Serialize the proto 303 | let csz = proto.compute_size(); 304 | let mut ser_bytes = Vec::with_capacity(csz as usize); 305 | proto 306 | .write_to_vec(&mut ser_bytes) 307 | .expect("failed to encode rust_protobuf proto!"); 308 | 309 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 310 | 311 | // Read our serialized bytes into a Bytes struct, this implements bytes::Buf 312 | let bytes_buf = Bytes::from(ser_bytes); 313 | 314 | b.iter(|| { 315 | // Deserialize our proto 316 | let mut input_stream = CodedInputStream::from_tokio_bytes(&bytes_buf); 317 | let mut de_proto = BytesData::default(); 318 | de_proto 319 | .merge_from(&mut input_stream) 320 | .expect("failed to decode rust_protobuf proto!"); 321 | assert!(de_proto.has_data()); 322 | }); 323 | } 324 | 325 | #[bench] 326 | fn bench_deserialize_rust_protobuf_string(b: &mut Bencher) { 327 | let data = String::from(include_str!("../data/moby_dick.txt")); 328 | 329 | // Create our proto struct 330 | let mut proto = StringMessage::new(); 331 | proto.set_data(data); 332 | 333 | // Serialize the proto 334 | let csz = proto.compute_size(); 335 | let mut ser_bytes = Vec::with_capacity(csz as usize); 336 | proto 337 | .write_to_vec(&mut ser_bytes) 338 | .expect("failed to encode rust_protobuf proto!"); 339 | 340 | // Serialized proto gets theoretically sent across ☁️ The Internet ☁️ 341 | 342 | // Read our serialized bytes into a Bytes struct, this implements bytes::Buf 343 | let bytes_buf = Bytes::from(ser_bytes); 344 | 345 | b.iter(|| { 346 | // Deserialize our proto 347 | let mut input_stream = CodedInputStream::from_tokio_bytes(&bytes_buf); 348 | let mut de_proto = StringMessage::default(); 349 | de_proto 350 | .merge_from(&mut input_stream) 351 | .expect("failed to decode rust_protobuf proto!"); 352 | assert!(de_proto.has_data()); 353 | }); 354 | } 355 | } 356 | -------------------------------------------------------------------------------- /pb-test/src/gen/README.md: -------------------------------------------------------------------------------- 1 | # Why does this module exist? 2 | 3 | When trying to include generated code from [`protobuf-codegen`](https://github.com/stepancheg/rust-protobuf/tree/master/protobuf-codegen) in the same fashion as `PROST!`, e.g. 4 | ``` 5 | mod gen { 6 | include!(concat!(env!("CARGO_MANIFEST_DIR"), "/gen/prost/pbtest.rs")); 7 | } 8 | ``` 9 | you get a compiler error: 10 | ``` 11 | error: an inner attribute is not permitted following an outer attribute 12 | ``` 13 | According to issue [`#117`](https://github.com/stepancheg/rust-protobuf/issues/117) on `rust-protobuf`, the work around for this include issue is to bundle your code into a module like so. You don't need to be as deeply nested as this example is, but we found this organization to make sense. 14 | -------------------------------------------------------------------------------- /pb-test/src/gen/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod rust_protobuf; 2 | -------------------------------------------------------------------------------- /pb-test/src/gen/rust_protobuf/extensions.rs: -------------------------------------------------------------------------------- 1 | // This file is generated by rust-protobuf 3.3.0. Do not edit 2 | // .proto file is parsed by protoc 25.2 3 | // @generated 4 | 5 | // https://github.com/rust-lang/rust-clippy/issues/702 6 | #![allow(unknown_lints)] 7 | #![allow(clippy::all)] 8 | 9 | #![allow(unused_attributes)] 10 | #![cfg_attr(rustfmt, rustfmt::skip)] 11 | 12 | #![allow(box_pointers)] 13 | #![allow(dead_code)] 14 | #![allow(missing_docs)] 15 | #![allow(non_camel_case_types)] 16 | #![allow(non_snake_case)] 17 | #![allow(non_upper_case_globals)] 18 | #![allow(trivial_casts)] 19 | #![allow(unused_results)] 20 | #![allow(unused_mut)] 21 | 22 | //! Generated file from `rust/extensions.proto` 23 | 24 | /// Generated files are compatible only with the same version 25 | /// of protobuf runtime. 26 | const _PROTOBUF_VERSION_CHECK: () = ::protobuf::VERSION_3_3_0; 27 | 28 | /// Extension fields 29 | pub mod exts { 30 | 31 | pub const box_it: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50000, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 32 | 33 | pub const grpc_slices: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50003, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 34 | 35 | pub const blob: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50010, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 36 | 37 | pub const type_: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, ::std::string::String> = ::protobuf::ext::ExtFieldOptional::new(50004, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_STRING); 38 | 39 | pub const zero_copy: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50007, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 40 | 41 | pub const sso: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50009, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 42 | 43 | pub const nullable_field: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FieldOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50008, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 44 | 45 | pub const err_if_default_or_unknown: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::EnumOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50002, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 46 | 47 | pub const closed_enum: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::EnumOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50008, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 48 | 49 | pub const preserve_unrecognized: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::MessageOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50006, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 50 | 51 | pub const nullable: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::OneofOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50001, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 52 | 53 | pub const serde_derive: ::protobuf::ext::ExtFieldOptional<::protobuf::descriptor::FileOptions, bool> = ::protobuf::ext::ExtFieldOptional::new(50005, ::protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL); 54 | } 55 | 56 | static file_descriptor_proto_data: &'static [u8] = b"\ 57 | \n\x15rust/extensions.proto\x12\x04rust\x1a\x20google/protobuf/descripto\ 58 | r.proto:6\n\x06box_it\x18\xd0\x86\x03\x20\x01(\x08\x12\x1d.google.protob\ 59 | uf.FieldOptionsR\x05boxIt:@\n\x0bgrpc_slices\x18\xd3\x86\x03\x20\x01(\ 60 | \x08\x12\x1d.google.protobuf.FieldOptionsR\ngrpcSlices:3\n\x04blob\x18\ 61 | \xda\x86\x03\x20\x01(\x08\x12\x1d.google.protobuf.FieldOptionsR\x04blob:\ 62 | 3\n\x04type\x18\xd4\x86\x03\x20\x01(\t\x12\x1d.google.protobuf.FieldOpti\ 63 | onsR\x04type:<\n\tzero_copy\x18\xd7\x86\x03\x20\x01(\x08\x12\x1d.google.\ 64 | protobuf.FieldOptionsR\x08zeroCopy:1\n\x03sso\x18\xd9\x86\x03\x20\x01(\ 65 | \x08\x12\x1d.google.protobuf.FieldOptionsR\x03sso:L\n\x0enullable_field\ 66 | \x18\xd8\x86\x03\x20\x01(\x08\x12\x1d.google.protobuf.FieldOptions:\x04t\ 67 | rueR\rnullableField:X\n\x19err_if_default_or_unknown\x18\xd2\x86\x03\x20\ 68 | \x01(\x08\x12\x1c.google.protobuf.EnumOptionsR\x15errIfDefaultOrUnknown:\ 69 | ?\n\x0bclosed_enum\x18\xd8\x86\x03\x20\x01(\x08\x12\x1c.google.protobuf.\ 70 | EnumOptionsR\nclosedEnum:V\n\x15preserve_unrecognized\x18\xd6\x86\x03\ 71 | \x20\x01(\x08\x12\x1f.google.protobuf.MessageOptionsR\x14preserveUnrecog\ 72 | nized:A\n\x08nullable\x18\xd1\x86\x03\x20\x01(\x08\x12\x1d.google.protob\ 73 | uf.OneofOptions:\x04trueR\x08nullable:H\n\x0cserde_derive\x18\xd5\x86\ 74 | \x03\x20\x01(\x08\x12\x1c.google.protobuf.FileOptions:\x05falseR\x0bserd\ 75 | eDerive\ 76 | "; 77 | 78 | /// `FileDescriptorProto` object which was a source for this generated file 79 | fn file_descriptor_proto() -> &'static ::protobuf::descriptor::FileDescriptorProto { 80 | static file_descriptor_proto_lazy: ::protobuf::rt::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::rt::Lazy::new(); 81 | file_descriptor_proto_lazy.get(|| { 82 | ::protobuf::Message::parse_from_bytes(file_descriptor_proto_data).unwrap() 83 | }) 84 | } 85 | 86 | /// `FileDescriptor` object which allows dynamic access to files 87 | pub fn file_descriptor() -> &'static ::protobuf::reflect::FileDescriptor { 88 | static generated_file_descriptor_lazy: ::protobuf::rt::Lazy<::protobuf::reflect::GeneratedFileDescriptor> = ::protobuf::rt::Lazy::new(); 89 | static file_descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::FileDescriptor> = ::protobuf::rt::Lazy::new(); 90 | file_descriptor.get(|| { 91 | let generated_file_descriptor = generated_file_descriptor_lazy.get(|| { 92 | let mut deps = ::std::vec::Vec::with_capacity(1); 93 | deps.push(::protobuf::descriptor::file_descriptor().clone()); 94 | let mut messages = ::std::vec::Vec::with_capacity(0); 95 | let mut enums = ::std::vec::Vec::with_capacity(0); 96 | ::protobuf::reflect::GeneratedFileDescriptor::new_generated( 97 | file_descriptor_proto(), 98 | deps, 99 | messages, 100 | enums, 101 | ) 102 | }); 103 | ::protobuf::reflect::FileDescriptor::new_generated_2(generated_file_descriptor) 104 | }) 105 | } 106 | -------------------------------------------------------------------------------- /pb-test/src/gen/rust_protobuf/mod.rs: -------------------------------------------------------------------------------- 1 | // @generated 2 | 3 | pub mod bench; 4 | pub mod extensions; 5 | -------------------------------------------------------------------------------- /pb-test/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![warn(rust_2018_idioms)] 2 | #![feature(test)] 3 | 4 | #[allow(unused_extern_crates)] 5 | extern crate test; 6 | 7 | #[cfg(test)] 8 | mod bench; 9 | #[cfg(feature = "bench_rust_protobuf")] 10 | mod gen; 11 | #[cfg(test)] 12 | mod pbtest; 13 | #[cfg(test)] 14 | mod verify_generated_files; 15 | -------------------------------------------------------------------------------- /pb-test/src/verify_generated_files.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | env, 3 | fs, 4 | }; 5 | 6 | use pretty_assertions::assert_eq; 7 | use walkdir::WalkDir; 8 | 9 | /// Ensure that generated files match .expected 10 | #[test] 11 | fn verify_generated_files() { 12 | let proto_files: Vec<_> = WalkDir::new("gen/pb-jelly") 13 | .into_iter() 14 | .filter_map(|e| match e { 15 | Err(_) => None, 16 | Ok(entry) => { 17 | let ext = entry.path().extension().map(|ext| ext.to_str().unwrap()); 18 | match ext { 19 | Some("rs") | Some("toml") => Some(entry.into_path()), 20 | _ => None, 21 | } 22 | }, 23 | }) 24 | .collect(); 25 | 26 | // Assert the correct number of pb-test generated files 27 | // Developers - please change this number if the change is intentional 28 | // assert_eq!(proto_files.len(), 16); 29 | 30 | // Assert contents of the generated files 31 | for proto_file in proto_files { 32 | let filename = proto_file.file_name().unwrap().to_owned().into_string().unwrap() + ".expected"; 33 | let expected = proto_file.parent().unwrap().join(filename); 34 | 35 | if env::var("VALIDATE").is_err() { 36 | // In CI - don't copy over to `.expected` so we can do a validation 37 | fs::copy(&proto_file, &expected).unwrap(); 38 | } 39 | 40 | let contents = fs::read_to_string(proto_file).unwrap(); 41 | let expected_contents = fs::read_to_string(&expected).unwrap(); 42 | assert_eq!( 43 | contents.lines().collect::>(), 44 | expected_contents.lines().collect::>(), 45 | ".expected files don't match - Please run `cd pb-test ; cargo test` to generate" 46 | ); 47 | } 48 | } 49 | --------------------------------------------------------------------------------