├── .github └── workflows │ └── main.yml ├── .gitignore ├── .gitmodules ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── brotli-sys ├── Cargo.toml ├── build.rs └── src │ └── lib.rs ├── examples └── all-read-write-roundtrips.rs ├── src ├── bufread.rs ├── lib.rs ├── raw.rs ├── read.rs └── write.rs ├── systest ├── Cargo.toml ├── build.rs └── src │ └── main.rs └── tests └── drop-incomplete.rs /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | name: Test 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | build: [stable, beta, nightly, macos, win32, win64] 11 | include: 12 | - build: stable 13 | os: ubuntu-latest 14 | rust: stable 15 | - build: beta 16 | os: ubuntu-latest 17 | rust: beta 18 | - build: nightly 19 | os: ubuntu-latest 20 | rust: nightly 21 | - build: macos 22 | os: macos-latest 23 | rust: stable 24 | - build: win32 25 | os: windows-latest 26 | rust: stable-i686-msvc 27 | - build: win64 28 | os: windows-latest 29 | rust: stable-x86_64-msvc 30 | steps: 31 | - uses: actions/checkout@master 32 | - name: Install Rust (rustup) 33 | run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }} 34 | shell: bash 35 | - run: cargo test 36 | - run: cargo run --example all-read-write-roundtrips --release 37 | - run: cargo run --manifest-path systest/Cargo.toml 38 | 39 | rustfmt: 40 | name: Rustfmt 41 | runs-on: ubuntu-latest 42 | steps: 43 | - uses: actions/checkout@master 44 | - name: Install Rust 45 | run: rustup update stable && rustup default stable && rustup component add rustfmt 46 | - run: cargo fmt -- --check 47 | 48 | systest: 49 | name: Systest 50 | runs-on: ubuntu-latest 51 | steps: 52 | - uses: actions/checkout@master 53 | - name: Install Rust 54 | run: rustup update stable && rustup default stable 55 | - run: cargo run --manifest-path systest/Cargo.toml 56 | 57 | publish_docs: 58 | name: Publish Documentation 59 | runs-on: ubuntu-latest 60 | steps: 61 | - uses: actions/checkout@master 62 | - name: Install Rust 63 | run: rustup update stable && rustup default stable 64 | - run: cargo doc --no-deps --all-features 65 | - run: cargo doc --no-deps --all-features --manifest-path brotli-sys/Cargo.toml 66 | - name: Publish documentation 67 | run: | 68 | cd target/doc 69 | git init 70 | git add . 71 | git -c user.name='ci' -c user.email='ci' commit -m init 72 | git push -f -q https://git:${{ secrets.github_token }}@github.com/${{ github.repository }} HEAD:gh-pages 73 | if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "brotli-sys/brotli"] 2 | path = brotli-sys/brotli 3 | url = https://github.com/google/brotli 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "brotli2" 3 | version = "0.3.2" 4 | authors = ["Alex Crichton "] 5 | license = "MIT/Apache-2.0" 6 | readme = "README.md" 7 | repository = "https://github.com/alexcrichton/brotli2-rs" 8 | homepage = "https://github.com/alexcrichton/brotli2-rs" 9 | documentation = "https://docs.rs/brotli2" 10 | description = """ 11 | Bindings to libbrotli to provide brotli decompression and compression to Rust 12 | """ 13 | categories = ["compression", "api-bindings"] 14 | 15 | [dependencies] 16 | brotli-sys = { path = "brotli-sys", version = "0.3.1" } 17 | libc = "0.2" 18 | 19 | [dev-dependencies] 20 | rand = "0.7" 21 | quickcheck = "1.0" 22 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Alex Crichton 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # brotli2 2 | 3 | [Documentation](https://docs.rs/brotli2) 4 | 5 | Bindings to the official [brotli] implementation in Rust. 6 | 7 | [brotli]: https://github.com/google/brotli 8 | 9 | ```toml 10 | # Cargo.toml 11 | [dependencies] 12 | brotli2 = "0.3" 13 | ``` 14 | 15 | # License 16 | 17 | This project is licensed under either of 18 | 19 | * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or 20 | http://www.apache.org/licenses/LICENSE-2.0) 21 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or 22 | http://opensource.org/licenses/MIT) 23 | 24 | at your option. 25 | 26 | ### Contribution 27 | 28 | Unless you explicitly state otherwise, any contribution intentionally submitted 29 | for inclusion in brotli2 by you, as defined in the Apache-2.0 license, shall be 30 | dual licensed as above, without any additional terms or conditions. 31 | -------------------------------------------------------------------------------- /brotli-sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "brotli-sys" 3 | version = "0.3.2" 4 | authors = ["Alex Crichton "] 5 | build = "build.rs" 6 | links = "brotli" 7 | license = "MIT/Apache-2.0" 8 | repository = "https://github.com/alexcrichton/brotli2-rs" 9 | homepage = "https://github.com/alexcrichton/brotli2-rs" 10 | documentation = "https://docs.rs/brotli-sys" 11 | description = """ 12 | Raw bindings to libbrotli 13 | """ 14 | categories = ["external-ffi-bindings"] 15 | 16 | include = [ 17 | "src/*", 18 | "brotli/enc/*", 19 | "brotli/dec/*", 20 | "brotli/common/*", 21 | "brotli/include/*", 22 | "Cargo.toml", 23 | "build.rs" 24 | ] 25 | 26 | [dependencies] 27 | libc = "0.2" 28 | 29 | [build-dependencies] 30 | cc = "1.0" 31 | -------------------------------------------------------------------------------- /brotli-sys/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cc; 2 | 3 | use std::env; 4 | use std::process::Command; 5 | use std::path::Path; 6 | 7 | fn main() { 8 | if !Path::new("brotli/.git").exists() { 9 | let _ = Command::new("git").args(&["submodule", "update", "--init"]) 10 | .status(); 11 | } 12 | 13 | let src = env::current_dir().unwrap(); 14 | println!("cargo:include={}", src.join("brotli/include").display()); 15 | 16 | cc::Build::new() 17 | .include("brotli/include") 18 | .warnings(false) 19 | .file("brotli/common/dictionary.c") 20 | .file("brotli/dec/bit_reader.c") 21 | .file("brotli/dec/decode.c") 22 | .file("brotli/dec/huffman.c") 23 | .file("brotli/dec/state.c") 24 | .file("brotli/enc/backward_references.c") 25 | .file("brotli/enc/backward_references_hq.c") 26 | .file("brotli/enc/bit_cost.c") 27 | .file("brotli/enc/block_splitter.c") 28 | .file("brotli/enc/brotli_bit_stream.c") 29 | .file("brotli/enc/cluster.c") 30 | .file("brotli/enc/compress_fragment.c") 31 | .file("brotli/enc/compress_fragment_two_pass.c") 32 | .file("brotli/enc/dictionary_hash.c") 33 | .file("brotli/enc/encode.c") 34 | .file("brotli/enc/entropy_encode.c") 35 | .file("brotli/enc/histogram.c") 36 | .file("brotli/enc/literal_cost.c") 37 | .file("brotli/enc/memory.c") 38 | .file("brotli/enc/metablock.c") 39 | .file("brotli/enc/static_dict.c") 40 | .file("brotli/enc/utf8_util.c") 41 | .compile("libbrotli.a"); 42 | } 43 | -------------------------------------------------------------------------------- /brotli-sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(bad_style)] 2 | #![doc(html_root_url = "https://docs.rs/brotli-sys/0.2")] 3 | 4 | extern crate libc; 5 | 6 | use libc::{c_void, size_t, c_int, c_char}; 7 | 8 | #[cfg(target_env = "msvc")] 9 | #[doc(hidden)] 10 | pub type __enum_ty = libc::c_int; 11 | #[cfg(not(target_env = "msvc"))] 12 | #[doc(hidden)] 13 | pub type __enum_ty = libc::c_uint; 14 | 15 | pub type __enum_ty_s = libc::c_int; 16 | 17 | pub type brotli_alloc_func = Option *mut c_void>; 18 | pub type brotli_free_func = Option; 19 | 20 | // ========== Decoder functionality ========== 21 | 22 | pub type BrotliDecoderResult = __enum_ty; 23 | pub type BrotliDecoderErrorCode = __enum_ty_s; 24 | 25 | pub enum BrotliDecoderState {} 26 | 27 | pub const BROTLI_DECODER_RESULT_ERROR: BrotliDecoderResult = 0; 28 | pub const BROTLI_DECODER_RESULT_SUCCESS: BrotliDecoderResult = 1; 29 | pub const BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: BrotliDecoderResult = 2; 30 | pub const BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: BrotliDecoderResult = 3; 31 | 32 | pub const BROTLI_DECODER_NO_ERROR: BrotliDecoderErrorCode = 0; 33 | pub const BROTLI_DECODER_SUCCESS: BrotliDecoderErrorCode = 1; 34 | pub const BROTLI_DECODER_NEEDS_MORE_INPUT: BrotliDecoderErrorCode = 2; 35 | pub const BROTLI_DECODER_NEEDS_MORE_OUTPUT: BrotliDecoderErrorCode = 3; 36 | pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE: BrotliDecoderErrorCode = -1; 37 | pub const BROTLI_DECODER_ERROR_FORMAT_RESERVED: BrotliDecoderErrorCode = -2; 38 | pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE: BrotliDecoderErrorCode = -3; 39 | pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET: BrotliDecoderErrorCode = -4; 40 | pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME: BrotliDecoderErrorCode = -5; 41 | pub const BROTLI_DECODER_ERROR_FORMAT_CL_SPACE: BrotliDecoderErrorCode = -6; 42 | pub const BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE: BrotliDecoderErrorCode = -7; 43 | pub const BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT: BrotliDecoderErrorCode = -8; 44 | pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1: BrotliDecoderErrorCode = -9; 45 | pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2: BrotliDecoderErrorCode = -10; 46 | pub const BROTLI_DECODER_ERROR_FORMAT_TRANSFORM: BrotliDecoderErrorCode = -11; 47 | pub const BROTLI_DECODER_ERROR_FORMAT_DICTIONARY: BrotliDecoderErrorCode = -12; 48 | pub const BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: BrotliDecoderErrorCode = -13; 49 | pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_1: BrotliDecoderErrorCode = -14; 50 | pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_2: BrotliDecoderErrorCode = -15; 51 | pub const BROTLI_DECODER_ERROR_INVALID_ARGUMENTS: BrotliDecoderErrorCode = -20; 52 | pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES: BrotliDecoderErrorCode = -21; 53 | pub const BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS: BrotliDecoderErrorCode = -22; 54 | pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP: BrotliDecoderErrorCode = -25; 55 | pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1: BrotliDecoderErrorCode = -26; 56 | pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2: BrotliDecoderErrorCode = -27; 57 | pub const BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES: BrotliDecoderErrorCode = -30; 58 | pub const BROTLI_DECODER_ERROR_UNREACHABLE: BrotliDecoderErrorCode = -31; 59 | 60 | extern "C" { 61 | pub fn BrotliDecoderCreateInstance(alloc_func: brotli_alloc_func, 62 | free_func: brotli_free_func, 63 | opaque: *mut c_void) 64 | -> *mut BrotliDecoderState; 65 | pub fn BrotliDecoderDestroyInstance(state: *mut BrotliDecoderState); 66 | pub fn BrotliDecoderDecompress(encoded_size: size_t, 67 | encoded_buffer: *const u8, 68 | decoded_size: *mut size_t, 69 | decoded_buffer: *mut u8) -> 70 | BrotliDecoderResult; 71 | pub fn BrotliDecoderDecompressStream(state: *mut BrotliDecoderState, 72 | available_in: *mut size_t, 73 | next_in: *mut *const u8, 74 | available_out: *mut size_t, 75 | next_out: *mut *mut u8, 76 | total_out: *mut size_t) 77 | -> BrotliDecoderResult; 78 | pub fn BrotliDecoderSetCustomDictionary(state: *mut BrotliDecoderState, 79 | size: size_t, 80 | dict: *const u8); 81 | pub fn BrotliDecoderHasMoreOutput(state: *const BrotliDecoderState) -> c_int; 82 | pub fn BrotliDecoderTakeOutput(state: *mut BrotliDecoderState, 83 | size: *mut size_t) 84 | -> *const u8; 85 | pub fn BrotliDecoderIsUsed(state: *const BrotliDecoderState) -> c_int; 86 | pub fn BrotliDecoderIsFinished(state: *const BrotliDecoderState) -> c_int; 87 | pub fn BrotliDecoderGetErrorCode(state: *const BrotliDecoderState) -> BrotliDecoderErrorCode; 88 | pub fn BrotliDecoderErrorString(c: BrotliDecoderErrorCode) -> *const c_char; 89 | pub fn BrotliDecoderVersion() -> u32; 90 | } 91 | 92 | 93 | 94 | // ========== Encoder functionality ========== 95 | 96 | pub type BrotliEncoderMode = __enum_ty; 97 | pub type BrotliEncoderParameter = __enum_ty; 98 | pub type BrotliEncoderOperation = __enum_ty; 99 | 100 | pub const BROTLI_MODE_GENERIC: BrotliEncoderMode = 0; 101 | pub const BROTLI_MODE_TEXT: BrotliEncoderMode = 1; 102 | pub const BROTLI_MODE_FONT: BrotliEncoderMode = 2; 103 | 104 | pub const BROTLI_PARAM_MODE: BrotliEncoderParameter = 0; 105 | pub const BROTLI_PARAM_QUALITY: BrotliEncoderParameter = 1; 106 | pub const BROTLI_PARAM_LGWIN: BrotliEncoderParameter = 2; 107 | pub const BROTLI_PARAM_LGBLOCK: BrotliEncoderParameter = 3; 108 | pub const BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING: BrotliEncoderParameter = 4; 109 | pub const BROTLI_PARAM_SIZE_HINT: BrotliEncoderParameter = 5; 110 | 111 | pub const BROTLI_OPERATION_PROCESS: BrotliEncoderOperation = 0; 112 | pub const BROTLI_OPERATION_FLUSH: BrotliEncoderOperation = 1; 113 | pub const BROTLI_OPERATION_FINISH: BrotliEncoderOperation = 2; 114 | pub const BROTLI_OPERATION_EMIT_METADATA: BrotliEncoderOperation = 3; 115 | 116 | pub const BROTLI_DEFAULT_QUALITY: u32 = 11; 117 | pub const BROTLI_DEFAULT_WINDOW: u32 = 22; 118 | pub const BROTLI_DEFAULT_MODE: u32 = 0; 119 | 120 | pub enum BrotliEncoderState {} 121 | 122 | extern "C" { 123 | pub fn BrotliEncoderSetParameter(state: *mut BrotliEncoderState, 124 | param: BrotliEncoderParameter, 125 | value: u32) 126 | -> c_int; 127 | pub fn BrotliEncoderCreateInstance(alloc_func: brotli_alloc_func, 128 | free_func: brotli_free_func, 129 | opaque: *mut c_void) 130 | -> *mut BrotliEncoderState; 131 | pub fn BrotliEncoderDestroyInstance(state: *mut BrotliEncoderState); 132 | // These three are deprecated 133 | //pub fn BrotliEncoderInputBlockSize(state: *mut BrotliEncoderState) -> size_t; 134 | //pub fn BrotliEncoderCopyInputToRingBuffer(state: *mut BrotliEncoderState, 135 | // input_size: size_t, 136 | // input_buffer: *const u8); 137 | //pub fn BrotliEncoderWriteData(state: *mut BrotliEncoderState, 138 | // is_last: c_int, 139 | // force_flush: c_int, 140 | // out_size: *mut size_t, 141 | // output: *mut *mut u8) 142 | // -> c_int; 143 | pub fn BrotliEncoderSetCustomDictionary(state: *mut BrotliEncoderState, 144 | size: size_t, 145 | dict: *const u8); 146 | pub fn BrotliEncoderMaxCompressedSize(input_size: size_t) -> size_t; 147 | pub fn BrotliEncoderCompress(quality: c_int, 148 | lgwin: c_int, 149 | mode: BrotliEncoderMode, 150 | input_size: size_t, 151 | input_buffer: *const u8, 152 | encoded_size: *mut size_t, 153 | encoded_buffer: *mut u8) 154 | -> c_int; 155 | pub fn BrotliEncoderCompressStream(state: *mut BrotliEncoderState, 156 | op: BrotliEncoderOperation, 157 | available_in: *mut size_t, 158 | next_in: *mut *const u8, 159 | available_out: *mut size_t, 160 | next_out: *mut *mut u8, 161 | total_out: *mut size_t) 162 | -> c_int; 163 | pub fn BrotliEncoderIsFinished(state: *mut BrotliEncoderState) -> c_int; 164 | pub fn BrotliEncoderHasMoreOutput(state: *mut BrotliEncoderState) -> c_int; 165 | pub fn BrotliEncoderTakeOutput(state: *mut BrotliEncoderState, 166 | size: *mut usize) 167 | -> *const u8; 168 | pub fn BrotliEncoderVersion() -> u32; 169 | } 170 | -------------------------------------------------------------------------------- /examples/all-read-write-roundtrips.rs: -------------------------------------------------------------------------------- 1 | extern crate brotli2; 2 | extern crate rand; 3 | 4 | use brotli2::raw::{compress_buf, decompress_buf}; 5 | use brotli2::read; 6 | use brotli2::write; 7 | use brotli2::CompressParams; 8 | use rand::Rng; 9 | use std::io::{Read, Write}; 10 | 11 | // Used in functions as temporary storage space before producing a vec 12 | const BIGBUF_SIZE: usize = 120 * 1024 * 1024; 13 | static mut BIGBUF: [u8; BIGBUF_SIZE] = [0; BIGBUF_SIZE]; 14 | 15 | fn main() { 16 | let v1 = vec![1; 1024]; 17 | let v2 = vec![44; 10 * 1024 * 1024]; 18 | let datas: &[&[u8]] = &[b"", b"a", b"aaaaa", b";", &v1, &v2]; 19 | let mut params = CompressParams::new(); 20 | params.quality(6); 21 | let params = ¶ms; 22 | 23 | fn bufencode(data: &[u8], params: &CompressParams) -> Vec { 24 | let bufref = &mut unsafe { &mut BIGBUF[..] }; 25 | let n = compress_buf(params, data, bufref).unwrap(); 26 | assert!(n > 0 && n <= BIGBUF_SIZE && n == bufref.len()); 27 | bufref.to_owned() 28 | } 29 | fn bufdecode(data: &[u8]) -> Vec { 30 | let bufref = &mut unsafe { &mut BIGBUF[..] }; 31 | let n = decompress_buf(data, bufref).unwrap(); 32 | assert!(n <= BIGBUF_SIZE && n == bufref.len()); 33 | bufref.to_owned() 34 | } 35 | fn ioreadencode(data: &[u8], params: &CompressParams) -> Vec { 36 | let mut buf = vec![]; 37 | read::BrotliEncoder::from_params(data, params) 38 | .read_to_end(&mut buf) 39 | .unwrap(); 40 | assert!(buf.len() > 0); 41 | buf 42 | } 43 | fn ioreaddecode(data: &[u8]) -> Vec { 44 | let mut buf = vec![]; 45 | read::BrotliDecoder::new(data) 46 | .read_to_end(&mut buf) 47 | .unwrap(); 48 | buf 49 | } 50 | fn iowriteencode(data: &[u8], params: &CompressParams) -> Vec { 51 | let mut enc = write::BrotliEncoder::from_params(vec![], params); 52 | enc.write_all(data).unwrap(); 53 | enc.finish().unwrap() 54 | } 55 | fn iowritedecode(data: &[u8]) -> Vec { 56 | let mut dec = write::BrotliDecoder::new(vec![]); 57 | dec.write_all(data).unwrap(); 58 | dec.finish().unwrap() 59 | } 60 | 61 | let check = |data: &[u8]| { 62 | let c1 = &bufencode(data, params); 63 | let c2 = &ioreadencode(data, params); 64 | let c3 = &iowriteencode(data, params); 65 | assert!(data == &*bufdecode(c1)); 66 | assert!(data == &*ioreaddecode(c1)); 67 | assert!(data == &*iowritedecode(c1)); 68 | // it's valid for them to be different, but we need to do more work 69 | if c2 != c1 { 70 | assert!(data == &*bufdecode(c2)); 71 | assert!(data == &*ioreaddecode(c2)); 72 | assert!(data == &*iowritedecode(c2)); 73 | } 74 | if c3 != c1 && c3 != c2 { 75 | assert!(data == &*bufdecode(c2)); 76 | assert!(data == &*ioreaddecode(c2)); 77 | assert!(data == &*iowritedecode(c2)); 78 | } 79 | }; 80 | 81 | for &data in datas.iter() { 82 | check(data) 83 | } 84 | let mut rng = rand::thread_rng(); 85 | for _ in 0..3 { 86 | let rnum: usize = rng.gen_range(1, 100 * 1024 * 1024); 87 | let mut buf = vec![0; rnum]; 88 | rng.fill(&mut buf[..]); 89 | check(&buf) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/bufread.rs: -------------------------------------------------------------------------------- 1 | //! I/O streams for wrapping `BufRead` types as encoders/decoders 2 | 3 | use std::io; 4 | use std::io::prelude::*; 5 | 6 | use super::CompressParams; 7 | use raw::{self, CoStatus, Compress, CompressOp, DeStatus, Decompress}; 8 | 9 | #[derive(Clone, Copy, Eq, PartialEq)] 10 | enum DoneStatus { 11 | Processing, 12 | Finishing, 13 | Done, 14 | } 15 | 16 | /// A brotli encoder, or compressor. 17 | /// 18 | /// This structure implements a `BufRead` interface and will read uncompressed 19 | /// data from an underlying stream and emit a stream of compressed data. 20 | pub struct BrotliEncoder { 21 | obj: R, 22 | data: Compress, 23 | done: DoneStatus, 24 | err: Option, 25 | } 26 | 27 | /// A brotli decoder, or decompressor. 28 | /// 29 | /// This structure implements a `BufRead` interface and takes a stream of 30 | /// compressed data as input, providing the decompressed data when read from. 31 | pub struct BrotliDecoder { 32 | obj: R, 33 | data: Decompress, 34 | err: Option, 35 | } 36 | 37 | impl BrotliEncoder { 38 | /// Creates a new encoder which will read uncompressed data from the given 39 | /// stream and emit the compressed stream. 40 | /// 41 | /// The `level` argument here is typically 0-11. 42 | pub fn new(r: R, level: u32) -> BrotliEncoder { 43 | let mut data = Compress::new(); 44 | data.set_params(CompressParams::new().quality(level)); 45 | BrotliEncoder { 46 | obj: r, 47 | data: data, 48 | done: DoneStatus::Processing, 49 | err: None, 50 | } 51 | } 52 | 53 | /// Creates a new encoder with a custom `CompressParams`. 54 | pub fn from_params(r: R, params: &CompressParams) -> BrotliEncoder { 55 | let mut data = Compress::new(); 56 | data.set_params(params); 57 | BrotliEncoder { 58 | obj: r, 59 | data: data, 60 | done: DoneStatus::Processing, 61 | err: None, 62 | } 63 | } 64 | 65 | /// Acquires a reference to the underlying stream 66 | pub fn get_ref(&self) -> &R { 67 | &self.obj 68 | } 69 | 70 | /// Acquires a mutable reference to the underlying stream 71 | /// 72 | /// Note that mutation of the stream may result in surprising results if 73 | /// this encoder is continued to be used. 74 | pub fn get_mut(&mut self) -> &mut R { 75 | &mut self.obj 76 | } 77 | 78 | /// Consumes this encoder, returning the underlying reader. 79 | pub fn into_inner(self) -> R { 80 | self.obj 81 | } 82 | } 83 | 84 | impl Read for BrotliEncoder { 85 | fn read(&mut self, mut buf: &mut [u8]) -> io::Result { 86 | if buf.is_empty() { 87 | return Ok(0); 88 | } 89 | // If the compressor has failed at some point, this is set. 90 | // Unfortunately we have no idea what status is in the compressor 91 | // was in when it failed so we can't do anything except bail again. 92 | if let Some(ref err) = self.err { 93 | return Err(err.clone().into()); 94 | } 95 | 96 | if let Some(data) = self.data.take_output(Some(buf.len())) { 97 | buf[..data.len()].copy_from_slice(data); 98 | return Ok(data.len()); 99 | } 100 | 101 | match self.done { 102 | DoneStatus::Done => return Ok(0), 103 | DoneStatus::Finishing => return tryfinish(self, buf), 104 | DoneStatus::Processing => (), 105 | } 106 | 107 | loop { 108 | let amt_in; 109 | let amt_out; 110 | { 111 | let input = &mut self.obj.fill_buf()?; 112 | let avail_in = input.len(); 113 | if avail_in == 0 { 114 | break; 115 | } 116 | let output = &mut buf; 117 | let avail_out = output.len(); 118 | if let Err(err) = self.data.compress(CompressOp::Process, input, output) { 119 | self.err = Some(err.clone().into()); 120 | return Err(err.into()); 121 | } 122 | amt_in = avail_in - input.len(); 123 | amt_out = avail_out - output.len(); 124 | } 125 | self.obj.consume(amt_in); 126 | 127 | if amt_out == 0 { 128 | assert!(amt_in != 0); 129 | continue; 130 | } 131 | return Ok(amt_out); 132 | } 133 | self.done = DoneStatus::Finishing; 134 | return tryfinish(self, buf); 135 | 136 | fn tryfinish( 137 | enc: &mut BrotliEncoder, 138 | mut buf: &mut [u8], 139 | ) -> io::Result { 140 | let output = &mut buf; 141 | let avail_out = output.len(); 142 | let iscomplete = match enc.data.compress(CompressOp::Finish, &mut &[][..], output) { 143 | Ok(c) => c, 144 | Err(err) => { 145 | enc.err = err.clone().into(); 146 | return Err(err.into()); 147 | } 148 | }; 149 | let written = avail_out - output.len(); 150 | assert!(written != 0 || iscomplete == CoStatus::Finished); 151 | if iscomplete == CoStatus::Finished { 152 | enc.done = DoneStatus::Done 153 | } 154 | Ok(written) 155 | } 156 | } 157 | } 158 | 159 | impl BrotliDecoder { 160 | /// Creates a new decoder which will decompress data read from the given 161 | /// stream. 162 | pub fn new(r: R) -> BrotliDecoder { 163 | BrotliDecoder { 164 | data: Decompress::new(), 165 | obj: r, 166 | err: None, 167 | } 168 | } 169 | 170 | /// Acquires a reference to the underlying stream 171 | pub fn get_ref(&self) -> &R { 172 | &self.obj 173 | } 174 | 175 | /// Acquires a mutable reference to the underlying stream 176 | /// 177 | /// Note that mutation of the stream may result in surprising results if 178 | /// this encoder is continued to be used. 179 | pub fn get_mut(&mut self) -> &mut R { 180 | &mut self.obj 181 | } 182 | 183 | /// Consumes this decoder, returning the underlying reader. 184 | pub fn into_inner(self) -> R { 185 | self.obj 186 | } 187 | } 188 | 189 | impl Read for BrotliDecoder { 190 | fn read(&mut self, mut buf: &mut [u8]) -> io::Result { 191 | if buf.is_empty() { 192 | return Ok(0); 193 | } 194 | // If the decompressor has failed at some point, this is set. 195 | // Unfortunately we have no idea what status is in the compressor 196 | // was in when it failed so we can't do anything except bail again. 197 | if let Some(ref err) = self.err { 198 | return Err(err.clone().into()); 199 | } 200 | 201 | loop { 202 | let amt_in; 203 | let amt_out; 204 | let status; 205 | { 206 | let mut input = self.obj.fill_buf()?; 207 | let avail_in = input.len(); 208 | let avail_out = buf.len(); 209 | status = match self.data.decompress(&mut input, &mut buf) { 210 | Ok(s) => s, 211 | Err(err) => { 212 | self.err = Some(err.clone().into()); 213 | return Err(err.into()); 214 | } 215 | }; 216 | amt_in = avail_in - input.len(); 217 | amt_out = avail_out - buf.len() 218 | } 219 | self.obj.consume(amt_in); 220 | 221 | if amt_in == 0 && status == DeStatus::NeedInput { 222 | return Err(io::Error::new( 223 | io::ErrorKind::Other, 224 | "corrupted brotli stream", 225 | )); 226 | } 227 | if amt_out == 0 && status != DeStatus::Finished { 228 | assert!(amt_in != 0); 229 | continue; 230 | } 231 | 232 | return Ok(amt_out); 233 | } 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Brotli Compression/Decompression for Rust 2 | //! 3 | //! This crate is a binding to the [official brotli implementation][brotli] and 4 | //! provides in-memory and I/O streams for Rust wrappers. 5 | //! 6 | //! [brotli]: https://github.com/google/brotli 7 | //! 8 | //! # Examples 9 | //! 10 | //! ``` 11 | //! use std::io::prelude::*; 12 | //! use brotli2::read::{BrotliEncoder, BrotliDecoder}; 13 | //! 14 | //! // Round trip some bytes from a byte source, into a compressor, into a 15 | //! // decompressor, and finally into a vector. 16 | //! let data = "Hello, World!".as_bytes(); 17 | //! let compressor = BrotliEncoder::new(data, 9); 18 | //! let mut decompressor = BrotliDecoder::new(compressor); 19 | //! 20 | //! let mut contents = String::new(); 21 | //! decompressor.read_to_string(&mut contents).unwrap(); 22 | //! assert_eq!(contents, "Hello, World!"); 23 | //! ``` 24 | 25 | #![deny(missing_docs)] 26 | #![doc(html_root_url = "https://docs.rs/brotli2/0.2")] 27 | 28 | extern crate brotli_sys; 29 | extern crate libc; 30 | 31 | #[cfg(test)] 32 | extern crate quickcheck; 33 | #[cfg(test)] 34 | extern crate rand; 35 | 36 | pub mod bufread; 37 | pub mod raw; 38 | pub mod read; 39 | pub mod write; 40 | 41 | /// Possible choices for modes of compression 42 | #[repr(isize)] 43 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 44 | pub enum CompressMode { 45 | /// Default compression mode, the compressor does not know anything in 46 | /// advance about the properties of the input. 47 | Generic = brotli_sys::BROTLI_MODE_GENERIC as isize, 48 | /// Compression mode for utf-8 formatted text input. 49 | Text = brotli_sys::BROTLI_MODE_TEXT as isize, 50 | /// Compression mode in WOFF 2.0. 51 | Font = brotli_sys::BROTLI_MODE_FONT as isize, 52 | } 53 | 54 | /// Parameters passed to various compression routines. 55 | #[derive(Clone, Debug)] 56 | pub struct CompressParams { 57 | /// Compression mode. 58 | mode: u32, 59 | /// Controls the compression-speed vs compression-density tradeoffs. The higher the `quality`, 60 | /// the slower the compression. Range is 0 to 11. 61 | quality: u32, 62 | /// Base 2 logarithm of the sliding window size. Range is 10 to 24. 63 | lgwin: u32, 64 | /// Base 2 logarithm of the maximum input block size. Range is 16 to 24. If set to 0, the value 65 | /// will be set based on the quality. 66 | lgblock: u32, 67 | } 68 | 69 | impl CompressParams { 70 | /// Creates a new default set of compression parameters. 71 | pub fn new() -> CompressParams { 72 | CompressParams { 73 | mode: brotli_sys::BROTLI_DEFAULT_MODE, 74 | quality: brotli_sys::BROTLI_DEFAULT_QUALITY, 75 | lgwin: brotli_sys::BROTLI_DEFAULT_WINDOW, 76 | lgblock: 0, 77 | } 78 | } 79 | 80 | /// Set the mode of this compression. 81 | pub fn mode(&mut self, mode: CompressMode) -> &mut CompressParams { 82 | self.mode = mode as u32; 83 | self 84 | } 85 | 86 | /// Controls the compression-speed vs compression-density tradeoffs. 87 | /// 88 | /// The higher the quality, the slower the compression. Currently the range 89 | /// for the quality is 0 to 11. 90 | pub fn quality(&mut self, quality: u32) -> &mut CompressParams { 91 | self.quality = quality; 92 | self 93 | } 94 | 95 | /// Sets the base 2 logarithm of the sliding window size. 96 | /// 97 | /// Currently the range is 10 to 24. 98 | pub fn lgwin(&mut self, lgwin: u32) -> &mut CompressParams { 99 | self.lgwin = lgwin; 100 | self 101 | } 102 | 103 | /// Sets the base 2 logarithm of the maximum input block size. 104 | /// 105 | /// Currently the range is 16 to 24, and if set to 0 the value will be set 106 | /// based on the quality. 107 | pub fn lgblock(&mut self, lgblock: u32) -> &mut CompressParams { 108 | self.lgblock = lgblock; 109 | self 110 | } 111 | 112 | /// Get the current block size 113 | #[inline] 114 | pub fn get_lgblock_readable(&self) -> usize { 115 | 1usize << self.lgblock 116 | } 117 | 118 | /// Get the native lgblock size 119 | #[inline] 120 | pub fn get_lgblock(&self) -> u32 { 121 | self.lgblock.clone() 122 | } 123 | /// Get the current window size 124 | #[inline] 125 | pub fn get_lgwin_readable(&self) -> usize { 126 | 1usize << self.lgwin 127 | } 128 | /// Get the native lgwin value 129 | #[inline] 130 | pub fn get_lgwin(&self) -> u32 { 131 | self.lgwin.clone() 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/raw.rs: -------------------------------------------------------------------------------- 1 | //! Raw interface to in-memory compression/decompression streams 2 | 3 | use std::error; 4 | use std::fmt; 5 | use std::io; 6 | use std::mem; 7 | use std::ptr; 8 | use std::slice; 9 | 10 | use brotli_sys; 11 | use libc::c_int; 12 | 13 | use super::CompressParams; 14 | 15 | /// In-memory state for decompressing brotli-encoded data. 16 | /// 17 | /// This stream is at the heart of the I/O streams and is used to decompress an 18 | /// incoming brotli stream. 19 | pub struct Decompress { 20 | state: *mut brotli_sys::BrotliDecoderState, 21 | } 22 | 23 | unsafe impl Send for Decompress {} 24 | unsafe impl Sync for Decompress {} 25 | 26 | /// In-memory state for compressing/encoding data with brotli 27 | /// 28 | /// This stream is at the heart of the I/O encoders and is used to compress 29 | /// data. 30 | pub struct Compress { 31 | state: *mut brotli_sys::BrotliEncoderState, 32 | } 33 | 34 | unsafe impl Send for Compress {} 35 | unsafe impl Sync for Compress {} 36 | 37 | /// Possible choices for the operation performed by the compressor. 38 | /// 39 | /// When using any operation except `Process`, you must *not* alter the 40 | /// input buffer or use a different operation until the current operation 41 | /// has 'completed'. An operation may need to be repeated with more space to 42 | /// write data until it can complete. 43 | #[repr(isize)] 44 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 45 | pub enum CompressOp { 46 | /// Compress input data 47 | Process = brotli_sys::BROTLI_OPERATION_PROCESS as isize, 48 | /// Compress input data, ensuring that all input so far has been 49 | /// written out 50 | Flush = brotli_sys::BROTLI_OPERATION_FLUSH as isize, 51 | /// Compress input data, ensuring that all input so far has been 52 | /// written out and then finalizing the stream so no more data can 53 | /// be written 54 | Finish = brotli_sys::BROTLI_OPERATION_FINISH as isize, 55 | /// Emit a metadata block to the stream, an opaque piece of out-of-band 56 | /// data that does not interfere with the main stream of data. Metadata 57 | /// blocks *must* be no longer than 16MiB 58 | EmitMetadata = brotli_sys::BROTLI_OPERATION_EMIT_METADATA as isize, 59 | } 60 | 61 | /// Error that can happen from decompressing or compressing a brotli stream. 62 | #[derive(Debug, Clone, PartialEq)] 63 | pub struct Error(()); 64 | 65 | /// Indication of whether a compression operation is 'complete'. This does 66 | /// not indicate whether the whole stream is complete - see `Compress::compress` 67 | /// for details. 68 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 69 | pub enum CoStatus { 70 | /// The operation completed successfully 71 | Finished, 72 | /// The operation has more work to do and needs to be called again with the 73 | /// same buffer 74 | Unfinished, 75 | } 76 | 77 | /// Possible status results returned from decompressing. 78 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 79 | pub enum DeStatus { 80 | /// Decompression was successful and has finished 81 | Finished, 82 | /// More input is needed to continue 83 | NeedInput, 84 | /// More output is needed to continue 85 | NeedOutput, 86 | } 87 | 88 | impl Decompress { 89 | /// Creates a new brotli decompression/decoding stream ready to receive 90 | /// data. 91 | pub fn new() -> Decompress { 92 | unsafe { 93 | let state = brotli_sys::BrotliDecoderCreateInstance(None, None, 0 as *mut _); 94 | assert!(!state.is_null()); 95 | Decompress { state: state } 96 | } 97 | } 98 | 99 | /// Decompress some input data and write it to a buffer of output data. 100 | /// 101 | /// This function will decompress the data in `input` and place the output 102 | /// in `output`, returning the result. Possible statuses that can be 103 | /// returned are that the stream is finished, more input is needed, or more 104 | /// output space is needed. 105 | /// 106 | /// The `input` slice is updated to point to the remaining data that was not 107 | /// consumed, and the `output` slice is updated to point to the portion of 108 | /// the output slice that still needs to be filled in. 109 | /// 110 | /// # Errors 111 | /// 112 | /// If the input stream is not a valid brotli stream, then an error is 113 | /// returned. 114 | pub fn decompress( 115 | &mut self, 116 | input: &mut &[u8], 117 | output: &mut &mut [u8], 118 | ) -> Result { 119 | let mut available_in = input.len(); 120 | let mut next_in = input.as_ptr(); 121 | let mut available_out = output.len(); 122 | let mut next_out = output.as_mut_ptr(); 123 | let r = unsafe { 124 | brotli_sys::BrotliDecoderDecompressStream( 125 | self.state, 126 | &mut available_in, 127 | &mut next_in, 128 | &mut available_out, 129 | &mut next_out, 130 | ptr::null_mut(), 131 | ) 132 | }; 133 | *input = &input[input.len() - available_in..]; 134 | let out_len = output.len(); 135 | *output = &mut mem::replace(output, &mut [])[out_len - available_out..]; 136 | Decompress::rc(r) 137 | } 138 | 139 | /// Retrieve a slice of the internal decompressor buffer up to `size_limit` in length 140 | /// (unlimited length if `None`), consuming it. As the internal buffer may not be 141 | /// contiguous, consecutive calls may return more output until this function returns 142 | /// `None`. 143 | pub fn take_output(&mut self, size_limit: Option) -> Option<&[u8]> { 144 | if let Some(0) = size_limit { 145 | return None; 146 | } 147 | let mut size_limit = size_limit.unwrap_or(0); // 0 now means unlimited 148 | unsafe { 149 | let ptr = brotli_sys::BrotliDecoderTakeOutput(self.state, &mut size_limit); 150 | if size_limit == 0 { 151 | // ptr may or may not be null 152 | None 153 | } else { 154 | assert!(!ptr.is_null()); 155 | Some(slice::from_raw_parts(ptr, size_limit)) 156 | } 157 | } 158 | } 159 | 160 | fn rc(rc: brotli_sys::BrotliDecoderResult) -> Result { 161 | match rc { 162 | // TODO: get info from BrotliDecoderGetErrorCode/BrotliDecoderErrorString 163 | // for these decode errors 164 | brotli_sys::BROTLI_DECODER_RESULT_ERROR => Err(Error(())), 165 | brotli_sys::BROTLI_DECODER_RESULT_SUCCESS => Ok(DeStatus::Finished), 166 | brotli_sys::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT => Ok(DeStatus::NeedInput), 167 | brotli_sys::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT => Ok(DeStatus::NeedOutput), 168 | n => panic!("unknown return code: {}", n), 169 | } 170 | } 171 | } 172 | 173 | impl Drop for Decompress { 174 | fn drop(&mut self) { 175 | unsafe { 176 | brotli_sys::BrotliDecoderDestroyInstance(self.state); 177 | } 178 | } 179 | } 180 | 181 | /// Decompress data in one go in memory. 182 | /// 183 | /// Decompresses the data in `input` into the `output` buffer. The `output` 184 | /// buffer is updated to point to the actual output slice if successful, or 185 | /// an error is returned. The output buffer being too small is considered 186 | /// to be an error. 187 | pub fn decompress_buf(input: &[u8], output: &mut &mut [u8]) -> Result { 188 | let mut size = output.len(); 189 | let r = unsafe { 190 | brotli_sys::BrotliDecoderDecompress( 191 | input.len(), 192 | input.as_ptr(), 193 | &mut size, 194 | output.as_mut_ptr(), 195 | ) 196 | }; 197 | *output = &mut mem::replace(output, &mut [])[..size]; 198 | if r == 0 { 199 | Err(Error(())) 200 | } else { 201 | Ok(size) 202 | } 203 | } 204 | 205 | impl Compress { 206 | /// Creates a new compressor ready to encode data into brotli 207 | pub fn new() -> Compress { 208 | unsafe { 209 | let state = brotli_sys::BrotliEncoderCreateInstance(None, None, 0 as *mut _); 210 | assert!(!state.is_null()); 211 | 212 | Compress { state: state } 213 | } 214 | } 215 | 216 | // TODO: add the BrotliEncoderOperation variants of 217 | // BrotliEncoderCompressStream here 218 | 219 | /// Pass some input data to the compressor and write it to a buffer of 220 | /// output data, compressing or otherwise handling it as instructed by 221 | /// the specified operation. 222 | /// 223 | /// This function will handle the data in `input` and place the output 224 | /// in `output`, returning the Result. Possible statuses are that the 225 | /// operation is complete or incomplete. 226 | /// 227 | /// The `input` slice is updated to point to the remaining data that was not 228 | /// consumed, and the `output` slice is updated to point to the portion of 229 | /// the output slice that still needs to be filled in. 230 | /// 231 | /// If the result of a compress operation is `Unfinished` (which it may be 232 | /// for any operation except `Process`), you *must* call the operation again 233 | /// with the same operation and input buffer and more space to output to. 234 | /// `Process` will never return `Unfinished`, but it is a logic error to end 235 | /// a buffer without calling either `Flush` or `Finish` as some output data 236 | /// may not have been written. 237 | /// 238 | /// # Errors 239 | /// 240 | /// Returns an error if brotli encountered an error while processing the stream. 241 | /// 242 | /// # Examples 243 | /// 244 | /// ``` 245 | /// use brotli2::raw::{Error, Compress, CompressOp, CoStatus, decompress_buf}; 246 | /// 247 | /// // An example of compressing `input` into the destination vector 248 | /// // `output`, expanding as necessary 249 | /// fn compress_vec(mut input: &[u8], 250 | /// output: &mut Vec) -> Result<(), Error> { 251 | /// let mut compress = Compress::new(); 252 | /// let nilbuf = &mut &mut [][..]; 253 | /// loop { 254 | /// // Compressing to a buffer is easiest when the slice is already 255 | /// // available - since we need to grow, extend from compressor 256 | /// // internal buffer. 257 | /// let status = compress.compress(CompressOp::Finish, &mut input, nilbuf)?; 258 | /// while let Some(buf) = compress.take_output(None) { 259 | /// output.extend_from_slice(buf) 260 | /// } 261 | /// match status { 262 | /// CoStatus::Finished => break, 263 | /// CoStatus::Unfinished => (), 264 | /// } 265 | /// } 266 | /// Ok(()) 267 | /// } 268 | /// 269 | /// fn assert_roundtrip(data: &[u8]) { 270 | /// let mut compressed = Vec::new(); 271 | /// compress_vec(data, &mut compressed).unwrap(); 272 | /// 273 | /// let mut decompressed = [0; 2048]; 274 | /// let mut decompressed = &mut decompressed[..]; 275 | /// decompress_buf(&compressed, &mut decompressed).unwrap(); 276 | /// assert_eq!(decompressed, data); 277 | /// } 278 | /// 279 | /// assert_roundtrip(b"Hello, World!"); 280 | /// assert_roundtrip(b""); 281 | /// assert_roundtrip(&[6; 1024]); 282 | /// ``` 283 | pub fn compress( 284 | &mut self, 285 | op: CompressOp, 286 | input: &mut &[u8], 287 | output: &mut &mut [u8], 288 | ) -> Result { 289 | let mut available_in = input.len(); 290 | let mut next_in = input.as_ptr(); 291 | let mut available_out = output.len(); 292 | let mut next_out = output.as_mut_ptr(); 293 | let r = unsafe { 294 | brotli_sys::BrotliEncoderCompressStream( 295 | self.state, 296 | op as brotli_sys::BrotliEncoderOperation, 297 | &mut available_in, 298 | &mut next_in, 299 | &mut available_out, 300 | &mut next_out, 301 | ptr::null_mut(), 302 | ) 303 | }; 304 | *input = &input[input.len() - available_in..]; 305 | let out_len = output.len(); 306 | *output = &mut mem::replace(output, &mut [])[out_len - available_out..]; 307 | if r == 0 { 308 | return Err(Error(())); 309 | } 310 | Ok(if op == CompressOp::Process { 311 | CoStatus::Finished 312 | } else if available_in != 0 { 313 | CoStatus::Unfinished 314 | } else if unsafe { brotli_sys::BrotliEncoderHasMoreOutput(self.state) } == 1 { 315 | CoStatus::Unfinished 316 | } else if op == CompressOp::Finish 317 | && unsafe { brotli_sys::BrotliEncoderIsFinished(self.state) } == 0 318 | { 319 | CoStatus::Unfinished 320 | } else { 321 | CoStatus::Finished 322 | }) 323 | } 324 | 325 | /// Retrieve a slice of the internal compressor buffer up to `size_limit` in length 326 | /// (unlimited length if `None`), consuming it. As the internal buffer may not be 327 | /// contiguous, consecutive calls may return more output until this function returns 328 | /// `None`. 329 | pub fn take_output(&mut self, size_limit: Option) -> Option<&[u8]> { 330 | if let Some(0) = size_limit { 331 | return None; 332 | } 333 | let mut size_limit = size_limit.unwrap_or(0); // 0 now means unlimited 334 | unsafe { 335 | let ptr = brotli_sys::BrotliEncoderTakeOutput(self.state, &mut size_limit); 336 | if size_limit == 0 { 337 | // ptr may or may not be null 338 | None 339 | } else { 340 | assert!(!ptr.is_null()); 341 | Some(slice::from_raw_parts(ptr, size_limit)) 342 | } 343 | } 344 | } 345 | 346 | /// Configure the parameters of this compression session. 347 | /// 348 | /// Note that this is likely to only successful if called before compression 349 | /// starts. 350 | pub fn set_params(&mut self, params: &CompressParams) { 351 | unsafe { 352 | brotli_sys::BrotliEncoderSetParameter( 353 | self.state, 354 | brotli_sys::BROTLI_PARAM_MODE, 355 | params.mode, 356 | ); 357 | brotli_sys::BrotliEncoderSetParameter( 358 | self.state, 359 | brotli_sys::BROTLI_PARAM_QUALITY, 360 | params.quality, 361 | ); 362 | brotli_sys::BrotliEncoderSetParameter( 363 | self.state, 364 | brotli_sys::BROTLI_PARAM_LGWIN, 365 | params.lgwin, 366 | ); 367 | brotli_sys::BrotliEncoderSetParameter( 368 | self.state, 369 | brotli_sys::BROTLI_PARAM_LGBLOCK, 370 | params.lgblock, 371 | ); 372 | // TODO: add these two 373 | // brotli_sys::BrotliEncoderSetParameter(self.state, 374 | // brotli_sys::BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, 375 | // params.lgblock); 376 | // brotli_sys::BrotliEncoderSetParameter(self.state, 377 | // brotli_sys::BROTLI_PARAM_SIZE_HINT, 378 | // params.lgblock); 379 | } 380 | } 381 | } 382 | 383 | impl Drop for Compress { 384 | fn drop(&mut self) { 385 | unsafe { 386 | brotli_sys::BrotliEncoderDestroyInstance(self.state); 387 | } 388 | } 389 | } 390 | 391 | /// Compresses the data in `input` into `output`. 392 | /// 393 | /// The `output` buffer is updated to point to the exact slice which contains 394 | /// the output data. 395 | /// 396 | /// If successful, the amount of compressed bytes are returned (the size of the 397 | /// `output` slice), or an error is returned. The output buffer being too small 398 | /// is considered to be an error. 399 | pub fn compress_buf( 400 | params: &CompressParams, 401 | input: &[u8], 402 | output: &mut &mut [u8], 403 | ) -> Result { 404 | let mut size = output.len(); 405 | let r = unsafe { 406 | brotli_sys::BrotliEncoderCompress( 407 | params.quality as c_int, 408 | params.lgwin as c_int, 409 | params.mode as brotli_sys::BrotliEncoderMode, 410 | input.len(), 411 | input.as_ptr(), 412 | &mut size, 413 | output.as_mut_ptr(), 414 | ) 415 | }; 416 | *output = &mut mem::replace(output, &mut [])[..size]; 417 | if r == 0 { 418 | Err(Error(())) 419 | } else { 420 | Ok(size) 421 | } 422 | } 423 | 424 | impl fmt::Display for Error { 425 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 426 | error::Error::description(self).fmt(f) 427 | } 428 | } 429 | 430 | impl error::Error for Error { 431 | fn description(&self) -> &str { 432 | "brotli error" 433 | } 434 | } 435 | 436 | impl From for io::Error { 437 | fn from(_err: Error) -> io::Error { 438 | io::Error::new(io::ErrorKind::Other, "brotli error") 439 | } 440 | } 441 | 442 | #[cfg(test)] 443 | mod tests { 444 | use super::*; 445 | 446 | #[test] 447 | fn decompress_error() { 448 | let mut d = Decompress::new(); 449 | d.decompress(&mut &[0; 1024][..], &mut &mut [0; 2048][..]) 450 | .unwrap_err(); 451 | } 452 | 453 | #[test] 454 | fn compress_buf_smoke() { 455 | let mut data = [0; 128]; 456 | let mut data = &mut data[..]; 457 | compress_buf(&CompressParams::new(), b"hello!", &mut data).unwrap(); 458 | 459 | let mut dst = [0; 128]; 460 | { 461 | let mut dst = &mut dst[..]; 462 | let n = decompress_buf(data, &mut dst).unwrap(); 463 | assert_eq!(n, dst.len()); 464 | assert_eq!(dst.len(), 6); 465 | } 466 | assert_eq!(&dst[..6], b"hello!"); 467 | } 468 | 469 | #[test] 470 | fn decompress_smoke() { 471 | let mut data = [0; 128]; 472 | let mut data = &mut data[..]; 473 | compress_buf(&CompressParams::new(), b"hello!", &mut data).unwrap(); 474 | 475 | let mut d = Decompress::new(); 476 | let mut dst = [0; 128]; 477 | { 478 | let mut data = &data[..]; 479 | let mut dst = &mut dst[..]; 480 | assert_eq!(d.decompress(&mut data, &mut dst), Ok(DeStatus::Finished)); 481 | } 482 | assert_eq!(&dst[..6], b"hello!"); 483 | } 484 | 485 | #[test] 486 | fn compress_smoke() { 487 | let mut data = [0; 128]; 488 | let mut dst = [0; 128]; 489 | 490 | { 491 | let mut data = &mut data[..]; 492 | let mut c = Compress::new(); 493 | let input = &mut &b"hello!"[..]; 494 | assert_eq!( 495 | c.compress(CompressOp::Finish, input, &mut data), 496 | Ok(CoStatus::Finished) 497 | ); 498 | assert!(input.is_empty()); 499 | } 500 | decompress_buf(&data, &mut &mut dst[..]).unwrap(); 501 | assert_eq!(&dst[..6], b"hello!"); 502 | 503 | { 504 | let mut data = &mut data[..]; 505 | let mut c = Compress::new(); 506 | let input = &mut &b"hel"[..]; 507 | assert_eq!( 508 | c.compress(CompressOp::Flush, input, &mut data), 509 | Ok(CoStatus::Finished) 510 | ); 511 | assert!(input.is_empty()); 512 | let input = &mut &b"lo!"[..]; 513 | assert_eq!( 514 | c.compress(CompressOp::Finish, input, &mut data), 515 | Ok(CoStatus::Finished) 516 | ); 517 | assert!(input.is_empty()); 518 | } 519 | decompress_buf(&data, &mut &mut dst[..]).unwrap(); 520 | assert_eq!(&dst[..6], b"hello!"); 521 | } 522 | } 523 | -------------------------------------------------------------------------------- /src/read.rs: -------------------------------------------------------------------------------- 1 | //! Reader-based compression/decompression streams 2 | 3 | use std::io::prelude::*; 4 | use std::io::{self, BufReader}; 5 | 6 | use bufread; 7 | 8 | use super::CompressParams; 9 | 10 | /// A compression stream which wraps an uncompressed stream of data. Compressed 11 | /// data will be read from the stream. 12 | pub struct BrotliEncoder { 13 | inner: bufread::BrotliEncoder>, 14 | } 15 | 16 | /// A decompression stream which wraps a compressed stream of data. Decompressed 17 | /// data will be read from the stream. 18 | pub struct BrotliDecoder { 19 | inner: bufread::BrotliDecoder>, 20 | } 21 | 22 | impl BrotliEncoder { 23 | /// Create a new compression stream which will compress at the given level 24 | /// to read compress output to the give output stream. 25 | /// 26 | /// The `level` argument here is typically 0-9 with 6 being a good default. 27 | pub fn new(r: R, level: u32) -> BrotliEncoder { 28 | BrotliEncoder { 29 | inner: bufread::BrotliEncoder::new(BufReader::new(r), level), 30 | } 31 | } 32 | 33 | /// Configure the compression parameters of this encoder. 34 | pub fn from_params(r: R, params: &CompressParams) -> BrotliEncoder { 35 | BrotliEncoder { 36 | inner: bufread::BrotliEncoder::from_params( 37 | BufReader::with_capacity(params.get_lgwin_readable(), r), 38 | params, 39 | ), 40 | } 41 | } 42 | 43 | /// Acquires a reference to the underlying stream 44 | pub fn get_ref(&self) -> &R { 45 | self.inner.get_ref().get_ref() 46 | } 47 | 48 | /// Acquires a mutable reference to the underlying stream 49 | /// 50 | /// Note that mutation of the stream may result in surprising results if 51 | /// this encoder is continued to be used. 52 | pub fn get_mut(&mut self) -> &mut R { 53 | self.inner.get_mut().get_mut() 54 | } 55 | 56 | /// Unwrap the underlying writer, finishing the compression stream. 57 | pub fn into_inner(self) -> R { 58 | self.inner.into_inner().into_inner() 59 | } 60 | } 61 | 62 | impl Read for BrotliEncoder { 63 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 64 | self.inner.read(buf) 65 | } 66 | } 67 | 68 | impl BrotliDecoder { 69 | /// Create a new decompression stream, which will read compressed 70 | /// data from the given input stream and decompress it. 71 | pub fn new(r: R) -> BrotliDecoder { 72 | BrotliDecoder { 73 | inner: bufread::BrotliDecoder::new(BufReader::new(r)), 74 | } 75 | } 76 | 77 | /// Acquires a reference to the underlying stream 78 | pub fn get_ref(&self) -> &R { 79 | self.inner.get_ref().get_ref() 80 | } 81 | 82 | /// Acquires a mutable reference to the underlying stream 83 | /// 84 | /// Note that mutation of the stream may result in surprising results if 85 | /// this encoder is continued to be used. 86 | pub fn get_mut(&mut self) -> &mut R { 87 | self.inner.get_mut().get_mut() 88 | } 89 | 90 | /// Unwrap the underlying writer, finishing the compression stream. 91 | pub fn into_inner(self) -> R { 92 | self.inner.into_inner().into_inner() 93 | } 94 | } 95 | 96 | impl Read for BrotliDecoder { 97 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 98 | self.inner.read(buf) 99 | } 100 | } 101 | 102 | #[cfg(test)] 103 | mod tests { 104 | use read::{BrotliDecoder, BrotliEncoder}; 105 | use std::io::prelude::*; 106 | 107 | use rand::distributions::Standard; 108 | use rand::{thread_rng, Rng}; 109 | 110 | #[test] 111 | fn smoke() { 112 | let m: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8]; 113 | let mut c = BrotliEncoder::new(m, 6); 114 | let mut data = vec![]; 115 | c.read_to_end(&mut data).unwrap(); 116 | let mut d = BrotliDecoder::new(&data[..]); 117 | let mut data2 = Vec::new(); 118 | d.read_to_end(&mut data2).unwrap(); 119 | assert_eq!(data2, m); 120 | } 121 | 122 | #[test] 123 | fn smoke2() { 124 | let m: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8]; 125 | let c = BrotliEncoder::new(m, 6); 126 | let mut d = BrotliDecoder::new(c); 127 | let mut data = vec![]; 128 | d.read_to_end(&mut data).unwrap(); 129 | assert_eq!(data, [1, 2, 3, 4, 5, 6, 7, 8]); 130 | } 131 | 132 | #[test] 133 | fn smoke3() { 134 | let m = vec![3u8; 128 * 1024 + 1]; 135 | let c = BrotliEncoder::new(&m[..], 6); 136 | let mut d = BrotliDecoder::new(c); 137 | let mut data = vec![]; 138 | d.read_to_end(&mut data).unwrap(); 139 | assert!(data == &m[..]); 140 | } 141 | 142 | #[test] 143 | fn self_terminating() { 144 | let m = vec![3u8; 128 * 1024 + 1]; 145 | let mut c = BrotliEncoder::new(&m[..], 6); 146 | 147 | let mut result = Vec::new(); 148 | c.read_to_end(&mut result).unwrap(); 149 | 150 | let v = thread_rng() 151 | .sample_iter(&Standard) 152 | .take(1024) 153 | .collect::>(); 154 | for _ in 0..200 { 155 | result.extend(v.iter().map(|x: &u8| *x)); 156 | } 157 | 158 | let mut d = BrotliDecoder::new(&result[..]); 159 | let mut data = Vec::with_capacity(m.len()); 160 | unsafe { 161 | data.set_len(m.len()); 162 | } 163 | assert!(d.read(&mut data).unwrap() == m.len()); 164 | assert!(data == &m[..]); 165 | } 166 | 167 | #[test] 168 | fn zero_length_read_at_eof() { 169 | let m = Vec::new(); 170 | let mut c = BrotliEncoder::new(&m[..], 6); 171 | 172 | let mut result = Vec::new(); 173 | c.read_to_end(&mut result).unwrap(); 174 | 175 | let mut d = BrotliDecoder::new(&result[..]); 176 | let mut data = Vec::new(); 177 | assert!(d.read(&mut data).unwrap() == 0); 178 | } 179 | 180 | #[test] 181 | fn zero_length_read_with_data() { 182 | let m = vec![3u8; 128 * 1024 + 1]; 183 | let mut c = BrotliEncoder::new(&m[..], 6); 184 | 185 | let mut result = Vec::new(); 186 | c.read_to_end(&mut result).unwrap(); 187 | 188 | let mut d = BrotliDecoder::new(&result[..]); 189 | let mut data = Vec::new(); 190 | assert!(d.read(&mut data).unwrap() == 0); 191 | } 192 | 193 | #[test] 194 | fn qc() { 195 | ::quickcheck::quickcheck(test as fn(_) -> _); 196 | 197 | fn test(v: Vec) -> bool { 198 | let r = BrotliEncoder::new(&v[..], 6); 199 | let mut r = BrotliDecoder::new(r); 200 | let mut v2 = Vec::new(); 201 | r.read_to_end(&mut v2).unwrap(); 202 | v == v2 203 | } 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/write.rs: -------------------------------------------------------------------------------- 1 | //! Writer-based compression/decompression streams 2 | 3 | use std::io; 4 | use std::io::prelude::*; 5 | 6 | use raw::{self, CoStatus, Compress, CompressOp, DeStatus, Decompress}; 7 | 8 | use super::CompressParams; 9 | 10 | const BUF_SIZE: usize = 32 * 1024; 11 | 12 | /// A compression stream which will have uncompressed data written to it and 13 | /// will write compressed data to an output stream. 14 | pub struct BrotliEncoder { 15 | data: Compress, 16 | obj: Option, 17 | buf: Vec, 18 | cur: usize, 19 | err: Option, 20 | } 21 | 22 | /// A compression stream which will have compressed data written to it and 23 | /// will write uncompressed data to an output stream. 24 | pub struct BrotliDecoder { 25 | data: Decompress, 26 | obj: Option, 27 | buf: Vec, 28 | cur: usize, 29 | err: Option, 30 | } 31 | 32 | impl BrotliEncoder { 33 | /// Create a new compression stream which will compress at the given level 34 | /// to write compress output to the give output stream. 35 | pub fn new(obj: W, level: u32) -> BrotliEncoder { 36 | let mut data = Compress::new(); 37 | data.set_params(CompressParams::new().quality(level)); 38 | BrotliEncoder { 39 | data: data, 40 | obj: Some(obj), 41 | buf: Vec::with_capacity(BUF_SIZE), 42 | cur: 0, 43 | err: None, 44 | } 45 | } 46 | 47 | /// Creates a new encoder with a custom `CompressParams`. 48 | pub fn from_params(obj: W, params: &CompressParams) -> BrotliEncoder { 49 | let mut data = Compress::new(); 50 | data.set_params(params); 51 | BrotliEncoder { 52 | data: data, 53 | obj: Some(obj), 54 | buf: Vec::with_capacity(BUF_SIZE), 55 | cur: 0, 56 | err: None, 57 | } 58 | } 59 | 60 | /// Acquires a reference to the underlying writer. 61 | pub fn get_ref(&self) -> &W { 62 | self.obj.as_ref().unwrap() 63 | } 64 | 65 | /// Acquires a mutable reference to the underlying writer. 66 | /// 67 | /// Note that mutating the output/input state of the stream may corrupt this 68 | /// object, so care must be taken when using this method. 69 | pub fn get_mut(&mut self) -> &mut W { 70 | self.obj.as_mut().unwrap() 71 | } 72 | 73 | fn dump(&mut self) -> io::Result<()> { 74 | loop { 75 | while !self.buf.is_empty() { 76 | let amt = self.obj.as_mut().unwrap().write(&self.buf[self.cur..])?; 77 | self.cur += amt; 78 | if self.cur == self.buf.len() { 79 | self.buf.clear(); 80 | self.cur = 0 81 | } 82 | } 83 | // TODO: if we could peek, the buffer wouldn't be necessary 84 | if let Some(data) = self.data.take_output(Some(BUF_SIZE)) { 85 | match self.obj.as_mut().unwrap().write(data) { 86 | Ok(n) => self.buf.extend_from_slice(&data[n..]), 87 | Err(e) => { 88 | self.buf.extend_from_slice(data); 89 | return Err(e); 90 | } 91 | } 92 | } else { 93 | break; 94 | } 95 | } 96 | Ok(()) 97 | } 98 | 99 | // Flush or finish stream, also flushing underlying stream 100 | fn do_flush_or_finish(&mut self, finish: bool) -> io::Result<()> { 101 | self.dump()?; 102 | let op = if finish { 103 | CompressOp::Finish 104 | } else { 105 | CompressOp::Flush 106 | }; 107 | loop { 108 | let status = match self.data.compress(op, &mut &[][..], &mut &mut [][..]) { 109 | Ok(s) => s, 110 | Err(err) => { 111 | self.err = Some(err.clone()); 112 | return Err(err.into()); 113 | } 114 | }; 115 | let obj = self.obj.as_mut().unwrap(); 116 | while let Some(data) = self.data.take_output(None) { 117 | obj.write_all(data)?; 118 | } 119 | match status { 120 | CoStatus::Finished => { 121 | obj.flush()?; 122 | return Ok(()); 123 | } 124 | CoStatus::Unfinished => (), 125 | } 126 | } 127 | } 128 | 129 | /// Consumes this encoder, flushing the output stream. 130 | /// 131 | /// This will flush the underlying data stream and then return the contained 132 | /// writer if the flush succeeded. 133 | pub fn finish(mut self) -> io::Result { 134 | self.do_flush_or_finish(true)?; 135 | Ok(self.obj.take().unwrap()) 136 | } 137 | } 138 | 139 | impl Write for BrotliEncoder { 140 | fn write(&mut self, mut data: &[u8]) -> io::Result { 141 | if data.is_empty() { 142 | return Ok(0); 143 | } 144 | // If the decompressor has failed at some point, this is set. 145 | // Unfortunately we have no idea what status is in the compressor 146 | // was in when it failed so we can't do anything except bail again. 147 | if let Some(ref err) = self.err { 148 | return Err(err.clone().into()); 149 | } 150 | self.dump()?; 151 | // Zero-length output buf to keep it all inside the compressor buffer 152 | let avail_in = data.len(); 153 | if let Err(err) = self 154 | .data 155 | .compress(CompressOp::Process, &mut data, &mut &mut [][..]) 156 | { 157 | self.err = Some(err.clone()); 158 | return Err(err.into()); 159 | } 160 | assert!(avail_in != data.len()); 161 | Ok(avail_in - data.len()) 162 | } 163 | 164 | fn flush(&mut self) -> io::Result<()> { 165 | self.do_flush_or_finish(false) 166 | } 167 | } 168 | 169 | impl Drop for BrotliEncoder { 170 | fn drop(&mut self) { 171 | if self.obj.is_some() { 172 | let _ = self.do_flush_or_finish(true); 173 | } 174 | } 175 | } 176 | 177 | impl BrotliDecoder { 178 | /// Creates a new decoding stream which will decode all input written to it 179 | /// into `obj`. 180 | pub fn new(obj: W) -> BrotliDecoder { 181 | BrotliDecoder { 182 | data: Decompress::new(), 183 | obj: Some(obj), 184 | buf: Vec::with_capacity(BUF_SIZE), 185 | cur: 0, 186 | err: None, 187 | } 188 | } 189 | 190 | /// Acquires a reference to the underlying writer. 191 | pub fn get_ref(&self) -> &W { 192 | self.obj.as_ref().unwrap() 193 | } 194 | 195 | /// Acquires a mutable reference to the underlying writer. 196 | /// 197 | /// Note that mutating the output/input state of the stream may corrupt this 198 | /// object, so care must be taken when using this method. 199 | pub fn get_mut(&mut self) -> &mut W { 200 | self.obj.as_mut().unwrap() 201 | } 202 | 203 | fn dump(&mut self) -> io::Result<()> { 204 | loop { 205 | while !self.buf.is_empty() { 206 | let amt = self.obj.as_mut().unwrap().write(&self.buf[self.cur..])?; 207 | self.cur += amt; 208 | if self.cur == self.buf.len() { 209 | self.buf.clear(); 210 | self.cur = 0 211 | } 212 | } 213 | // TODO: if we could peek, the buffer wouldn't be necessary 214 | if let Some(data) = self.data.take_output(Some(BUF_SIZE)) { 215 | self.buf.extend_from_slice(data) 216 | } else { 217 | break; 218 | } 219 | } 220 | Ok(()) 221 | } 222 | 223 | fn do_finish(&mut self) -> io::Result<()> { 224 | self.dump()?; 225 | loop { 226 | let status = match self.data.decompress(&mut &[][..], &mut &mut [][..]) { 227 | Ok(s) => s, 228 | Err(err) => { 229 | self.err = Some(err.clone()); 230 | return Err(err.into()); 231 | } 232 | }; 233 | let obj = self.obj.as_mut().unwrap(); 234 | while let Some(data) = self.data.take_output(None) { 235 | obj.write_all(data)?; 236 | } 237 | match status { 238 | DeStatus::Finished => { 239 | obj.flush()?; 240 | return Ok(()); 241 | } 242 | // When decoding a truncated file, brotli returns DeStatus::NeedInput. 243 | // Since we're finishing, we cannot provide more data so this is an 244 | // error. 245 | DeStatus::NeedInput => { 246 | let msg = "brotli compressed stream is truncated or otherwise corrupt"; 247 | return Err(io::Error::new(io::ErrorKind::UnexpectedEof, msg)); 248 | } 249 | DeStatus::NeedOutput => (), 250 | } 251 | } 252 | } 253 | 254 | /// Unwrap the underlying writer, finishing the compression stream. 255 | pub fn finish(&mut self) -> io::Result { 256 | self.do_finish()?; 257 | Ok(self.obj.take().unwrap()) 258 | } 259 | } 260 | 261 | impl Write for BrotliDecoder { 262 | fn write(&mut self, mut data: &[u8]) -> io::Result { 263 | if data.is_empty() { 264 | return Ok(0); 265 | } 266 | // If the decompressor has failed at some point, this is set. 267 | // Unfortunately we have no idea what status is in the compressor 268 | // was in when it failed so we can't do anything except bail again. 269 | if let Some(ref err) = self.err { 270 | return Err(err.clone().into()); 271 | } 272 | self.dump()?; 273 | // Zero-length output buf to keep it all inside the decompressor buffer 274 | let avail_in = data.len(); 275 | let status = match self.data.decompress(&mut data, &mut &mut [][..]) { 276 | Ok(s) => s, 277 | Err(err) => { 278 | self.err = Some(err.clone()); 279 | return Err(err.into()); 280 | } 281 | }; 282 | assert!(avail_in != data.len() || status == DeStatus::Finished); 283 | Ok(avail_in - data.len()) 284 | } 285 | 286 | fn flush(&mut self) -> io::Result<()> { 287 | self.dump()?; 288 | self.obj.as_mut().unwrap().flush() 289 | } 290 | } 291 | 292 | impl Drop for BrotliDecoder { 293 | fn drop(&mut self) { 294 | if self.obj.is_some() { 295 | let _ = self.do_finish(); 296 | } 297 | } 298 | } 299 | 300 | #[cfg(test)] 301 | mod tests { 302 | use super::{BrotliDecoder, BrotliEncoder}; 303 | use std::io::prelude::*; 304 | use std::iter::repeat; 305 | 306 | #[test] 307 | fn smoke() { 308 | let d = BrotliDecoder::new(Vec::new()); 309 | let mut c = BrotliEncoder::new(d, 6); 310 | c.write_all(b"12834").unwrap(); 311 | let s = repeat("12345").take(100000).collect::(); 312 | c.write_all(s.as_bytes()).unwrap(); 313 | let data = c.finish().unwrap().finish().unwrap(); 314 | assert_eq!(&data[0..5], b"12834"); 315 | assert_eq!(data.len(), 500005); 316 | assert!(format!("12834{}", s).as_bytes() == &*data); 317 | } 318 | 319 | #[test] 320 | fn write_empty() { 321 | let d = BrotliDecoder::new(Vec::new()); 322 | let mut c = BrotliEncoder::new(d, 6); 323 | c.write(b"").unwrap(); 324 | let data = c.finish().unwrap().finish().unwrap(); 325 | assert_eq!(&data[..], b""); 326 | } 327 | 328 | #[test] 329 | fn qc() { 330 | ::quickcheck::quickcheck(test as fn(_) -> _); 331 | 332 | fn test(v: Vec) -> bool { 333 | let w = BrotliDecoder::new(Vec::new()); 334 | let mut w = BrotliEncoder::new(w, 6); 335 | w.write_all(&v).unwrap(); 336 | v == w.finish().unwrap().finish().unwrap() 337 | } 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /systest/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "systest" 3 | version = "0.1.0" 4 | authors = ["Alex Crichton "] 5 | build = "build.rs" 6 | 7 | [dependencies] 8 | brotli-sys = { path = "../brotli-sys" } 9 | libc = "0.2" 10 | 11 | [build-dependencies] 12 | ctest = "0.1" 13 | -------------------------------------------------------------------------------- /systest/build.rs: -------------------------------------------------------------------------------- 1 | extern crate ctest; 2 | 3 | use std::env; 4 | 5 | fn main() { 6 | let include = env::var("DEP_BROTLI_INCLUDE").unwrap(); 7 | let mut cfg = ctest::TestGenerator::new(); 8 | 9 | if env::var("TARGET").unwrap().contains("msvc") { 10 | cfg.flag("/wd2220"); // allow "no object file was generated" 11 | cfg.flag("/wd4127"); // allow "conditional expression is constant" 12 | cfg.flag("/wd4464"); // allow "relative include path contains '..'" 13 | } 14 | cfg.header("brotli/decode.h") 15 | .header("brotli/encode.h"); 16 | cfg.include(&include); 17 | cfg.type_name(|s, _| s.to_string()); 18 | cfg.skip_type(|n| n == "__enum_ty" || n == "__enum_ty_s"); 19 | cfg.skip_signededness(|s| s.ends_with("_func")); 20 | cfg.generate("../brotli-sys/src/lib.rs", "all.rs"); 21 | } 22 | -------------------------------------------------------------------------------- /systest/src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(bad_style)] 2 | 3 | extern crate brotli_sys; 4 | extern crate libc; 5 | 6 | use brotli_sys::*; 7 | 8 | include!(concat!(env!("OUT_DIR"), "/all.rs")); 9 | -------------------------------------------------------------------------------- /tests/drop-incomplete.rs: -------------------------------------------------------------------------------- 1 | extern crate brotli2; 2 | 3 | use brotli2::write::BrotliDecoder; 4 | use std::io::prelude::*; 5 | 6 | // This is a BR file generated by head -c10 /dev/urandom | bro --output file.br 7 | const DATA: &'static [u8] = &[ 8 | 139, 4, 128, 227, 139, 226, 91, 233, 134, 14, 218, 140, 196, 3, 9 | ]; 10 | 11 | /// In this test, we drop a write::BrotliDecoder after supplying it a truncated input stream. 12 | /// 13 | /// The decoder should detect that it is impossible to decode more data and not 14 | /// go into an infinite loop waiting for more data. 15 | #[test] 16 | fn drop_writer_incomplete_input_no_loop() { 17 | let mut decoder = BrotliDecoder::new(Vec::new()); 18 | const PREFIX_LEN: usize = 10; 19 | decoder.write_all(&DATA[..PREFIX_LEN]).unwrap(); 20 | } 21 | 22 | /// Same as above, but verifying that we get an error if we manually call `finish`; 23 | #[test] 24 | fn finish_writer_incomplete_input_error() { 25 | let mut decoder = BrotliDecoder::new(Vec::new()); 26 | const PREFIX_LEN: usize = 10; 27 | decoder.write_all(&DATA[..PREFIX_LEN]).unwrap(); 28 | decoder 29 | .finish() 30 | .err() 31 | .expect("finish should error because of incomplete input"); 32 | } 33 | --------------------------------------------------------------------------------