├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── dependabot.yml ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE.BSD-3-Clause ├── LICENSE.MIT ├── README.md ├── appveyor.yml ├── c ├── .gitignore ├── Cargo.toml ├── Makefile ├── README.md ├── arg.h ├── brotli.c ├── brotli │ ├── broccoli.h │ ├── decode.h │ ├── encode.h │ ├── multiencode.h │ ├── port.h │ └── types.h ├── catbrotli.c ├── custom_alloc.h ├── decompressor.c ├── fuzz.c ├── go │ ├── brotli │ │ ├── brotli.go │ │ ├── brotli │ │ │ ├── broccoli.h │ │ │ ├── decode.h │ │ │ ├── encode.h │ │ │ ├── multiencode.h │ │ │ ├── port.h │ │ │ └── types.h │ │ ├── data_test.go │ │ ├── go.mod │ │ ├── header.go │ │ └── interface_test.go │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── stress │ │ ├── main.go │ │ └── test_file.go │ └── verify.go ├── multiexample.c ├── py │ ├── __init__.py │ ├── brotli.py │ ├── brotli_test.py │ └── testdata.py ├── src │ └── lib.rs └── vec_u8.h ├── ci ├── before_deploy.ps1 ├── before_deploy.sh ├── install.sh └── script.sh ├── examples ├── compress.rs └── decompress.rs ├── justfile ├── research ├── concatenate_some.py ├── frombase2.py └── tobase2.py ├── src ├── bin │ ├── brotli.rs │ ├── catbrotli.rs │ ├── integration_tests.rs │ ├── test_broccoli.rs │ ├── test_custom_dict.rs │ ├── test_threading.rs │ ├── tests.rs │ ├── util.rs │ └── validate.rs ├── concat │ └── mod.rs ├── enc │ ├── backward_references │ │ ├── benchmark.rs │ │ ├── hash_to_binary_tree.rs │ │ ├── hq.rs │ │ ├── mod.rs │ │ └── test.rs │ ├── bit_cost.rs │ ├── block_split.rs │ ├── block_splitter.rs │ ├── brotli_bit_stream.rs │ ├── cluster.rs │ ├── combined_alloc.rs │ ├── command.rs │ ├── compat.rs │ ├── compress_fragment.rs │ ├── compress_fragment_two_pass.rs │ ├── constants.rs │ ├── context_map_entropy.rs │ ├── dictionary_hash.rs │ ├── encode.rs │ ├── entropy_encode.rs │ ├── find_stride.rs │ ├── fixed_queue.rs │ ├── histogram.rs │ ├── input_pair.rs │ ├── interface.rs │ ├── ir_interpret.rs │ ├── literal_cost.rs │ ├── log_table_16.rs │ ├── log_table_8.rs │ ├── metablock.rs │ ├── mod.rs │ ├── multithreading.rs │ ├── parameters.rs │ ├── pdf.rs │ ├── prior_eval.rs │ ├── reader.rs │ ├── singlethreading.rs │ ├── static_dict.rs │ ├── static_dict_lut.rs │ ├── stride_eval.rs │ ├── test.rs │ ├── threading.rs │ ├── utf8_util.rs │ ├── util.rs │ ├── vectorization.rs │ ├── weights.rs │ ├── worker_pool.rs │ └── writer.rs ├── ffi │ ├── alloc_util.rs │ ├── broccoli.rs │ ├── compressor.rs │ ├── decompressor.rs │ ├── mod.rs │ └── multicompress │ │ ├── mod.rs │ │ └── test.rs └── lib.rs └── testdata ├── .gitattributes ├── 10x10y ├── 64x ├── 64x.compressed ├── aaabaaaa ├── alice29.txt ├── alice29.txt.compressed ├── asyoulik.txt ├── asyoulik.txt.compressed ├── backward65536 ├── backward65536.compressed ├── compressed_file ├── compressed_file.compressed ├── compressed_repeated ├── compressed_repeated.compressed ├── empty ├── empty.compressed ├── empty.compressed.00 ├── empty.compressed.01 ├── empty.compressed.02 ├── empty.compressed.03 ├── empty.compressed.04 ├── empty.compressed.05 ├── empty.compressed.06 ├── empty.compressed.07 ├── empty.compressed.08 ├── empty.compressed.09 ├── empty.compressed.10 ├── empty.compressed.11 ├── empty.compressed.12 ├── empty.compressed.13 ├── empty.compressed.14 ├── empty.compressed.15 ├── empty.compressed.16 ├── empty.compressed.17 ├── empty.compressed.18 ├── ends_with_truncated_dictionary ├── monkey ├── quickfox ├── quickfox_repeated ├── random_org_10k.bin ├── random_then_unicode ├── ukkonooa ├── x ├── x.compressed.00 ├── x.compressed.01 ├── x.compressed.02 ├── x.compressed.03 └── xyzzy /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | - package-ecosystem: cargo 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | time: "02:00" 13 | open-pull-requests-limit: 10 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: { } # Since the primary repo has no CI, every fork must do the CI work in every branch 5 | # push: 6 | # branches: [ main, master ] 7 | pull_request: 8 | branches: [ main, master ] 9 | release: 10 | types: [ published ] 11 | workflow_dispatch: 12 | 13 | env: 14 | CARGO_TERM_COLOR: always 15 | 16 | defaults: 17 | run: 18 | shell: bash 19 | 20 | jobs: 21 | test: 22 | name: Test 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: taiki-e/install-action@v2 26 | with: { tool: "just" } 27 | - uses: actions/checkout@v4 28 | - uses: Swatinem/rust-cache@v2 29 | if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 30 | - name: Install nightly toolchain 31 | run: rustup toolchain install nightly 32 | - name: Run `just ci-test` to lint, build, and test everything 33 | run: just ci-test 34 | - name: Check if changes break public API and need a new version. Use `just semver-checks` to run locally. 35 | uses: obi1kenobi/cargo-semver-checks-action@v2 36 | msrv: 37 | name: Test MSRV 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: taiki-e/install-action@v2 41 | with: { tool: just } 42 | - uses: actions/checkout@v4 43 | - uses: Swatinem/rust-cache@v2 44 | if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 45 | - name: Read rust-version from Cargo.toml 46 | id: metadata 47 | run: | 48 | MSRV="$(just read-msrv)" 49 | echo "rust-version=$MSRV" >> $GITHUB_OUTPUT 50 | echo "Detected MSRV $MSRV" 51 | - name: Install Rust 52 | uses: dtolnay/rust-toolchain@stable 53 | with: 54 | toolchain: ${{ steps.metadata.outputs.rust-version }} 55 | - name: Run `just ci-test-msrv` to build and test everything with the MSRV 56 | run: just ci-test-msrv 57 | -------------------------------------------------------------------------------- /.github/workflows/dependabot.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-merge 2 | on: pull_request 3 | 4 | permissions: write-all 5 | 6 | jobs: 7 | dependabot: 8 | runs-on: ubuntu-latest 9 | if: github.actor == 'dependabot[bot]' 10 | steps: 11 | - name: Dependabot metadata 12 | id: metadata 13 | uses: dependabot/fetch-metadata@v2.1.0 14 | with: 15 | github-token: "${{ secrets.GITHUB_TOKEN }}" 16 | - name: Approve Dependabot PRs 17 | if: steps.metadata.outputs.update-type == 'version-update:semver-patch' 18 | run: gh pr review --approve "$PR_URL" 19 | env: 20 | PR_URL: ${{github.event.pull_request.html_url}} 21 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 22 | - name: Enable auto-merge for Dependabot PRs 23 | if: steps.metadata.outputs.update-type == 'version-update:semver-patch' 24 | run: gh pr merge --auto --squash "$PR_URL" 25 | env: 26 | PR_URL: ${{github.event.pull_request.html_url}} 27 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | .vscode/ 4 | Cargo.lock 5 | target/ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | rust: 3 | - nightly 4 | - stable 5 | - 1.12.0 6 | 7 | os: 8 | - linux 9 | - osx 10 | 11 | script: 12 | - rustc --version | grep nightly && cargo test --features=simd || ( echo skip && rustc --version | grep -v nightly ) 13 | - cargo test --no-default-features 14 | - cargo test --no-default-features --features=std 15 | - cargo test --no-default-features --features=std --release 16 | - rustc --version | grep 1[.][89][.] || rustc --version | grep -v 1[.]2[789][.] | grep 1[.][12][0-9][.]||cargo build --features=validation 17 | 18 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "brotli" 3 | version = "8.0.1" 4 | authors = ["Daniel Reiter Horn ", "The Brotli Authors"] 5 | description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe." 6 | license = "BSD-3-Clause AND MIT" 7 | documentation = "https://docs.rs/brotli/" 8 | homepage = "https://github.com/dropbox/rust-brotli" 9 | repository = "https://github.com/dropbox/rust-brotli" 10 | keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"] 11 | categories = ["compression", "no-std"] 12 | readme = "README.md" 13 | autobins = false 14 | edition = "2015" 15 | rust-version = "1.59.0" 16 | include = [ 17 | "/src/**/*.rs", 18 | "/examples/**/*.rs", 19 | "/Cargo.toml", 20 | "/README.md", 21 | "/LICENSE.MIT", 22 | ] 23 | 24 | [[bin]] 25 | doc = false 26 | name = "brotli" 27 | 28 | [[bin]] 29 | doc = false 30 | name = "catbrotli" 31 | 32 | [profile.release] 33 | lto = true 34 | incremental = false 35 | 36 | [dependencies] 37 | "alloc-no-stdlib" = { version = "2.0" } 38 | "alloc-stdlib" = { version = "~0.2", optional = true } 39 | "brotli-decompressor" = { version = "~5.0", default-features = false } 40 | 41 | "sha2" = { version = "~0.10", optional = true } 42 | 43 | [features] 44 | default = ["std"] 45 | benchmark = ["brotli-decompressor/benchmark"] 46 | billing = [] 47 | disable-timer = ["brotli-decompressor/disable-timer"] 48 | disallow_large_window_size = [] 49 | external-literal-probability = [] 50 | ffi-api = ["brotli-decompressor/ffi-api"] 51 | float64 = [] 52 | floating_point_context_mixing = [] 53 | no-stdlib-ffi-binding = [] 54 | pass-through-ffi-panics = [] 55 | seccomp = ["brotli-decompressor/seccomp"] 56 | simd = [] 57 | std = ["alloc-stdlib", "brotli-decompressor/std"] 58 | validation = ["sha2"] 59 | vector_scratch_space = [] 60 | -------------------------------------------------------------------------------- /LICENSE.BSD-3-Clause: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Dropbox, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /LICENSE.MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | RUST_VERSION: stable 3 | CRATE_NAME: brotli 4 | matrix: 5 | - TARGET: x86_64-pc-windows-gnu 6 | BITS: 64 7 | MSYS2: 1 8 | - TARGET: x86_64-pc-windows-msvc 9 | BITS: 64 10 | - TARGET: i686-pc-windows-gnu 11 | BITS: 32 12 | MSYS2: 1 13 | - TARGET: i686-pc-windows-msvc 14 | BITS: 32 15 | install: 16 | - curl -sSf -o rustup-init.exe https://win.rustup.rs/ 17 | - rustup-init.exe -y --default-host %TARGET% --default-toolchain %RUST_VERSION% 18 | - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin 19 | - if defined MSYS2 set PATH=C:\msys64\mingw%BITS%\bin;%PATH% 20 | - rustc -V 21 | - cargo -V 22 | build: false 23 | test_script: 24 | - cargo test --verbose 25 | - set RUSTFLAGS=-C panic=abort 26 | - set CARGO_PROFILE_RELEASE=panic=abort 27 | - cargo build --verbose --release --features=validation 28 | before_deploy: 29 | - set CARGO_PROFILE_RELEASE=panic=abort 30 | - set RUSTFLAGS=-C panic=abort 31 | - cargo build --verbose --release --features=validation 32 | - ps: ci\before_deploy.ps1 33 | 34 | deploy: 35 | artifact: /.*\.zip/ 36 | auth_token: 37 | secure: jKP3OCU0ukoMKenkxB/lhG9/tOXppd8ZWxCm9dS1VLzbkUoIUKt3+18pG3S54VdL 38 | descriprion: 'windows rust-brotli build' 39 | on: 40 | RUST_VERSION: stable 41 | provider: GitHub 42 | 43 | cache: 44 | - C:\Users\appveyor\.cargo\registry 45 | - target 46 | 47 | branches: 48 | only: 49 | # Release tags 50 | - /^v\d+\.\d+\.\d+.*$/ 51 | - master 52 | -------------------------------------------------------------------------------- /c/.gitignore: -------------------------------------------------------------------------------- 1 | # These are the targets that are built by make 2 | # Ideally they should go into some output subdirectory 3 | multiexample 4 | fuzz 5 | fuzz_d 6 | multiexample_d 7 | decompressor 8 | catbrotli 9 | decompressor_d 10 | catbrotli_d 11 | brotli_tool 12 | brotli_tool_d 13 | -------------------------------------------------------------------------------- /c/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "brotli-ffi" 3 | version = "1.1.2" 4 | authors = ["Daniel Reiter Horn ", "The Brotli Authors"] 5 | description = "A brotli compressor and decompressor that with an interface exactly matching https://github.com/google/brotli. All included code is safe except the thin exported C-compatible functions." 6 | license = "BSD-3-Clause OR MIT" 7 | documentation = "https://github.com/dropbox/rust-brotli/blob/master/README.md" 8 | homepage = "https://github.com/dropbox/rust-brotli" 9 | repository = "https://github.com/dropbox/rust-brotli" 10 | keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"] 11 | categories = ["compression", "no-std", "external-ffi-bindings"] 12 | readme = "README.md" 13 | autobins = false 14 | 15 | [lib] 16 | path = "src/lib.rs" 17 | crate-type = ["cdylib", "staticlib", "rlib"] 18 | 19 | [profile.release] 20 | lto = true 21 | 22 | [dependencies] 23 | "brotli" = { version = "~8.0", default-features = false, features = ["ffi-api"] } 24 | 25 | [features] 26 | default = ["std"] 27 | benchmark = ["brotli/benchmark"] 28 | disable-timer = ["brotli/disable-timer"] 29 | seccomp = ["brotli/seccomp"] 30 | simd = ["brotli/simd"] 31 | std = ["brotli/std"] 32 | validation = ["brotli/validation"] 33 | vector_scratch_space = ["brotli/vector_scratch_space"] 34 | -------------------------------------------------------------------------------- /c/Makefile: -------------------------------------------------------------------------------- 1 | multiexample: multiexample_d decompressor multiexample.c vec_u8.h arg.h custom_alloc.h fuzz fuzz_d 2 | gcc -O3 -o multiexample -g multiexample.c -I. target/release/libbrotli_ffi.[ds][lyo]* 3 | multiexample_d: decompressor multiexample.c vec_u8.h arg.h custom_alloc.h 4 | gcc -o multiexample_d -g multiexample.c -I. target/debug/libbrotli_ffi.[ds][lyo]* 5 | decompressor: decompressor_d decompressor.c target/release/libbrotli.so catbrotli 6 | gcc -O3 -o decompressor -g decompressor.c -I. target/release/libbrotli_ffi.[ds][lyo]* 7 | decompressor_d: decompressor.c target/debug/libbrotli.so brotli_tool catbrotli_d 8 | cargo build && gcc -o decompressor_d -g decompressor.c -I. target/debug/libbrotli_ffi.[ds][lyo]* 9 | fuzz: fuzz_d fuzz.c target/release/libbrotli.so catbrotli 10 | gcc -O3 -o fuzz -g fuzz.c -I. target/release/libbrotli_ffi.[ds][lyo]* 11 | fuzz_d: fuzz.c target/debug/libbrotli.so brotli_tool catbrotli_d 12 | cargo build && gcc -o fuzz_d -g fuzz.c -I. target/debug/libbrotli_ffi.[ds][lyo]* 13 | 14 | brotli_tool: brotli_tool_d decompressor.c target/release/libbrotli.so 15 | gcc -O3 -o brotli_tool -g brotli.c -I. target/release/libbrotli_ffi.[ds][lyo]* 16 | brotli_tool_d: decompressor.c target/debug/libbrotli.so 17 | cargo build && gcc -o brotli_tool_d -g brotli.c -I. target/debug/libbrotli_ffi.[ds][lyo]* 18 | 19 | 20 | catbrotli: catbrotli_d decompressor.c catbrotli.c target/release/libbrotli.so 21 | gcc -O3 -o catbrotli -g catbrotli.c -I. target/release/libbrotli_ffi.[ds][lyo]* 22 | catbrotli_d: catbrotli.c decompressor.c target/debug/libbrotli.so 23 | cargo build && gcc -o catbrotli_d -g catbrotli.c -I. target/debug/libbrotli_ffi.[ds][lyo]* 24 | 25 | 26 | 27 | 28 | target/release/libbrotli.so: 29 | cargo build --release 30 | target/debug/libbrotli.so: 31 | cargo build 32 | clean: 33 | cargo clean && rm -f multiexample multiexample_d catbrotli catbrotli_d decompressor decompressor_d brotli_tool brotli_tool_d 34 | -------------------------------------------------------------------------------- /c/README.md: -------------------------------------------------------------------------------- 1 | # rust-brotli 2 | 3 | [![crates.io](https://img.shields.io/crates/v/brotli.svg)](https://crates.io/crates/brotli) 4 | [![Build Status](https://travis-ci.org/dropbox/rust-brotli.svg?branch=master)](https://travis-ci.org/dropbox/rust-brotli) 5 | 6 | * A fully compatible FFI for drop-in compatibiltiy with the https://github.com/google/brotli binaries 7 | * custom allocators fully supported 8 | * Multithreaded compression so multiple threads can operate in unison on a single file 9 | * Concatenatability mode to add the feature requested in https://github.com/google/brotli/issues/628 10 | * binary tool catbrotli can accomplish this if the first file was specified with -apendable and the second with -catable 11 | * validation mode where a file is double-checked to be able to be decompressed with the same settings; useful for benchmarking or fuzzing 12 | * Magic Number: where the brotli file can have a useful header with a few magic bytes, concatability info and a final output size for pre-allocating memory 13 | 14 | ## Project Requirements 15 | 16 | Direct no-stdlib port of the C brotli compressor to Rust 17 | 18 | no dependency on the Rust stdlib: this library would be ideal for decompressing within a rust kernel among other things. 19 | 20 | This is useful to see how C and Rust compare in an apples-to-apples 21 | comparison where the same algorithms and data structures and 22 | optimizations are employed. 23 | 24 | ## Using the C interface 25 | 26 | rust-brotli is a drop-in replacement for the official https://github.com/google/brotli C 27 | implementation. That means you can use it from any place that supports that library. 28 | To build rust-brotli in this manner enter the c subdirectory and run make there 29 | 30 | cd c && make 31 | 32 | this should build c/target/release/libbrotli.so and should build the vanilla 33 | command line tool in C for compressing and decompressing any brotli file. 34 | 35 | the libbrotli.so in c/target/release should be able to replace any other libbrotli.so 36 | file, but with all the advantages of using safe rust (except in the FFI bindings) 37 | 38 | The code also allows a wider range of options, including forcing the prediction mode 39 | (eg UTF8 vs signed vs MSB vs LSB) and changing the weight of the literal cost from 540 40 | to other values. 41 | 42 | Additionally the CATABLE and APPENDABLE options are exposed and allow concatenation of files 43 | created in this manner. 44 | 45 | Specifically CATABLE files can be concatenated in any order using the catbrotli tool 46 | and APPENDABLE files can be the first file in a sequence of catable files... 47 | eg you can combine 48 | appendable.br catable1.br catable2.br catable3.br 49 | 50 | or simply 51 | catable0.br catable1.br catable2.br catable3.br 52 | 53 | # Multithreaded Compression 54 | The C FFI allows you to create a workpool which may be used to compress multiple files without recreating threads on each compression 55 | ```rust 56 | BrotliEncoderWorkPool *work_pool = BrotliEncoderCreateWorkPool(num_threads != 0 ? num_threads - 1 : 0, NULL /* custom allocator */, NULL, NULL); 57 | if (!work_pool) { 58 | return 0; 59 | } 60 | size_t out_len = BrotliEncoderMaxCompressedSizeMulti(len, num_threads); 61 | reinit_vec_u8(ret_buffer, out_len); 62 | ret = BrotliEncoderCompressWorkPool( 63 | work_pool, 64 | num_params, 65 | param_keys, 66 | param_values, 67 | len, 68 | data, 69 | &out_len, 70 | ret_buffer->data, 71 | num_threads, 72 | NULL /* custom allocator*/, NULL, NULL); 73 | 74 | BrotliEncoderDestroyWorkPool(work_pool); 75 | ``` 76 | 77 | An example can be seen in multiexample.c 78 | -------------------------------------------------------------------------------- /c/arg.h: -------------------------------------------------------------------------------- 1 | #ifndef _WIN32 2 | #include 3 | #endif 4 | char * find_first_arg(int argc, char**argv, long long int *truncation_location) { 5 | int i; 6 | for (i = 1; i < argc; ++i) { 7 | if (strstr(argv[i], "-trunc=") == argv[i]) { 8 | *truncation_location = strtoll(argv[i] + 7, NULL, 10); 9 | } 10 | } 11 | for (i = 1; i < argc; ++i) { 12 | if (argv[i][0] != '-') { 13 | return argv[i]; 14 | } 15 | } 16 | return NULL; 17 | } 18 | size_t set_options(BrotliEncoderParameter *out_encoder_param_keys, 19 | uint32_t *out_encoder_param_values, 20 | int argc, char **argv, 21 | size_t size_hint, 22 | size_t* out_num_threads) { 23 | int i; 24 | size_t ret = 0; 25 | int used_cm = 0; 26 | out_encoder_param_keys[ret] = BROTLI_PARAM_SIZE_HINT; 27 | out_encoder_param_values[ret] = (uint32_t)size_hint; 28 | ret += 1; 29 | for (i = 1; i < argc; ++i) { 30 | if (strstr(argv[i], "-q") == argv[i]) { 31 | out_encoder_param_keys[ret] = BROTLI_PARAM_QUALITY; 32 | out_encoder_param_values[ret] = atoi(argv[i] + 2); 33 | ret += 1; 34 | } 35 | if (strstr(argv[i], "-p") == argv[i]) { 36 | out_encoder_param_keys[ret] = BROTLI_PARAM_LITERAL_BYTE_SCORE; 37 | out_encoder_param_values[ret] = atoi(argv[i] + 2); 38 | ret += 1; 39 | } 40 | if (strstr(argv[i], "-l") == argv[i]) { 41 | out_encoder_param_keys[ret] = BROTLI_PARAM_LGBLOCK; 42 | out_encoder_param_values[ret] = atoi(argv[i] + 2); 43 | ret += 1; 44 | } 45 | if (strstr(argv[i], "-w") == argv[i]) { 46 | out_encoder_param_keys[ret] = BROTLI_PARAM_LGWIN; 47 | out_encoder_param_values[ret] = atoi(argv[i] + 2); 48 | ret += 1; 49 | } 50 | if (strstr(argv[i], "-j") == argv[i]) { 51 | *out_num_threads = atoi(argv[i] + 2); 52 | } 53 | if (strstr(argv[i], "-timeout=") == argv[i]) { 54 | static int has_been_set = 0; 55 | int deadline = atoi(argv[i] + 9); 56 | #ifndef _WIN32 57 | if (!has_been_set) { 58 | (void)alarm(deadline); 59 | has_been_set = 1; 60 | } 61 | #endif 62 | } 63 | if (strstr(argv[i], "-m") == argv[i]) { 64 | out_encoder_param_keys[ret] = BROTLI_PARAM_MAGIC_NUMBER; 65 | out_encoder_param_values[ret] = 1; 66 | ret += 1; 67 | } 68 | if (strstr(argv[i], "-catable") == argv[i]) { 69 | out_encoder_param_keys[ret] = BROTLI_PARAM_CATABLE; 70 | out_encoder_param_values[ret] = 1; 71 | ret += 1; 72 | } 73 | } 74 | return ret; 75 | } 76 | -------------------------------------------------------------------------------- /c/brotli/broccoli.h: -------------------------------------------------------------------------------- 1 | #ifndef BROTLI_BROCCOLI_H 2 | #define BROTLI_BROCCOLI_H 3 | #include 4 | typedef struct BroccoliState_ { 5 | void *unused; 6 | unsigned char data[248]; 7 | } BroccoliState; 8 | 9 | typedef enum BroccoliResult_ { 10 | BroccoliSuccess = 0, 11 | BroccoliNeedsMoreInput = 1, 12 | BroccoliNeedsMoreOutput = 2, 13 | BroccoliBrotliFileNotCraftedForAppend = 124, 14 | BroccoliInvalidWindowSize = 125, 15 | BroccoliWindowSizeLargerThanPreviousFile = 126, 16 | BroccoliBrotliFileNotCraftedForConcatenation = 127, 17 | } BroccoliResult; 18 | 19 | 20 | BroccoliState BroccoliCreateInstance(); 21 | 22 | BroccoliState BroccoliCreateInstanceWithWindowSize(uint8_t window_size); 23 | 24 | void BroccoliDestroyInstance(BroccoliState state); 25 | 26 | void BroccoliNewBrotliFile(BroccoliState *state); 27 | 28 | BroccoliResult BroccoliConcatStream( 29 | BroccoliState *state, 30 | size_t *available_in, 31 | const uint8_t **input_buf_ptr, 32 | size_t *available_out, 33 | uint8_t **output_buf_ptr); 34 | 35 | BroccoliResult BroccoliConcatStreaming( 36 | BroccoliState *state, 37 | size_t *available_in, 38 | const uint8_t *input_buf_ptr, 39 | size_t *available_out, 40 | uint8_t *output_buf_ptr); 41 | 42 | BroccoliResult BroccoliConcatFinish(BroccoliState * state, 43 | size_t *available_out, 44 | uint8_t**output_buf); 45 | BroccoliResult BroccoliConcatFinished(BroccoliState * state, 46 | size_t *available_out, 47 | uint8_t*output_buf); 48 | #endif 49 | -------------------------------------------------------------------------------- /c/brotli/multiencode.h: -------------------------------------------------------------------------------- 1 | #include "encode.h" 2 | 3 | 4 | /** 5 | * Opaque structure that holds workpool thrads 6 | * 7 | * Allocated and initialized with ::BrotliEncoderCreateWorkPool. 8 | * Cleaned up and deallocated with ::BrotliEncoderDestroyWorkPool. 9 | */ 10 | typedef struct BrotliEncoderWorkPoolStruct BrotliEncoderWorkPool; 11 | 12 | /** 13 | * Performs one-shot memory-to-memory compression. 14 | * 15 | * Compresses the data in @p input_buffer into @p encoded_buffer, and sets 16 | * @p *encoded_size to the compressed length. 17 | * 18 | * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero 19 | * value, then output is guaranteed to be no longer than that. 20 | * 21 | * @param num_params indicates how long both the param_keys and param_values arrays will be 22 | * @param param_keys is an array of BrotliEncoderParameters that must be the same length as param_values 23 | * @param param_values is an array of uint32_t and for each one of these, the matching param_keys will be changed to this value 24 | * @param input_size size of @p input_buffer 25 | * @param input_buffer input data buffer with at least @p input_size 26 | * addressable bytes 27 | * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n 28 | * @b out: length of compressed data written to 29 | * @p encoded_buffer, or @c 0 if compression fails 30 | * @param encoded_buffer compressed data destination buffer 31 | * @param desired_num_threads is an integer about the number of threads to spawn to encode the input_buffer 32 | * @param alloc_func is an optional allocator that will be called for any allocations. If null, builtin malloc will be used 33 | * @param free_func is an optional allocator that will be called for any frees. If null, builtin free will be used 34 | * @param alloc_opaque_per_thread is an opaque function that will be passed into both alloc and free. 35 | * each thread will have an independent allocator and will pass its own opaque. So this array must 36 | * either be entirely null (and the above functions also be null) or sized to the desired_num_threads 37 | * @returns ::BROTLI_FALSE in case of compression error 38 | * @returns ::BROTLI_FALSE if output buffer is too small 39 | * @returns ::BROTLI_TRUE otherwise 40 | */ 41 | BROTLI_ENC_API int32_t BrotliEncoderCompressMulti( 42 | size_t num_params, 43 | const BrotliEncoderParameter* param_keys, 44 | const uint32_t* param_values, 45 | size_t input_size, 46 | const uint8_t *input_buffer, 47 | size_t *encoded_size, 48 | uint8_t *encoded, 49 | size_t desired_num_threads, 50 | brotli_alloc_func alloc_func, brotli_free_func free_func, 51 | void** alloc_opaque_per_thread); 52 | 53 | 54 | 55 | BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSizeMulti(size_t input_size, size_t num_threads); 56 | 57 | /** 58 | * Creates an instance of ::BrotliEncoderWorkPool and initializes it, spawning num_threads threads 59 | * 60 | * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the 61 | * case they are both zero, default memory allocators are used. @p opaque is 62 | * passed to @p alloc_func and @p free_func when they are called. @p free_func 63 | * has to return without doing anything when asked to free a NULL pointer. 64 | * 65 | * @param alloc_func custom memory allocation function 66 | * @param free_func custom memory free function 67 | * @param opaque custom memory manager handle 68 | * @returns @c 0 if instance can not be allocated or initialized 69 | * @returns pointer to initialized ::BrotliEncoderWorkPool otherwise 70 | */ 71 | BROTLI_ENC_API BrotliEncoderWorkPool* BrotliEncoderCreateWorkPool( 72 | size_t num_threads, 73 | brotli_alloc_func alloc_func, 74 | brotli_free_func free_func, 75 | void** alloc_opaque_per_thread); 76 | 77 | 78 | 79 | /** 80 | * Deinitializes and frees ::BrotliEncoderWorkPool instance. 81 | * 82 | * @param state work pool instance to be cleaned up and deallocated 83 | */ 84 | BROTLI_ENC_API void BrotliEncoderDestroyWorkPool(BrotliEncoderWorkPool* work_pool); 85 | 86 | 87 | /** 88 | * Performs one-shot memory-to-memory compression. 89 | * 90 | * Compresses the data in @p input_buffer into @p encoded_buffer, and sets 91 | * @p *encoded_size to the compressed length. Runs compression using existing threads spawned in work_pool 92 | * 93 | * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero 94 | * value, then output is guaranteed to be no longer than that. 95 | * 96 | * @param num_params indicates how long both the param_keys and param_values arrays will be 97 | * @param param_keys is an array of BrotliEncoderParameters that must be the same length as param_values 98 | * @param param_values is an array of uint32_t and for each one of these, the matching param_keys will be changed to this value 99 | * @param input_size size of @p input_buffer 100 | * @param input_buffer input data buffer with at least @p input_size 101 | * addressable bytes 102 | * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n 103 | * @b out: length of compressed data written to 104 | * @p encoded_buffer, or @c 0 if compression fails 105 | * @param encoded_buffer compressed data destination buffer 106 | * @param desired_num_threads is an integer about the number of compression jobs the file should be split into 107 | * @param alloc_func is an optional allocator that will be called for any allocations. If null, builtin malloc will be used 108 | * @param free_func is an optional allocator that will be called for any frees. If null, builtin free will be used 109 | * @param alloc_opaque_per_thread is an opaque function that will be passed into both alloc and free. 110 | * each thread will have an independent allocator and will pass its own opaque. So this array must 111 | * either be entirely null (and the above functions also be null) or sized to the desired_num_threads 112 | * @returns ::BROTLI_FALSE in case of compression error 113 | * @returns ::BROTLI_FALSE if output buffer is too small 114 | * @returns ::BROTLI_TRUE otherwise 115 | */ 116 | BROTLI_ENC_API int32_t BrotliEncoderCompressWorkPool( 117 | BrotliEncoderWorkPool *work_pool, 118 | size_t num_params, 119 | const BrotliEncoderParameter* param_keys, 120 | const uint32_t* param_values, 121 | size_t input_size, 122 | const uint8_t *input_buffer, 123 | size_t *encoded_size, 124 | uint8_t *encoded, 125 | size_t desired_num_threads, 126 | brotli_alloc_func alloc_func, brotli_free_func free_func, 127 | void** alloc_opaque_per_thread); 128 | 129 | 130 | -------------------------------------------------------------------------------- /c/brotli/port.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Macros for compiler / platform specific features and build options. 8 | 9 | Build options are: 10 | * BROTLI_BUILD_MODERN_COMPILER forces to use modern compilers built-ins, 11 | features and attributes 12 | */ 13 | 14 | #ifndef BROTLI_COMMON_PORT_H_ 15 | #define BROTLI_COMMON_PORT_H_ 16 | 17 | #if defined(__GNUC__) && defined(__GNUC_MINOR__) 18 | #define BROTLI_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 19 | #else 20 | #define BROTLI_GCC_VERSION 0 21 | #endif 22 | 23 | #if defined(__ICC) 24 | #define BROTLI_ICC_VERSION __ICC 25 | #else 26 | #define BROTLI_ICC_VERSION 0 27 | #endif 28 | 29 | #if defined(BROTLI_BUILD_MODERN_COMPILER) 30 | #define BROTLI_MODERN_COMPILER 1 31 | #elif BROTLI_GCC_VERSION >= 304 || BROTLI_ICC_VERSION >= 1600 32 | #define BROTLI_MODERN_COMPILER 1 33 | #else 34 | #define BROTLI_MODERN_COMPILER 0 35 | #endif 36 | 37 | #if defined(BROTLI_SHARED_COMPILATION) && defined(_WIN32) 38 | #if defined(BROTLICOMMON_SHARED_COMPILATION) 39 | #define BROTLI_COMMON_API __declspec(dllexport) 40 | #else 41 | #define BROTLI_COMMON_API __declspec(dllimport) 42 | #endif /* BROTLICOMMON_SHARED_COMPILATION */ 43 | #if defined(BROTLIDEC_SHARED_COMPILATION) 44 | #define BROTLI_DEC_API __declspec(dllexport) 45 | #else 46 | #define BROTLI_DEC_API __declspec(dllimport) 47 | #endif /* BROTLIDEC_SHARED_COMPILATION */ 48 | #if defined(BROTLIENC_SHARED_COMPILATION) 49 | #define BROTLI_ENC_API __declspec(dllexport) 50 | #else 51 | #define BROTLI_ENC_API __declspec(dllimport) 52 | #endif /* BROTLIENC_SHARED_COMPILATION */ 53 | #else /* BROTLI_SHARED_COMPILATION && _WIN32 */ 54 | #define BROTLI_COMMON_API 55 | #define BROTLI_DEC_API 56 | #define BROTLI_ENC_API 57 | #endif 58 | 59 | #endif /* BROTLI_COMMON_PORT_H_ */ 60 | -------------------------------------------------------------------------------- /c/brotli/types.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /** 8 | * @file 9 | * Common types used in decoder and encoder API. 10 | */ 11 | 12 | #ifndef BROTLI_COMMON_TYPES_H_ 13 | #define BROTLI_COMMON_TYPES_H_ 14 | 15 | #include /* for size_t */ 16 | 17 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 18 | typedef __int8 int8_t; 19 | typedef unsigned __int8 uint8_t; 20 | typedef __int16 int16_t; 21 | typedef unsigned __int16 uint16_t; 22 | typedef __int32 int32_t; 23 | typedef unsigned __int32 uint32_t; 24 | typedef unsigned __int64 uint64_t; 25 | typedef __int64 int64_t; 26 | #else 27 | #include 28 | #endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */ 29 | 30 | /** 31 | * A portable @c bool replacement. 32 | * 33 | * ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it 34 | * denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE. 35 | * 36 | * ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or 37 | * ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros. 38 | * 39 | * ::BROTLI_BOOL values returned by Brotli should not be tested for equality 40 | * with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be 41 | * evaluated, for example: @code{.cpp} 42 | * if (SomeBrotliFunction(encoder, BROTLI_TRUE) && 43 | * !OtherBrotliFunction(decoder, BROTLI_FALSE)) { 44 | * bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4)); 45 | * DoSomething(x); 46 | * } 47 | * @endcode 48 | */ 49 | #define BROTLI_BOOL int 50 | /** Portable @c true replacement. */ 51 | #define BROTLI_TRUE 1 52 | /** Portable @c false replacement. */ 53 | #define BROTLI_FALSE 0 54 | /** @c bool to ::BROTLI_BOOL conversion macros. */ 55 | #define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE) 56 | 57 | #define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low) 58 | 59 | #define BROTLI_UINT32_MAX (~((uint32_t)0)) 60 | #define BROTLI_SIZE_MAX (~((size_t)0)) 61 | 62 | /** 63 | * Allocating function pointer type. 64 | * 65 | * @param opaque custom memory manager handle provided by client 66 | * @param size requested memory region size; can not be @c 0 67 | * @returns @c 0 in the case of failure 68 | * @returns a valid pointer to a memory region of at least @p size bytes 69 | * long otherwise 70 | */ 71 | typedef void* (*brotli_alloc_func)(void* opaque, size_t size); 72 | 73 | /** 74 | * Deallocating function pointer type. 75 | * 76 | * This function @b SHOULD do nothing if @p address is @c 0. 77 | * 78 | * @param opaque custom memory manager handle provided by client 79 | * @param address memory region pointer returned by ::brotli_alloc_func, or @c 0 80 | */ 81 | typedef void (*brotli_free_func)(void* opaque, void* address); 82 | 83 | #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ 84 | !defined(__cplusplus) && !defined(__PGI) 85 | #define BROTLI_ARRAY_PARAM(L) L 86 | #else 87 | #define BROTLI_ARRAY_PARAM(L) 88 | #endif 89 | 90 | #endif /* BROTLI_COMMON_TYPES_H_ */ 91 | -------------------------------------------------------------------------------- /c/catbrotli.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "brotli/broccoli.h" 6 | 7 | void usage() { 8 | fprintf(stderr, 9 | "Usage: [-w] filename0 filename1 filename2..."); 10 | } 11 | 12 | 13 | int main(int argc, char**argv) { 14 | unsigned char window_size = 0; 15 | unsigned char has_window_size = 0; 16 | unsigned char double_dash = 0; 17 | size_t buffer_size = 4096; 18 | int i; 19 | if (argc == 1) { 20 | usage(); 21 | return 1; 22 | } 23 | for (i = 1; i < argc; ++i) { 24 | if (strcmp(argv[i], "-help") == 0 && !double_dash) { 25 | usage(); 26 | return 1; 27 | } 28 | if (strcmp(argv[i], "--help") == 0 && !double_dash) { 29 | usage(); 30 | return 1; 31 | } 32 | if (strcmp(argv[i], "-h") == 0 && !double_dash) { 33 | usage(); 34 | return 1; 35 | } 36 | if (strncmp(argv[i], "-w", 2) == 0 && !double_dash) { 37 | int j; 38 | has_window_size = 1; 39 | window_size = atoi(argv[i] + 2); 40 | for (j = i; j + 1 < argc; ++j) { 41 | argv[j] = argv[j+1]; 42 | } 43 | --i; 44 | --argc; 45 | continue; 46 | } 47 | if (strncmp(argv[i], "-bs", 3) == 0 && !double_dash) { 48 | int j; 49 | buffer_size = atoi(argv[i] + 3); 50 | for (j = i; j + 1 < argc; ++j) { 51 | argv[j] = argv[j+1]; 52 | } 53 | --i; 54 | --argc; 55 | continue; 56 | } 57 | if (strcmp(argv[i], "--") == 0) { 58 | int j; 59 | double_dash = 1; 60 | for (j = i; j + 1 < argc; ++j) { 61 | argv[j] = argv[j+1]; 62 | } 63 | --i; 64 | --argc; 65 | continue; 66 | } 67 | } 68 | char ** filenames = &argv[1]; 69 | unsigned char * ibuffer = (unsigned char*)malloc(buffer_size); 70 | unsigned char * obuffer = (unsigned char*)malloc(buffer_size); 71 | unsigned char* obuffer_ptr = obuffer; 72 | size_t avail_out = buffer_size; 73 | BroccoliState state; 74 | if (has_window_size) { 75 | state = BroccoliCreateInstanceWithWindowSize(window_size); 76 | } else { 77 | state = BroccoliCreateInstance(); 78 | } 79 | for (i = 1; i < argc; ++i) { 80 | BroccoliNewBrotliFile(&state); 81 | FILE * input_file = fopen(argv[i], "rb"); 82 | if (!input_file) { 83 | fprintf(stderr, "Could not open %s\n", argv[i]); 84 | usage(); 85 | return 1; 86 | } 87 | while(1) { 88 | size_t cur_read = fread(ibuffer, 1, buffer_size, input_file); 89 | const unsigned char *ibuffer_ptr = ibuffer; 90 | size_t avail_in = cur_read; 91 | if (cur_read == 0) { 92 | break; 93 | } 94 | while(1) { 95 | BroccoliResult res = BroccoliConcatStream( 96 | &state, 97 | &avail_in, 98 | &ibuffer_ptr, 99 | &avail_out, 100 | &obuffer_ptr); 101 | if (res == BroccoliSuccess) { 102 | abort(); 103 | } 104 | if (res == BroccoliNeedsMoreInput) { 105 | break; 106 | } 107 | if (res == BroccoliNeedsMoreOutput) { 108 | fwrite(obuffer, 1, (obuffer_ptr - obuffer), stdout); 109 | obuffer_ptr = obuffer; 110 | avail_out = buffer_size; 111 | continue; 112 | } 113 | abort(); 114 | } 115 | } 116 | } 117 | while(1) { 118 | BroccoliResult res = BroccoliConcatFinish( 119 | &state, 120 | &avail_out, 121 | &obuffer_ptr); 122 | if (res == BroccoliNeedsMoreOutput) { 123 | fwrite(obuffer, 1, (obuffer_ptr - obuffer), stdout); 124 | obuffer_ptr = obuffer; 125 | avail_out = buffer_size; 126 | continue; 127 | } 128 | if (res == BroccoliNeedsMoreInput) { 129 | abort(); 130 | } 131 | if (res == BroccoliSuccess) { 132 | fwrite(obuffer, 1, (obuffer_ptr - obuffer), stdout); 133 | break; 134 | } 135 | abort(); //failure 136 | } 137 | return 0; 138 | } 139 | -------------------------------------------------------------------------------- /c/custom_alloc.h: -------------------------------------------------------------------------------- 1 | int use_real_malloc = 1; 2 | int use_fake_malloc = 0; 3 | void* custom_alloc_opaque = &use_real_malloc; 4 | unsigned char huge_buffer[1024*1024 * 255]; 5 | size_t huge_buffer_offset = 0; 6 | const uint32_t science = 0x5C1E11CE; 7 | 8 | void * custom_malloc_f(void* opaque, size_t user_size) { 9 | unsigned char * retval; 10 | size_t amt = user_size + 2*sizeof(opaque) + 4 + 32; 11 | if (opaque == &use_fake_malloc) { 12 | retval = &huge_buffer[huge_buffer_offset]; 13 | huge_buffer_offset += amt; 14 | } else { 15 | retval = (unsigned char*)malloc(amt); 16 | } 17 | memset(retval, 0x34, 2*sizeof(opaque) + 4 + 32); // make sure control areas are initialized to something--to help debug 18 | memcpy(retval, &science, 4); 19 | memcpy(retval + 4, &opaque, sizeof(opaque)); 20 | memcpy(retval + 4 + sizeof(opaque), &user_size, sizeof(size_t)); 21 | signed char alignment_offset = (32 - (((size_t)(retval + 4 + sizeof(opaque) + sizeof(size_t) + 1)) & 0x1f)) & 0x1f; 22 | retval[sizeof(opaque) + sizeof(size_t) + 4 + alignment_offset] = alignment_offset; 23 | void * final_return = retval + sizeof(opaque) + sizeof(size_t) + 4 + 1 + alignment_offset; 24 | assert((((size_t)final_return)&0x1f) == 0); 25 | return final_return; 26 | } 27 | void * (*custom_malloc)(void* opaque, size_t data) = &custom_malloc_f; 28 | void custom_free_f(void* opaque, void *mfd) { 29 | void * local_opaque; 30 | uint32_t local_science; 31 | size_t local_size = 0; 32 | char * local_mfd = (char *)mfd; 33 | if (mfd == NULL) { 34 | return; 35 | } 36 | local_mfd -= 1; 37 | local_mfd -= *local_mfd; 38 | local_mfd -= 4; 39 | local_mfd -= sizeof(opaque); 40 | local_mfd -= sizeof(size_t); 41 | memcpy(&local_science, local_mfd, 4); 42 | assert(local_science == science); 43 | memcpy(&local_opaque, local_mfd + 4, sizeof(opaque)); 44 | memcpy(&local_size, local_mfd + 4 + sizeof(opaque), sizeof(size_t)); 45 | assert(opaque == local_opaque); 46 | if (opaque == &use_fake_malloc) { 47 | void *retval = &huge_buffer[huge_buffer_offset]; 48 | if ((void*)(retval - local_size) == mfd) { 49 | huge_buffer_offset -= 4 + sizeof(opaque) + sizeof(size_t) + local_size; 50 | } 51 | } else { 52 | free(local_mfd); 53 | } 54 | } 55 | 56 | void (*custom_free)(void* opaque, void *mfd) = &custom_free_f; 57 | void custom_atoi(char * dst, size_t data) { 58 | if (!data) { 59 | memcpy(dst, "0\0", 2); 60 | return; 61 | } 62 | char *ptr = dst; 63 | while(data) { 64 | *ptr = '0' + (data % 10); 65 | ++ptr; 66 | data /= 10; 67 | } 68 | *ptr = '\0'; 69 | int del = (int)(ptr - dst); 70 | int i; 71 | for (i = 0;i < del/2;i+= 1) { 72 | char tmp = dst[i]; 73 | dst[i] = *(ptr - i - 1); 74 | *(ptr - i - 1) = tmp; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /c/decompressor.c: -------------------------------------------------------------------------------- 1 | #include "brotli/decode.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | int custom_alloc_data = 0; 7 | void * custom_alloc(void*opaque, size_t size) { 8 | assert(opaque == &custom_alloc_data); 9 | return malloc(size); 10 | } 11 | void custom_free(void*opaque, void* addr) { 12 | assert(opaque == &custom_alloc_data); 13 | free(addr); 14 | } 15 | 16 | void simple_test() { 17 | #ifndef NOSTDLIB 18 | const unsigned char brotli_file[] = {0x1b, 0x30, 0x00, 0xe0, 0x8d, 0xd4, 0x59, 0x2d, 0x39, 0x37, 0xb5, 0x02, 19 | 0x48, 0x10, 0x95, 0x2a, 0x9a, 0xea, 0x42, 0x0e, 0x51, 0xa4, 0x16, 0xb9, 20 | 0xcb, 0xf5, 0xf8, 0x5c, 0x64, 0xb9, 0x2f, 0xc9, 0x6a, 0x3f, 0xb1, 0xdc, 21 | 0xa8, 0xe0, 0x35, 0x07}; 22 | const unsigned char key[] = "THIS IS A TEST OF THE EMERGENCY BROADCAST SYSTEM"; 23 | unsigned char output[sizeof(key) * 2]; 24 | size_t decoded_size = sizeof(output) * 2; 25 | BrotliDecoderDecompress(sizeof(brotli_file), brotli_file, &decoded_size, output); 26 | assert(decoded_size == sizeof(key)); 27 | assert(memcmp(output, key, sizeof(key) - 1) == 0); 28 | assert(output[sizeof(key) - 1] == '\n'); 29 | #endif 30 | } 31 | 32 | int main() { 33 | simple_test(); 34 | BrotliDecoderState * state = BrotliDecoderCreateInstance(custom_alloc, custom_free, &custom_alloc_data); 35 | unsigned char ibuffer[4096]; 36 | unsigned char obuffer[4096]; 37 | size_t total_out = 0; 38 | BrotliDecoderResult rest; 39 | while(1) { 40 | size_t avail_in = fread(ibuffer, 1, sizeof(ibuffer), stdin); 41 | int is_eof = (avail_in == 0); 42 | const unsigned char *i_ptr = &ibuffer[0]; 43 | while (1) { 44 | unsigned char *o_ptr = &obuffer[0]; 45 | size_t avail_out = sizeof(obuffer); 46 | rest = BrotliDecoderDecompressStream(state, &avail_in, &i_ptr, &avail_out, &o_ptr, &total_out); 47 | if (o_ptr != &obuffer[0]) { 48 | size_t ret = fwrite(obuffer, 1, o_ptr - &obuffer[0], stdout); 49 | assert(ret == o_ptr - &obuffer[0]); 50 | } 51 | if (rest == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { 52 | break; 53 | } 54 | if (rest == BROTLI_DECODER_RESULT_SUCCESS || rest == BROTLI_DECODER_RESULT_ERROR) { 55 | break; 56 | } 57 | } 58 | if (rest == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && is_eof) { 59 | fprintf(stderr, "Unexpected EOF\n"); 60 | exit(1); 61 | } 62 | if (rest == BROTLI_DECODER_RESULT_SUCCESS || rest == BROTLI_DECODER_RESULT_ERROR) { 63 | break; 64 | } 65 | } 66 | BrotliDecoderDestroyInstance(state); 67 | } 68 | -------------------------------------------------------------------------------- /c/fuzz.c: -------------------------------------------------------------------------------- 1 | #include "brotli/decode.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | int custom_alloc_data = 0; 7 | void * custom_alloc(void*opaque, size_t size) { 8 | assert(opaque == &custom_alloc_data); 9 | return malloc(size); 10 | } 11 | void custom_free(void*opaque, void* addr) { 12 | assert(opaque == &custom_alloc_data); 13 | free(addr); 14 | } 15 | 16 | int main(int argc, char **argv) { 17 | FILE * fp = argc > 1 ? fopen(argv[1], "rb") : stdin; 18 | BrotliDecoderState * state = BrotliDecoderCreateInstance(custom_alloc, custom_free, &custom_alloc_data); 19 | unsigned char ibuffer[4096]; 20 | unsigned char obuffer[4096]; 21 | size_t total_out = 0; 22 | BrotliDecoderResult rest; 23 | while(1) { 24 | size_t avail_in = fread(ibuffer, 1, sizeof(ibuffer), fp); 25 | int is_eof = (avail_in == 0); 26 | const unsigned char *i_ptr = &ibuffer[0]; 27 | while (1) { 28 | unsigned char *o_ptr = &obuffer[0]; 29 | size_t avail_out = sizeof(obuffer); 30 | rest = BrotliDecoderDecompressStream(state, &avail_in, &i_ptr, &avail_out, &o_ptr, &total_out); 31 | if (o_ptr != &obuffer[0]) { 32 | // don't actually write 33 | } 34 | if (rest == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { 35 | break; 36 | } 37 | if (rest == BROTLI_DECODER_RESULT_SUCCESS || rest == BROTLI_DECODER_RESULT_ERROR) { 38 | break; 39 | } 40 | } 41 | if (rest == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && is_eof) { 42 | fprintf(stderr, "Unexpected EOF\n"); 43 | break; 44 | } 45 | if (rest == BROTLI_DECODER_RESULT_SUCCESS) { 46 | break; 47 | } 48 | if (rest == BROTLI_DECODER_RESULT_ERROR) { 49 | fprintf(stderr, "Error: %s\n", BrotliDecoderGetErrorString(state)); 50 | if (BrotliDecoderGetErrorCode(state) == BROTLI_DECODER_ERROR_UNREACHABLE) { 51 | abort(); // possibly a stack trace 52 | } 53 | break; 54 | } 55 | } 56 | BrotliDecoderDestroyInstance(state); 57 | if (rest == BROTLI_DECODER_RESULT_ERROR) { 58 | fprintf(stderr, "Not a valid brotli file\n"); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /c/go/brotli/brotli/broccoli.h: -------------------------------------------------------------------------------- 1 | #ifndef BROTLI_BROCCOLI_H 2 | #define BROTLI_BROCCOLI_H 3 | #include 4 | typedef struct BroccoliState_ { 5 | void *unused; 6 | unsigned char data[248]; 7 | } BroccoliState; 8 | 9 | typedef enum BroccoliResult_ { 10 | BroccoliSuccess = 0, 11 | BroccoliNeedsMoreInput = 1, 12 | BroccoliNeedsMoreOutput = 2, 13 | BroccoliBrotliFileNotCraftedForAppend = 124, 14 | BroccoliInvalidWindowSize = 125, 15 | BroccoliWindowSizeLargerThanPreviousFile = 126, 16 | BroccoliBrotliFileNotCraftedForConcatenation = 127, 17 | } BroccoliResult; 18 | 19 | 20 | BroccoliState BroccoliCreateInstance(); 21 | 22 | BroccoliState BroccoliCreateInstanceWithWindowSize(uint8_t window_size); 23 | 24 | void BroccoliDestroyInstance(BroccoliState state); 25 | 26 | void BroccoliNewBrotliFile(BroccoliState *state); 27 | 28 | BroccoliResult BroccoliConcatStream( 29 | BroccoliState *state, 30 | size_t *available_in, 31 | const uint8_t **input_buf_ptr, 32 | size_t *available_out, 33 | uint8_t **output_buf_ptr); 34 | 35 | BroccoliResult BroccoliConcatFinish(BroccoliState * state, 36 | size_t *available_out, 37 | uint8_t**output_buf); 38 | #endif 39 | -------------------------------------------------------------------------------- /c/go/brotli/brotli/multiencode.h: -------------------------------------------------------------------------------- 1 | #include "encode.h" 2 | 3 | 4 | /** 5 | * Opaque structure that holds workpool thrads 6 | * 7 | * Allocated and initialized with ::BrotliEncoderCreateWorkPool. 8 | * Cleaned up and deallocated with ::BrotliEncoderDestroyWorkPool. 9 | */ 10 | typedef struct BrotliEncoderWorkPoolStruct BrotliEncoderWorkPool; 11 | 12 | /** 13 | * Performs one-shot memory-to-memory compression. 14 | * 15 | * Compresses the data in @p input_buffer into @p encoded_buffer, and sets 16 | * @p *encoded_size to the compressed length. 17 | * 18 | * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero 19 | * value, then output is guaranteed to be no longer than that. 20 | * 21 | * @param num_params indicates how long both the param_keys and param_values arrays will be 22 | * @param param_keys is an array of BrotliEncoderParameters that must be the same length as param_values 23 | * @param param_values is an array of uint32_t and for each one of these, the matching param_keys will be changed to this value 24 | * @param input_size size of @p input_buffer 25 | * @param input_buffer input data buffer with at least @p input_size 26 | * addressable bytes 27 | * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n 28 | * @b out: length of compressed data written to 29 | * @p encoded_buffer, or @c 0 if compression fails 30 | * @param encoded_buffer compressed data destination buffer 31 | * @param desired_num_threads is an integer about the number of threads to spawn to encode the input_buffer 32 | * @param alloc_func is an optional allocator that will be called for any allocations. If null, builtin malloc will be used 33 | * @param free_func is an optional allocator that will be called for any frees. If null, builtin free will be used 34 | * @param alloc_opaque_per_thread is an opaque function that will be passed into both alloc and free. 35 | * each thread will have an independent allocator and will pass its own opaque. So this array must 36 | * either be entirely null (and the above functions also be null) or sized to the desired_num_threads 37 | * @returns ::BROTLI_FALSE in case of compression error 38 | * @returns ::BROTLI_FALSE if output buffer is too small 39 | * @returns ::BROTLI_TRUE otherwise 40 | */ 41 | BROTLI_ENC_API int32_t BrotliEncoderCompressMulti( 42 | size_t num_params, 43 | const BrotliEncoderParameter* param_keys, 44 | const uint32_t* param_values, 45 | size_t input_size, 46 | const uint8_t *input_buffer, 47 | size_t *encoded_size, 48 | uint8_t *encoded, 49 | size_t desired_num_threads, 50 | brotli_alloc_func alloc_func, brotli_free_func free_func, 51 | void** alloc_opaque_per_thread); 52 | 53 | 54 | 55 | BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSizeMulti(size_t input_size, size_t num_threads); 56 | 57 | /** 58 | * Creates an instance of ::BrotliEncoderWorkPool and initializes it, spawning num_threads threads 59 | * 60 | * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the 61 | * case they are both zero, default memory allocators are used. @p opaque is 62 | * passed to @p alloc_func and @p free_func when they are called. @p free_func 63 | * has to return without doing anything when asked to free a NULL pointer. 64 | * 65 | * @param alloc_func custom memory allocation function 66 | * @param free_func custom memory free function 67 | * @param opaque custom memory manager handle 68 | * @returns @c 0 if instance can not be allocated or initialized 69 | * @returns pointer to initialized ::BrotliEncoderWorkPool otherwise 70 | */ 71 | BROTLI_ENC_API BrotliEncoderWorkPool* BrotliEncoderCreateWorkPool( 72 | size_t num_threads, 73 | brotli_alloc_func alloc_func, 74 | brotli_free_func free_func, 75 | void** alloc_opaque_per_thread); 76 | 77 | 78 | 79 | /** 80 | * Deinitializes and frees ::BrotliEncoderWorkPool instance. 81 | * 82 | * @param state work pool instance to be cleaned up and deallocated 83 | */ 84 | BROTLI_ENC_API void BrotliEncoderDestroyWorkPool(BrotliEncoderWorkPool* work_pool); 85 | 86 | 87 | /** 88 | * Performs one-shot memory-to-memory compression. 89 | * 90 | * Compresses the data in @p input_buffer into @p encoded_buffer, and sets 91 | * @p *encoded_size to the compressed length. Runs compression using existing threads spawned in work_pool 92 | * 93 | * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero 94 | * value, then output is guaranteed to be no longer than that. 95 | * 96 | * @param num_params indicates how long both the param_keys and param_values arrays will be 97 | * @param param_keys is an array of BrotliEncoderParameters that must be the same length as param_values 98 | * @param param_values is an array of uint32_t and for each one of these, the matching param_keys will be changed to this value 99 | * @param input_size size of @p input_buffer 100 | * @param input_buffer input data buffer with at least @p input_size 101 | * addressable bytes 102 | * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n 103 | * @b out: length of compressed data written to 104 | * @p encoded_buffer, or @c 0 if compression fails 105 | * @param encoded_buffer compressed data destination buffer 106 | * @param desired_num_threads is an integer about the number of compression jobs the file should be split into 107 | * @param alloc_func is an optional allocator that will be called for any allocations. If null, builtin malloc will be used 108 | * @param free_func is an optional allocator that will be called for any frees. If null, builtin free will be used 109 | * @param alloc_opaque_per_thread is an opaque function that will be passed into both alloc and free. 110 | * each thread will have an independent allocator and will pass its own opaque. So this array must 111 | * either be entirely null (and the above functions also be null) or sized to the desired_num_threads 112 | * @returns ::BROTLI_FALSE in case of compression error 113 | * @returns ::BROTLI_FALSE if output buffer is too small 114 | * @returns ::BROTLI_TRUE otherwise 115 | */ 116 | BROTLI_ENC_API int32_t BrotliEncoderCompressWorkPool( 117 | BrotliEncoderWorkPool *work_pool, 118 | size_t num_params, 119 | const BrotliEncoderParameter* param_keys, 120 | const uint32_t* param_values, 121 | size_t input_size, 122 | const uint8_t *input_buffer, 123 | size_t *encoded_size, 124 | uint8_t *encoded, 125 | size_t desired_num_threads, 126 | brotli_alloc_func alloc_func, brotli_free_func free_func, 127 | void** alloc_opaque_per_thread); 128 | 129 | 130 | -------------------------------------------------------------------------------- /c/go/brotli/brotli/port.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Macros for compiler / platform specific features and build options. 8 | 9 | Build options are: 10 | * BROTLI_BUILD_MODERN_COMPILER forces to use modern compilers built-ins, 11 | features and attributes 12 | */ 13 | 14 | #ifndef BROTLI_COMMON_PORT_H_ 15 | #define BROTLI_COMMON_PORT_H_ 16 | 17 | #if defined(__GNUC__) && defined(__GNUC_MINOR__) 18 | #define BROTLI_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 19 | #else 20 | #define BROTLI_GCC_VERSION 0 21 | #endif 22 | 23 | #if defined(__ICC) 24 | #define BROTLI_ICC_VERSION __ICC 25 | #else 26 | #define BROTLI_ICC_VERSION 0 27 | #endif 28 | 29 | #if defined(BROTLI_BUILD_MODERN_COMPILER) 30 | #define BROTLI_MODERN_COMPILER 1 31 | #elif BROTLI_GCC_VERSION >= 304 || BROTLI_ICC_VERSION >= 1600 32 | #define BROTLI_MODERN_COMPILER 1 33 | #else 34 | #define BROTLI_MODERN_COMPILER 0 35 | #endif 36 | 37 | #if defined(BROTLI_SHARED_COMPILATION) && defined(_WIN32) 38 | #if defined(BROTLICOMMON_SHARED_COMPILATION) 39 | #define BROTLI_COMMON_API __declspec(dllexport) 40 | #else 41 | #define BROTLI_COMMON_API __declspec(dllimport) 42 | #endif /* BROTLICOMMON_SHARED_COMPILATION */ 43 | #if defined(BROTLIDEC_SHARED_COMPILATION) 44 | #define BROTLI_DEC_API __declspec(dllexport) 45 | #else 46 | #define BROTLI_DEC_API __declspec(dllimport) 47 | #endif /* BROTLIDEC_SHARED_COMPILATION */ 48 | #if defined(BROTLIENC_SHARED_COMPILATION) 49 | #define BROTLI_ENC_API __declspec(dllexport) 50 | #else 51 | #define BROTLI_ENC_API __declspec(dllimport) 52 | #endif /* BROTLIENC_SHARED_COMPILATION */ 53 | #else /* BROTLI_SHARED_COMPILATION && _WIN32 */ 54 | #define BROTLI_COMMON_API 55 | #define BROTLI_DEC_API 56 | #define BROTLI_ENC_API 57 | #endif 58 | 59 | #endif /* BROTLI_COMMON_PORT_H_ */ 60 | -------------------------------------------------------------------------------- /c/go/brotli/brotli/types.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /** 8 | * @file 9 | * Common types used in decoder and encoder API. 10 | */ 11 | 12 | #ifndef BROTLI_COMMON_TYPES_H_ 13 | #define BROTLI_COMMON_TYPES_H_ 14 | 15 | #include /* for size_t */ 16 | 17 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 18 | typedef __int8 int8_t; 19 | typedef unsigned __int8 uint8_t; 20 | typedef __int16 int16_t; 21 | typedef unsigned __int16 uint16_t; 22 | typedef __int32 int32_t; 23 | typedef unsigned __int32 uint32_t; 24 | typedef unsigned __int64 uint64_t; 25 | typedef __int64 int64_t; 26 | #else 27 | #include 28 | #endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */ 29 | 30 | /** 31 | * A portable @c bool replacement. 32 | * 33 | * ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it 34 | * denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE. 35 | * 36 | * ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or 37 | * ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros. 38 | * 39 | * ::BROTLI_BOOL values returned by Brotli should not be tested for equality 40 | * with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be 41 | * evaluated, for example: @code{.cpp} 42 | * if (SomeBrotliFunction(encoder, BROTLI_TRUE) && 43 | * !OtherBrotliFunction(decoder, BROTLI_FALSE)) { 44 | * bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4)); 45 | * DoSomething(x); 46 | * } 47 | * @endcode 48 | */ 49 | #define BROTLI_BOOL int 50 | /** Portable @c true replacement. */ 51 | #define BROTLI_TRUE 1 52 | /** Portable @c false replacement. */ 53 | #define BROTLI_FALSE 0 54 | /** @c bool to ::BROTLI_BOOL conversion macros. */ 55 | #define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE) 56 | 57 | #define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low) 58 | 59 | #define BROTLI_UINT32_MAX (~((uint32_t)0)) 60 | #define BROTLI_SIZE_MAX (~((size_t)0)) 61 | 62 | /** 63 | * Allocating function pointer type. 64 | * 65 | * @param opaque custom memory manager handle provided by client 66 | * @param size requested memory region size; can not be @c 0 67 | * @returns @c 0 in the case of failure 68 | * @returns a valid pointer to a memory region of at least @p size bytes 69 | * long otherwise 70 | */ 71 | typedef void* (*brotli_alloc_func)(void* opaque, size_t size); 72 | 73 | /** 74 | * Deallocating function pointer type. 75 | * 76 | * This function @b SHOULD do nothing if @p address is @c 0. 77 | * 78 | * @param opaque custom memory manager handle provided by client 79 | * @param address memory region pointer returned by ::brotli_alloc_func, or @c 0 80 | */ 81 | typedef void (*brotli_free_func)(void* opaque, void* address); 82 | 83 | #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ 84 | !defined(__cplusplus) && !defined(__PGI) 85 | #define BROTLI_ARRAY_PARAM(L) L 86 | #else 87 | #define BROTLI_ARRAY_PARAM(L) 88 | #endif 89 | 90 | #endif /* BROTLI_COMMON_TYPES_H_ */ 91 | -------------------------------------------------------------------------------- /c/go/brotli/data_test.go: -------------------------------------------------------------------------------- 1 | package brotli 2 | 3 | const bufsize = 4096 * 1024 4 | 5 | func testData() [bufsize]byte { 6 | poem := []byte(`Once upon a midnight dreary, while I pondered, weak and weary, 7 | Over many a quaint and curious volume of forgotten lore- 8 | While I nodded, nearly napping, suddenly there came a tapping, 9 | As of some one gently rapping, rapping at my chamber door. 10 | "'Tis some visitor," I muttered, "tapping at my chamber door- 11 | Only this and nothing more." 12 | 13 | Ah, distinctly I remember it was in the bleak December; 14 | And each separate dying ember wrought its ghost upon the floor. 15 | Eagerly I wished the morrow;-vainly I had sought to borrow 16 | From my books surcease of sorrow-sorrow for the lost Lenore- 17 | For the rare and radiant maiden whom the angels name Lenore- 18 | Nameless here for evermore. 19 | 20 | And the silken, sad, uncertain rustling of each purple curtain 21 | Thrilled me-filled me with fantastic terrors never felt before; 22 | So that now, to still the beating of my heart, I stood repeating 23 | "'Tis some visitor entreating entrance at my chamber door- 24 | Some late visitor entreating entrance at my chamber door;- 25 | This it is and nothing more." 26 | 27 | Presently my soul grew stronger; hesitating then no longer, 28 | "Sir," said I, "or Madam, truly your forgiveness I implore; 29 | But the fact is I was napping, and so gently you came rapping, 30 | And so faintly you came tapping, tapping at my chamber door, 31 | That I scarce was sure I heard you"-here I opened wide the door;- 32 | Darkness there and nothing more. 33 | 34 | Deep into that darkness peering, long I stood there wondering, fearing, 35 | Doubting, dreaming dreams no mortal ever dared to dream before; 36 | But the silence was unbroken, and the stillness gave no token, 37 | And the only word there spoken was the whispered word, "Lenore?" 38 | This I whispered, and an echo murmured back the word, "Lenore!"- 39 | Merely this and nothing more. 40 | 41 | Back into the chamber turning, all my soul within me burning, 42 | Soon again I heard a tapping somewhat louder than before. 43 | "Surely," said I, "surely that is something at my window lattice; 44 | Let me see, then, what thereat is, and this mystery explore- 45 | Let my heart be still a moment and this mystery explore;- 46 | 'Tis the wind and nothing more!" 47 | 48 | Open here I flung the shutter, when, with many a flirt and flutter, 49 | In there stepped a stately Raven of the saintly days of yore; 50 | Not the least obeisance made he; not a minute stopped or stayed he; 51 | But, with mien of lord or lady, perched above my chamber door- 52 | Perched upon a bust of Pallas just above my chamber door- 53 | Perched, and sat, and nothing more. 54 | 55 | Then this ebony bird beguiling my sad fancy into smiling, 56 | By the grave and stern decorum of the countenance it wore, 57 | "Though thy crest be shorn and shaven, thou," I said, "art sure no craven, 58 | Ghastly grim and ancient Raven wandering from the Nightly shore- 59 | Tell me what thy lordly name is on the Night's Plutonian shore!" 60 | Quoth the Raven "Nevermore." 61 | 62 | Much I marvelled this ungainly fowl to hear discourse so plainly, 63 | Though its answer little meaning-little relevancy bore; 64 | For we cannot help agreeing that no living human being 65 | Ever yet was blessed with seeing bird above his chamber door- 66 | Bird or beast upon the sculptured bust above his chamber door, 67 | With such name as "Nevermore." 68 | 69 | But the Raven, sitting lonely on the placid bust, spoke only 70 | That one word, as if his soul in that one word he did outpour. 71 | Nothing farther then he uttered-not a feather then he fluttered- 72 | Till I scarcely more than muttered "Other friends have flown before- 73 | On the morrow he will leave me, as my Hopes have flown before." 74 | Then the bird said "Nevermore." 75 | 76 | Startled at the stillness broken by reply so aptly spoken, 77 | "Doubtless," said I, "what it utters is its only stock and store 78 | Caught from some unhappy master whom unmerciful Disaster 79 | Followed fast and followed faster till his songs one burden bore- 80 | Till the dirges of his Hope that melancholy burden bore 81 | Of ‘Never-nevermore'." 82 | 83 | But the Raven still beguiling all my fancy into smiling, 84 | Straight I wheeled a cushioned seat in front of bird, and bust and door; 85 | Then, upon the velvet sinking, I betook myself to linking 86 | Fancy unto fancy, thinking what this ominous bird of yore- 87 | What this grim, ungainly, ghastly, gaunt, and ominous bird of yore 88 | Meant in croaking "Nevermore." 89 | 90 | This I sat engaged in guessing, but no syllable expressing 91 | To the fowl whose fiery eyes now burned into my bosom's core; 92 | This and more I sat divining, with my head at ease reclining 93 | On the cushion's velvet lining that the lamp-light gloated o'er, 94 | But whose velvet-violet lining with the lamp-light gloating o'er, 95 | She shall press, ah, nevermore! 96 | 97 | Then, methought, the air grew denser, perfumed from an unseen censer 98 | Swung by Seraphim whose foot-falls tinkled on the tufted floor. 99 | "Wretch," I cried, "thy God hath lent thee-by these angels he hath sent thee 100 | Respite-respite and nepenthe from thy memories of Lenore; 101 | Quaff, oh quaff this kind nepenthe and forget this lost Lenore!" 102 | Quoth the Raven "Nevermore." 103 | 104 | "Prophet!" said I, "thing of evil!-prophet still, if bird or devil!- 105 | Whether Tempter sent, or whether tempest tossed thee here ashore, 106 | Desolate yet all undaunted, on this desert land enchanted- 107 | On this home by Horror haunted-tell me truly, I implore- 108 | Is there-is there balm in Gilead?-tell me-tell me, I implore!" 109 | Quoth the Raven "Nevermore." 110 | 111 | "Prophet!" said I, "thing of evil!-prophet still, if bird or devil! 112 | By that Heaven that bends above us-by that God we both adore- 113 | Tell this soul with sorrow laden if, within the distant Aidenn, 114 | It shall clasp a sainted maiden whom the angels name Lenore- 115 | Clasp a rare and radiant maiden whom the angels name Lenore." 116 | Quoth the Raven "Nevermore." 117 | 118 | "Be that word our sign of parting, bird or fiend!" I shrieked, upstarting- 119 | "Get thee back into the tempest and the Night's Plutonian shore! 120 | Leave no black plume as a token of that lie thy soul hath spoken! 121 | Leave my loneliness unbroken!-quit the bust above my door! 122 | Take thy beak from out my heart, and take thy form from off my door!" 123 | Quoth the Raven "Nevermore." 124 | 125 | And the Raven, never flitting, still is sitting, still is sitting 126 | On the pallid bust of Pallas just above my chamber door; 127 | And his eyes have all the seeming of a demon's that is dreaming, 128 | And the lamp-light o'er him streaming throws his shadow on the floor; 129 | And my soul from out that shadow that lies floating on the floor 130 | Shall be lifted-nevermore!`) 131 | var ret [bufsize]byte 132 | count := 0 133 | for index := 0; index+len(poem) <= bufsize; index += len(poem) { 134 | copy(ret[index:index+len(poem)], poem) 135 | for si := 0; si < len(poem); si += 1 { 136 | ret[index+si] += byte(count) 137 | } 138 | count += 1 139 | } 140 | return ret 141 | } 142 | -------------------------------------------------------------------------------- /c/go/brotli/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dropbox/rust-brotli/c/go/brotli 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /c/go/brotli/header.go: -------------------------------------------------------------------------------- 1 | package brotli 2 | 3 | import ( 4 | "encoding/hex" 5 | "errors" 6 | ) 7 | 8 | func BrotliParseHeader(data []byte) (returnedVersion byte, returnedSize uint64, err error) { 9 | if len(data) < 4 { 10 | return 0, 0, errors.New("Insufficient data provided in string: " + hex.EncodeToString(data)) 11 | } 12 | hdr := uint64(data[0]) + uint64(data[1])*256 + uint64(data[2])*65536 + uint64(data[3])*65536*256 13 | var bits uint 14 | if hdr&1 == 0 { 15 | bits = 1 16 | } else if (hdr & 15) != 1 { 17 | bits = 4 18 | } else if (hdr & 127) != 0x11 { 19 | bits = 7 20 | } else { 21 | bits = 14 22 | } 23 | hdr >>= bits 24 | if (hdr & 1) != 0 { 25 | return 0, 0, errors.New("Header incorrectly marked as last block") 26 | } 27 | hdr >>= 1 28 | bits += 1 29 | if (hdr & 3) != 3 { 30 | return 0, 0, errors.New("Header incorrectly contains file data") 31 | } 32 | hdr >>= 2 33 | bits += 2 34 | if (hdr & 1) != 0 { 35 | return 0, 0, errors.New("Reserved 0 metadata bit is set to nonzero value") 36 | } 37 | hdr >>= 1 38 | bits += 1 39 | if (hdr & 3) != 1 { 40 | return 0, 0, errors.New("Header should only need 1 byte of length data") 41 | } 42 | hdr >>= 2 43 | bits += 2 44 | num_raw_header_bytes := 1 + (hdr & 0xff) 45 | bits += 8 46 | byte_offset := ((uint64(bits) + 7) / 8) 47 | if uint64(len(data)) < byte_offset+num_raw_header_bytes { 48 | return 0, 0, errors.New("Insufficient data to accomodate number of raw header bytes " + hex.EncodeToString(data)) 49 | } 50 | if data[byte_offset] != 0xe1 || data[byte_offset+1] != 0x97 || (data[byte_offset+2]&0xf0) != 0x80 || (data[byte_offset+2]&0xf) > 2 { 51 | return 0, 0, errors.New("Header does not start with E1978X: " + hex.EncodeToString(data[byte_offset:byte_offset+3])) 52 | } 53 | version := data[byte_offset+3] 54 | size_le := data[byte_offset+4 : byte_offset+num_raw_header_bytes] 55 | total_size := uint64(0) 56 | for index := uint(0); index < uint(len(size_le)); index += 1 { 57 | total_size += uint64(size_le[index]&0x7f) << (7 * index) 58 | if 0 == (size_le[index] & 0x80) { 59 | break 60 | } 61 | } 62 | return version, total_size, nil 63 | } 64 | -------------------------------------------------------------------------------- /c/go/go.mod: -------------------------------------------------------------------------------- 1 | go 1.18 2 | 3 | module main 4 | 5 | require github.com/dropbox/rust-brotli/c/go/brotli v0.0.0-20220217093550-f3a32293f213 6 | 7 | replace github.com/dropbox/rust-brotli/c/go/brotli => /home/danielrh/dev/rust-brotli/c/go/brotli 8 | -------------------------------------------------------------------------------- /c/go/go.sum: -------------------------------------------------------------------------------- 1 | github.com/dropbox/rust-brotli v0.0.0-20220217093550-f3a32293f213 h1:Owe69oFhx4/qjakt5WPh8rE0mvDmC4Cvp+HXehtNWv0= 2 | github.com/dropbox/rust-brotli v0.0.0-20220217093550-f3a32293f213/go.mod h1:o9aagpQtkLuBPi9cZUr2JaEc8bKXsTUre1iljDlfDLA= 3 | -------------------------------------------------------------------------------- /c/go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dropbox/rust-brotli/c/go/brotli" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | decompress := false 12 | options := brotli.CompressionOptions{ 13 | NumThreads: 1, 14 | Quality: 9.5, 15 | Catable: true, 16 | Appendable: true, 17 | Magic: false, 18 | } 19 | useWriter := false 20 | var toCat []string 21 | for index, arg := range os.Args { 22 | if index == 0 { 23 | continue 24 | } 25 | if arg == "-w" { 26 | useWriter = true 27 | } 28 | if arg == "-d" { 29 | decompress = true 30 | } 31 | if arg == "-dirtree" { 32 | recursiveVerify(os.Args[index+1]) 33 | return 34 | } 35 | if arg == "-cat" { 36 | toCat = append(toCat, os.Args[index+1:]...) 37 | break 38 | } 39 | } 40 | if toCat != nil { 41 | if useWriter { 42 | buffers := make([][]byte, len(toCat)) 43 | for index, fn := range toCat { 44 | var err error 45 | buffers[index], err = ioutil.ReadFile(fn) 46 | if err != nil { 47 | panic(err) 48 | } 49 | } 50 | final, err := brotli.BroccoliConcat(buffers...) 51 | if err != nil { 52 | panic(err) 53 | } 54 | _, err = os.Stdout.Write(final) 55 | if err != nil { 56 | panic(err) 57 | } 58 | } else { 59 | files := make([]io.Reader, len(toCat)) 60 | for index, fn := range toCat { 61 | var err error 62 | files[index], err = os.Open(fn) 63 | if err != nil { 64 | panic(err) 65 | } 66 | } 67 | _, err := io.Copy(os.Stdout, brotli.NewBroccoliConcatReader(files...)) 68 | if err != nil { 69 | panic(err) 70 | } 71 | for _, file := range files { 72 | if readCloser, ok := file.(io.ReadCloser); ok { 73 | _ = readCloser.Close() 74 | } 75 | } 76 | } 77 | return 78 | } else if useWriter { 79 | var writer io.Writer 80 | if decompress { 81 | writer = brotli.NewDecompressionWriter( 82 | os.Stdout, 83 | ) 84 | } else { 85 | if options.NumThreads == 1 { 86 | writer = brotli.NewCompressionWriter( 87 | os.Stdout, 88 | options, 89 | ) 90 | } else { 91 | writer = brotli.NewMultiCompressionWriter( 92 | os.Stdout, 93 | options, 94 | ) 95 | } 96 | } 97 | for { 98 | var buffer [65536]byte 99 | count, err := os.Stdin.Read(buffer[:]) 100 | if err == io.EOF { 101 | break 102 | } 103 | if err != nil { 104 | panic(err) 105 | } 106 | _, err = writer.Write(buffer[:count]) 107 | if err != nil { 108 | panic(err) 109 | } 110 | } 111 | if writeCloser, ok := writer.(io.WriteCloser); ok { 112 | err := writeCloser.Close() 113 | if err != nil { 114 | panic(err) 115 | } 116 | } 117 | } else { 118 | var reader io.Reader 119 | if decompress { 120 | reader = brotli.NewDecompressionReader( 121 | os.Stdin, 122 | ) 123 | } else { 124 | if options.NumThreads == 1 { 125 | reader = brotli.NewCompressionReader( 126 | os.Stdin, 127 | options, 128 | ) 129 | } else { 130 | reader = brotli.NewMultiCompressionReader( 131 | os.Stdin, 132 | options, 133 | ) 134 | } 135 | } 136 | for { 137 | var buffer [65536]byte 138 | size, err := reader.Read(buffer[:]) 139 | _, werr := os.Stdout.Write(buffer[:size]) 140 | if werr != nil { 141 | panic(werr) 142 | } 143 | if err == io.EOF { 144 | return 145 | } 146 | if err != nil { 147 | panic(err) 148 | } 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /c/go/stress/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "syscall" 5 | // "io/ioutil" 6 | "time" 7 | "bytes" 8 | "compress/zlib" 9 | "crypto/md5" 10 | "io" 11 | "flag" 12 | "sync" 13 | 14 | "github.com/dropbox/rust-brotli/c/go/brotli" 15 | ) 16 | 17 | var useWriter = flag.Bool("use_writer", false, "should use writer for brotli") 18 | var parallelism = flag.Int("workers", 16, "number of workers to run with") 19 | var threads = flag.Int("threads", 2, "number of threads to compress the file") 20 | var iters = flag.Int("iter", 1024 * 1024, "number of times to run the compression/decompression cycle") 21 | var sizeHint = flag.Uint64("size", 4096 * 1024, "Size of the test data") 22 | var quality = flag.Float64("quality", 5, "brotli quality level") 23 | var useZlib = flag.Int("zlib", 0, "1 for zlib 2 for both zlib and brotli") 24 | var nongraceful = flag.Bool("hardexit", false, "use syscall.ExitGroup to end the program") 25 | var anongoroutines = flag.Bool("anon", false, "use a separate anonymous goroutine for each invocation") 26 | var timeout = flag.Duration("timeout", 0, "timeout until process exits with code 1") 27 | func main() { 28 | flag.Parse() 29 | if *timeout != 0 { 30 | go func() { 31 | time.Sleep(*timeout) 32 | syscall.Exit(1) 33 | }() 34 | } 35 | options := brotli.CompressionOptions{ 36 | NumThreads: *threads, 37 | Quality: float32(*quality), 38 | Catable: true, 39 | Appendable: true, 40 | Magic: true, 41 | SizeHint: uint32(*sizeHint), 42 | } 43 | file := testData(int(options.SizeHint)) 44 | if *anongoroutines { 45 | var wg sync.WaitGroup 46 | wg.Add(*parallelism) 47 | for par :=0; par < *parallelism; par += 1 { 48 | go func() { 49 | for iter := 0; iter < *iters; iter += 1 { 50 | _ = stresst(options, file, *useZlib % 2 == 0) 51 | if *useZlib == 2 { 52 | _ = stresst(options, file, false) 53 | } 54 | } 55 | if *nongraceful { 56 | syscall.Exit(0) 57 | } 58 | wg.Done() 59 | }() 60 | } 61 | wg.Wait() 62 | } else { 63 | for iter := 0; iter < *iters; iter += 1 { 64 | var wg sync.WaitGroup 65 | wg.Add(*parallelism) 66 | for par :=0; par < *parallelism; par += 1 { 67 | go func() { 68 | _ = stresst(options, file, *useZlib % 2 == 0) 69 | if *useZlib == 2 { 70 | _ = stresst(options, file, false) 71 | } 72 | wg.Done() 73 | }() 74 | } 75 | wg.Wait() 76 | } 77 | if *nongraceful { 78 | syscall.Exit(0) 79 | } 80 | } 81 | } 82 | 83 | func stresst( 84 | options brotli.CompressionOptions, 85 | file []byte, 86 | useBrotli bool, 87 | ) ( 88 | md5out [md5.Size]byte, 89 | ) { 90 | var output bytes.Buffer 91 | input := bytes.NewBuffer(file) 92 | var compressionWriter io.WriteCloser 93 | var err error 94 | if useBrotli { 95 | compressionWriter = brotli.NewMultiCompressionWriter( 96 | &output, options) 97 | } else { 98 | compressionWriter = zlib.NewWriter(&output) 99 | } 100 | if err != nil { 101 | panic(err) 102 | } 103 | _, err = io.Copy(compressionWriter, input) 104 | if err != nil { 105 | panic(err) 106 | } 107 | err = compressionWriter.Close() 108 | if err != nil { 109 | panic(err) 110 | } 111 | md5out = md5.Sum(output.Bytes()) 112 | compressed := bytes.NewBuffer(output.Bytes()) 113 | var compressionReader io.ReadCloser 114 | if useBrotli { 115 | compressionReader = brotli.NewDecompressionReader(compressed) 116 | } else { 117 | compressionReader, err = zlib.NewReader(compressed) 118 | } 119 | if err != nil { 120 | panic(err) 121 | } 122 | var rt bytes.Buffer 123 | _, err = io.Copy(&rt, compressionReader) 124 | if err != nil { 125 | panic(err) 126 | } 127 | err = compressionReader.Close() 128 | if err != nil { 129 | panic(err) 130 | } 131 | if !bytes.Equal(rt.Bytes(), file) { 132 | panic ("Files differ " + string(rt.Bytes())) 133 | } 134 | return 135 | } 136 | -------------------------------------------------------------------------------- /c/go/verify.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/dropbox/rust-brotli/c/go/brotli" 8 | "io" 9 | insecure_random "math/rand" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | func makeOptions() brotli.CompressionOptions { 15 | ret := brotli.CompressionOptions{ 16 | NumThreads: insecure_random.Intn(15) + 1, 17 | Quality: float32(insecure_random.Intn(9) + 3), 18 | Catable: true, 19 | Appendable: true, 20 | Magic: true, 21 | } 22 | if ret.Quality > 9 { 23 | ret.Quality = 9.5 24 | } 25 | return ret 26 | } 27 | func verifyReader(path string) { 28 | f, err := os.Open(path) 29 | if err != nil { 30 | return 31 | } 32 | defer func() { _ = f.Close() }() 33 | origFile := bytes.NewBuffer(nil) 34 | options := makeOptions() 35 | reader := brotli.NewMultiCompressionReader( 36 | io.TeeReader(f, origFile), 37 | options, 38 | ) 39 | compressed := bytes.NewBuffer(nil) 40 | rtReader := brotli.NewDecompressionReader(io.TeeReader(reader, compressed)) 41 | rt := bytes.NewBuffer(nil) 42 | io.Copy(rt, rtReader) 43 | if string(rt.Bytes()) != string(origFile.Bytes()) { 44 | fi, err := os.Create("/tmp/IN.orig") 45 | if err != nil { 46 | defer func() { _ = fi.Close() }() 47 | _, _ = fi.Write(origFile.Bytes()) 48 | } 49 | fc, err := os.Create("/tmp/IN.br") 50 | if err != nil { 51 | defer func() { _ = fc.Close() }() 52 | _, _ = fc.Write(compressed.Bytes()) 53 | } 54 | fr, err := os.Create("/tmp/IN.rt") 55 | if err != nil { 56 | defer func() { _ = fr.Close() }() 57 | _, _ = fr.Write(rt.Bytes()) 58 | } 59 | panic(fmt.Sprintf("%v Bytes mismatch %d != %d\n", options, len(rt.Bytes()), len(origFile.Bytes()))) 60 | } 61 | } 62 | 63 | func verifyWriter(path string) { 64 | f, err := os.Open(path) 65 | if err != nil { 66 | return 67 | } 68 | options := makeOptions() 69 | defer func() { _ = f.Close() }() 70 | compressed := bytes.NewBuffer(nil) 71 | rt := bytes.NewBuffer(nil) 72 | origFile := bytes.NewBuffer(nil) 73 | dwriter := brotli.NewDecompressionWriter(rt) 74 | writer := brotli.NewMultiCompressionWriter( 75 | io.MultiWriter(compressed, dwriter), 76 | options, 77 | ) 78 | _, err = io.Copy(io.MultiWriter(writer, origFile), f) 79 | if err != nil { 80 | panic(err) 81 | } 82 | err = writer.Close() 83 | if err != nil { 84 | panic(err) 85 | } 86 | err = dwriter.Close() 87 | if err != nil { 88 | panic(err) 89 | } 90 | if string(rt.Bytes()) != string(origFile.Bytes()) { 91 | fi, err := os.Create("/tmp/INW.orig") 92 | if err != nil { 93 | defer func() { _ = fi.Close() }() 94 | _, _ = fi.Write(origFile.Bytes()) 95 | } 96 | fc, err := os.Create("/tmp/INW.br") 97 | if err != nil { 98 | defer func() { _ = fc.Close() }() 99 | _, _ = fc.Write(compressed.Bytes()) 100 | } 101 | fr, err := os.Create("/tmp/INW.rt") 102 | if err != nil { 103 | defer func() { _ = fr.Close() }() 104 | _, _ = fr.Write(rt.Bytes()) 105 | } 106 | panic(fmt.Sprintf("%v Bytes mismatch %d != %d\n", options, len(rt.Bytes()), len(origFile.Bytes()))) 107 | } else { 108 | fmt.Fprintf(os.Stderr, "Processing %s %v/%v = %f\n", 109 | path, len(rt.Bytes()), len(compressed.Bytes()), float32(len(compressed.Bytes()))/float32(len(rt.Bytes()))) 110 | } 111 | } 112 | 113 | func recursiveVerify(root string) { 114 | filepath.Walk(root, filepath.WalkFunc(func(path string, info os.FileInfo, err error) error { 115 | if info.Size() > 1 && !info.IsDir() { 116 | func() { 117 | f, err := os.Open(path) 118 | if err != nil { 119 | return 120 | } 121 | defer func() { _ = f.Close() }() 122 | var test [64]byte 123 | sz, err := f.Read(test[:]) 124 | if sz == 0 || err != nil { 125 | return 126 | } 127 | fmt.Fprintf(os.Stderr, "Processing %v (%d)\n", path, info.Size()) 128 | verifyReader(path) 129 | verifyWriter(path) 130 | }() 131 | } 132 | return nil 133 | })) 134 | } 135 | -------------------------------------------------------------------------------- /c/py/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/c/py/__init__.py -------------------------------------------------------------------------------- /c/py/brotli_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | from .brotli import * 4 | from .testdata import * 5 | class TestBrotliLibrary(unittest.TestCase): 6 | def setUp(self): 7 | self.test_data = make_test_data(4096 * 1024) 8 | def test_version(self): 9 | assert BrotliDecoderVersion() 10 | assert BrotliEncoderVersion() 11 | 12 | def test_wp_rt(self): 13 | wp = BrotliEncoderCreateWorkPool(8) 14 | output = BrotliEncoderCompressWorkPool(wp, 15 | self.test_data, 16 | { 17 | BROTLI_PARAM_QUALITY:5, 18 | BROTLI_PARAM_CATABLE:1, 19 | BROTLI_PARAM_MAGIC_NUMBER:1, 20 | }, 21 | 8) 22 | BrotliEncoderDestroyWorkPool(wp) 23 | rt = BrotliDecode(output) 24 | assert rt == self.test_data 25 | assert len(output) < 1024 * 1024 26 | 27 | def test_header(self): 28 | data = ''.join(chr(x) for x in [ 29 | 0x6b, 0x1d, 0x00, 0xe1, 0x97, 0x81, 0x01, 0xe8, 0x99, 0xf4, 0x01, 0x08, 30 | 0x00, 0x08, 0x79, 0x0a, 0x2c, 0x67, 0xe8, 0x81, 0x5f, 0x22, 0x2f, 0x1e, 31 | 0x8b, 0x08, 0x3e, 0x09, 0x7a, 0x06 32 | ]) 33 | parsed_header = BrotliParseHeader(data) 34 | assert parsed_header 35 | version, size = parsed_header 36 | assert size == 4001000 37 | assert version == 1 38 | 39 | def test_rt(self): 40 | output = BrotliCompress(self.test_data, 41 | { 42 | BROTLI_PARAM_QUALITY:5, 43 | BROTLI_PARAM_CATABLE:1, 44 | BROTLI_PARAM_MAGIC_NUMBER:1, 45 | }, 46 | 8) 47 | rt = BrotliDecode(output) 48 | assert rt == self.test_data 49 | assert len(output) < 1024 * 1024 50 | def test_tiny_alloc(self): 51 | output = BrotliCompress(self.test_data, 52 | { 53 | BROTLI_PARAM_QUALITY:5, 54 | BROTLI_PARAM_CATABLE:1, 55 | BROTLI_PARAM_MAGIC_NUMBER:1, 56 | }, 57 | 8) 58 | rt = BrotliDecode(output, 2) 59 | assert rt == self.test_data 60 | assert len(output) < 1024 * 1024 61 | def test_single_thread(self): 62 | output = BrotliCompress(self.test_data, 63 | { 64 | BROTLI_PARAM_QUALITY:5, 65 | BROTLI_PARAM_CATABLE:1, 66 | BROTLI_PARAM_MAGIC_NUMBER:1, 67 | }, 68 | 1) 69 | rt = BrotliDecode(output, 2) 70 | assert rt == self.test_data 71 | assert len(output) < 1024 * 1024 72 | def test_memory_view(self): 73 | output = BrotliCompress(memoryview(self.test_data), 74 | { 75 | BROTLI_PARAM_QUALITY:5, 76 | BROTLI_PARAM_CATABLE:1, 77 | BROTLI_PARAM_MAGIC_NUMBER:1, 78 | }, 79 | 8) 80 | rt = BrotliDecode(output) 81 | assert rt == self.test_data 82 | assert len(output) < 1024 * 1024 83 | def test_1(self): 84 | output = BrotliCompress(self.test_data[:65536], 85 | { 86 | BROTLI_PARAM_QUALITY:11, 87 | BROTLI_PARAM_CATABLE:1, 88 | BROTLI_PARAM_MAGIC_NUMBER:1, 89 | }, 90 | 8) 91 | rt = BrotliDecode(output) 92 | assert rt == self.test_data[:65536] 93 | assert len(output) < 1024 * 1024 94 | def test_rnd(self): 95 | random_data = os.urandom(131072) 96 | wp = BrotliEncoderCreateWorkPool(8) 97 | output = BrotliEncoderCompressWorkPool(wp, 98 | random_data, 99 | { 100 | BROTLI_PARAM_QUALITY:7, 101 | BROTLI_PARAM_CATABLE:1, 102 | BROTLI_PARAM_MAGIC_NUMBER:1, 103 | }, 104 | 8) 105 | BrotliEncoderDestroyWorkPool(wp) 106 | rt = BrotliDecode(output) 107 | assert rt == random_data 108 | assert len(output) > 130000 109 | def test_truncation(self): 110 | output = BrotliCompress(self.test_data[:65536], 111 | { 112 | BROTLI_PARAM_QUALITY:6, 113 | BROTLI_PARAM_CATABLE:1, 114 | BROTLI_PARAM_MAGIC_NUMBER:1, 115 | }, 116 | 8) 117 | corrupt = output[:len(output) - 1] 118 | rt = BrotliDecode(output) 119 | assert rt == self.test_data[:65536] 120 | assert len(output) < 1024 * 1024 121 | try: 122 | BrotliDecode(corrupt) 123 | except BrotliDecompressorException: 124 | pass 125 | else: 126 | assert False, "Should have errored" 127 | def test_corruption(self): 128 | output = BrotliCompress(self.test_data[:65536], 129 | { 130 | BROTLI_PARAM_QUALITY:6, 131 | BROTLI_PARAM_CATABLE:1, 132 | BROTLI_PARAM_MAGIC_NUMBER:1, 133 | }, 134 | 8) 135 | corrupt = output[:len(output)/2] + output[len(output)/2 + 1:] 136 | rt = BrotliDecode(output) 137 | assert rt == self.test_data[:65536] 138 | assert len(output) < 1024 * 1024 139 | try: 140 | BrotliDecode(corrupt) 141 | except BrotliDecompressorException: 142 | pass 143 | else: 144 | assert False, "Should have errored" 145 | if __name__ == '__main__': 146 | unittest.main() 147 | 148 | -------------------------------------------------------------------------------- /c/py/testdata.py: -------------------------------------------------------------------------------- 1 | def make_test_data(bufsize): 2 | poem = b"""Once upon a midnight dreary, while I pondered, weak and weary, 3 | Over many a quaint and curious volume of forgotten lore- 4 | While I nodded, nearly napping, suddenly there came a tapping, 5 | As of some one gently rapping, rapping at my chamber door. 6 | "'Tis some visitor," I muttered, "tapping at my chamber door- 7 | Only this and nothing more." 8 | 9 | Ah, distinctly I remember it was in the bleak December; 10 | And each separate dying ember wrought its ghost upon the floor. 11 | Eagerly I wished the morrow;-vainly I had sought to borrow 12 | From my books surcease of sorrow-sorrow for the lost Lenore- 13 | For the rare and radiant maiden whom the angels name Lenore- 14 | Nameless here for evermore. 15 | 16 | And the silken, sad, uncertain rustling of each purple curtain 17 | Thrilled me-filled me with fantastic terrors never felt before; 18 | So that now, to still the beating of my heart, I stood repeating 19 | "'Tis some visitor entreating entrance at my chamber door- 20 | Some late visitor entreating entrance at my chamber door;- 21 | This it is and nothing more." 22 | 23 | Presently my soul grew stronger; hesitating then no longer, 24 | "Sir," said I, "or Madam, truly your forgiveness I implore; 25 | But the fact is I was napping, and so gently you came rapping, 26 | And so faintly you came tapping, tapping at my chamber door, 27 | That I scarce was sure I heard you"-here I opened wide the door;- 28 | Darkness there and nothing more. 29 | 30 | Deep into that darkness peering, long I stood there wondering, fearing, 31 | Doubting, dreaming dreams no mortal ever dared to dream before; 32 | But the silence was unbroken, and the stillness gave no token, 33 | And the only word there spoken was the whispered word, "Lenore?" 34 | This I whispered, and an echo murmured back the word, "Lenore!"- 35 | Merely this and nothing more. 36 | 37 | Back into the chamber turning, all my soul within me burning, 38 | Soon again I heard a tapping somewhat louder than before. 39 | "Surely," said I, "surely that is something at my window lattice; 40 | Let me see, then, what thereat is, and this mystery explore- 41 | Let my heart be still a moment and this mystery explore;- 42 | 'Tis the wind and nothing more!" 43 | 44 | Open here I flung the shutter, when, with many a flirt and flutter, 45 | In there stepped a stately Raven of the saintly days of yore; 46 | Not the least obeisance made he; not a minute stopped or stayed he; 47 | But, with mien of lord or lady, perched above my chamber door- 48 | Perched upon a bust of Pallas just above my chamber door- 49 | Perched, and sat, and nothing more. 50 | 51 | Then this ebony bird beguiling my sad fancy into smiling, 52 | By the grave and stern decorum of the countenance it wore, 53 | "Though thy crest be shorn and shaven, thou," I said, "art sure no craven, 54 | Ghastly grim and ancient Raven wandering from the Nightly shore- 55 | Tell me what thy lordly name is on the Night's Plutonian shore!" 56 | Quoth the Raven "Nevermore." 57 | 58 | Much I marvelled this ungainly fowl to hear discourse so plainly, 59 | Though its answer little meaning-little relevancy bore; 60 | For we cannot help agreeing that no living human being 61 | Ever yet was blessed with seeing bird above his chamber door- 62 | Bird or beast upon the sculptured bust above his chamber door, 63 | With such name as "Nevermore." 64 | 65 | But the Raven, sitting lonely on the placid bust, spoke only 66 | That one word, as if his soul in that one word he did outpour. 67 | Nothing farther then he uttered-not a feather then he fluttered- 68 | Till I scarcely more than muttered "Other friends have flown before- 69 | On the morrow he will leave me, as my Hopes have flown before." 70 | Then the bird said "Nevermore." 71 | 72 | Startled at the stillness broken by reply so aptly spoken, 73 | "Doubtless," said I, "what it utters is its only stock and store 74 | Caught from some unhappy master whom unmerciful Disaster 75 | Followed fast and followed faster till his songs one burden bore- 76 | Till the dirges of his Hope that melancholy burden bore 77 | Of 'Never-nevermore'." 78 | 79 | But the Raven still beguiling all my fancy into smiling, 80 | Straight I wheeled a cushioned seat in front of bird, and bust and door; 81 | Then, upon the velvet sinking, I betook myself to linking 82 | Fancy unto fancy, thinking what this ominous bird of yore- 83 | What this grim, ungainly, ghastly, gaunt, and ominous bird of yore 84 | Meant in croaking "Nevermore." 85 | 86 | This I sat engaged in guessing, but no syllable expressing 87 | To the fowl whose fiery eyes now burned into my bosom's core; 88 | This and more I sat divining, with my head at ease reclining 89 | On the cushion's velvet lining that the lamp-light gloated o'er, 90 | But whose velvet-violet lining with the lamp-light gloating o'er, 91 | She shall press, ah, nevermore! 92 | 93 | Then, methought, the air grew denser, perfumed from an unseen censer 94 | Swung by Seraphim whose foot-falls tinkled on the tufted floor. 95 | "Wretch," I cried, "thy God hath lent thee-by these angels he hath sent thee 96 | Respite-respite and nepenthe from thy memories of Lenore; 97 | Quaff, oh quaff this kind nepenthe and forget this lost Lenore!" 98 | Quoth the Raven "Nevermore." 99 | 100 | "Prophet!" said I, "thing of evil!-prophet still, if bird or devil!- 101 | Whether Tempter sent, or whether tempest tossed thee here ashore, 102 | Desolate yet all undaunted, on this desert land enchanted- 103 | On this home by Horror haunted-tell me truly, I implore- 104 | Is there-is there balm in Gilead?-tell me-tell me, I implore!" 105 | Quoth the Raven "Nevermore." 106 | 107 | "Prophet!" said I, "thing of evil!-prophet still, if bird or devil! 108 | By that Heaven that bends above us-by that God we both adore- 109 | Tell this soul with sorrow laden if, within the distant Aidenn, 110 | It shall clasp a sainted maiden whom the angels name Lenore- 111 | Clasp a rare and radiant maiden whom the angels name Lenore." 112 | Quoth the Raven "Nevermore." 113 | 114 | "Be that word our sign of parting, bird or fiend!" I shrieked, upstarting- 115 | "Get thee back into the tempest and the Night's Plutonian shore! 116 | Leave no black plume as a token of that lie thy soul hath spoken! 117 | Leave my loneliness unbroken!-quit the bust above my door! 118 | Take thy beak from out my heart, and take thy form from off my door!" 119 | Quoth the Raven "Nevermore." 120 | 121 | And the Raven, never flitting, still is sitting, still is sitting 122 | On the pallid bust of Pallas just above my chamber door; 123 | And his eyes have all the seeming of a demon's that is dreaming, 124 | And the lamp-light o'er him streaming throws his shadow on the floor; 125 | And my soul from out that shadow that lies floating on the floor 126 | Shall be lifted-nevermore!""" 127 | ret = [] 128 | for index in range(0, bufsize, len(poem)): 129 | ret.append(''.join(chr((ord(x) + index)&255) for x in poem)) 130 | return ''.join(ret)[:bufsize] 131 | 132 | -------------------------------------------------------------------------------- /c/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![cfg_attr(not(feature = "std"), feature(lang_items))] 3 | 4 | pub extern crate brotli; 5 | #[cfg(feature = "std")] 6 | extern crate std; 7 | 8 | use core::ptr::null_mut; 9 | 10 | pub use brotli::ffi::compressor::*; 11 | // FIXME: this is just wrong, and needs to be fixed 12 | #[allow(unknown_lints, ambiguous_glob_reexports)] 13 | pub use brotli::ffi::decompressor::*; 14 | pub use brotli::ffi::multicompress::*; 15 | pub use brotli::*; 16 | 17 | #[cfg(feature = "std")] 18 | unsafe fn std_only_functions() { 19 | let _ = 20 | brotli::ffi::decompressor::CBrotliDecoderDecompress(0, null_mut(), null_mut(), null_mut()); 21 | } 22 | #[cfg(not(feature = "std"))] 23 | unsafe fn std_only_functions() {} 24 | 25 | #[no_mangle] 26 | pub unsafe extern "C" fn instantiate_functions(must_be_null: *const u8) { 27 | if !must_be_null.is_null() { 28 | let _ = brotli::ffi::compressor::BrotliEncoderVersion(); 29 | let _ = brotli::ffi::decompressor::CBrotliDecoderCreateInstance(None, None, null_mut()); 30 | let _ = brotli::ffi::decompressor::CBrotliDecoderSetParameter(null_mut(), brotli::ffi::decompressor::ffi::interface::BrotliDecoderParameter::BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, 0); 31 | let _ = brotli::ffi::decompressor::CBrotliDecoderDecompressStream( 32 | null_mut(), 33 | null_mut(), 34 | null_mut(), 35 | null_mut(), 36 | null_mut(), 37 | null_mut(), 38 | ); 39 | std_only_functions(); 40 | let _ = brotli::ffi::decompressor::CBrotliDecoderMallocU8(null_mut(), 0); 41 | let _ = brotli::ffi::decompressor::CBrotliDecoderMallocUsize(null_mut(), 0); 42 | let _ = brotli::ffi::decompressor::CBrotliDecoderFreeU8(null_mut(), null_mut(), 0); 43 | let _ = brotli::ffi::decompressor::CBrotliDecoderFreeUsize(null_mut(), null_mut(), 0); 44 | let _ = brotli::ffi::decompressor::CBrotliDecoderDestroyInstance(null_mut()); 45 | let _ = brotli::ffi::decompressor::CBrotliDecoderHasMoreOutput(null_mut()); 46 | let _ = brotli::ffi::decompressor::CBrotliDecoderTakeOutput(null_mut(), null_mut()); 47 | let _ = brotli::ffi::decompressor::CBrotliDecoderIsUsed(null_mut()); 48 | let _ = brotli::ffi::decompressor::CBrotliDecoderIsFinished(null_mut()); 49 | let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorCode(null_mut()); 50 | let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorString(null_mut()); 51 | let _ = brotli::ffi::decompressor::CBrotliDecoderErrorString( 52 | brotli::ffi::decompressor::ffi::BrotliDecoderErrorCode::BROTLI_DECODER_ERROR_UNREACHABLE); 53 | let _ = BrotliEncoderCreateInstance(None, None, null_mut()); 54 | let _ = BrotliEncoderSetParameter( 55 | null_mut(), 56 | brotli::enc::encode::BrotliEncoderParameter::BROTLI_PARAM_MODE, 57 | 0, 58 | ); 59 | let _ = BrotliEncoderDestroyInstance(null_mut()); 60 | let _ = BrotliEncoderIsFinished(null_mut()); 61 | let _ = BrotliEncoderHasMoreOutput(null_mut()); 62 | let _ = BrotliEncoderTakeOutput(null_mut(), null_mut()); 63 | let _ = BrotliEncoderMaxCompressedSize(0); 64 | let _ = BrotliEncoderSetCustomDictionary(null_mut(), 0, null_mut()); 65 | let _ = BrotliEncoderCompress( 66 | 0, 67 | 0, 68 | BrotliEncoderMode::BROTLI_MODE_GENERIC, 69 | 0, 70 | null_mut(), 71 | null_mut(), 72 | null_mut(), 73 | ); 74 | let _ = BrotliEncoderCompressStream( 75 | null_mut(), 76 | BrotliEncoderOperation::BROTLI_OPERATION_FINISH, 77 | null_mut(), 78 | null_mut(), 79 | null_mut(), 80 | null_mut(), 81 | null_mut(), 82 | ); 83 | let _ = BrotliEncoderMallocU8(null_mut(), 0); 84 | let _ = BrotliEncoderFreeU8(null_mut(), null_mut(), 0); 85 | let _ = BrotliEncoderMallocUsize(null_mut(), 0); 86 | let _ = BrotliEncoderFreeUsize(null_mut(), null_mut(), 0); 87 | let _ = BrotliEncoderMaxCompressedSizeMulti(0, 0); 88 | let _ = BrotliEncoderCompressMulti( 89 | 0, 90 | null_mut(), 91 | null_mut(), 92 | 0, 93 | null_mut(), 94 | null_mut(), 95 | null_mut(), 96 | 0, 97 | None, 98 | None, 99 | null_mut(), 100 | ); 101 | let _ = BrotliEncoderCreateWorkPool(0, None, None, null_mut()); 102 | let _ = BrotliEncoderDestroyWorkPool(null_mut()); 103 | let _ = BrotliEncoderCompressWorkPool( 104 | null_mut(), 105 | 0, 106 | null_mut(), 107 | null_mut(), 108 | 0, 109 | null_mut(), 110 | null_mut(), 111 | null_mut(), 112 | 0, 113 | None, 114 | None, 115 | null_mut(), 116 | ); 117 | } 118 | } 119 | 120 | #[cfg(not(feature = "std"))] 121 | #[panic_handler] 122 | extern "C" fn panic_impl(_: &::core::panic::PanicInfo) -> ! { 123 | loop {} 124 | } 125 | 126 | #[cfg(not(feature = "std"))] 127 | #[lang = "eh_personality"] 128 | extern "C" fn eh_personality() {} 129 | -------------------------------------------------------------------------------- /c/vec_u8.h: -------------------------------------------------------------------------------- 1 | struct VecU8 { 2 | unsigned char *data; 3 | size_t size; 4 | }; 5 | struct VecU8 new_vec_u8() { 6 | struct VecU8 ret; 7 | ret.data = NULL; 8 | ret.size = 0; 9 | return ret; 10 | } 11 | uint64_t round_up_to_power_of_two(uint64_t v) { 12 | v--; 13 | v |= v >> 1; 14 | v |= v >> 2; 15 | v |= v >> 4; 16 | v |= v >> 8; 17 | v |= v >> 16; 18 | { 19 | uint64_t tmp = v; 20 | tmp >>= 32; 21 | v |= tmp; 22 | } 23 | v++; 24 | return v; 25 | } 26 | 27 | void push_vec_u8(struct VecU8 *thus, const unsigned char*data, size_t size) { 28 | size_t new_actual_size = thus->size + size; 29 | if (size == 0 || new_actual_size < thus->size) { 30 | return; 31 | } 32 | { 33 | size_t new_alloc_size = round_up_to_power_of_two(new_actual_size); 34 | size_t old_alloc_size = round_up_to_power_of_two(thus->size); 35 | if (thus->size == 0 || old_alloc_size != new_alloc_size ) { 36 | unsigned char *tmp = custom_malloc_f(custom_alloc_opaque, new_alloc_size); 37 | size_t to_copy = old_alloc_size; 38 | if (new_alloc_size < old_alloc_size) { 39 | to_copy = new_alloc_size; 40 | } 41 | memcpy(tmp, thus->data, to_copy); 42 | custom_free_f(custom_alloc_opaque, thus->data); 43 | thus->data = tmp; 44 | } 45 | if (new_alloc_size < new_actual_size) { 46 | abort(); // assert 47 | } 48 | memcpy(thus->data + thus->size, data, size); 49 | thus->size = new_actual_size; 50 | } 51 | } 52 | 53 | void release_vec_u8(struct VecU8 *thus) { 54 | if (thus->size) { 55 | custom_free_f(custom_alloc_opaque, thus->data); 56 | thus->size = 0; 57 | thus->data = NULL; 58 | } 59 | } 60 | 61 | 62 | void trunc_vec_u8(struct VecU8 *thus, size_t new_len) { 63 | if (thus->size > new_len) { 64 | thus->size = new_len; 65 | } 66 | } 67 | void reinit_vec_u8(struct VecU8 *thus, size_t new_len) { 68 | release_vec_u8(thus); 69 | thus->data = custom_malloc_f(custom_alloc_opaque, new_len); 70 | thus->size = new_len; 71 | } 72 | -------------------------------------------------------------------------------- /ci/before_deploy.ps1: -------------------------------------------------------------------------------- 1 | # This script takes care of packaging the build artifacts that will go in the 2 | # release zipfile 3 | 4 | $SRC_DIR = $PWD.Path 5 | $STAGE = [System.Guid]::NewGuid().ToString() 6 | 7 | Set-Location $ENV:Temp 8 | New-Item -Type Directory -Name $STAGE 9 | Set-Location $STAGE 10 | 11 | $ZIP = "$SRC_DIR\$($Env:CRATE_NAME)-$($Env:APPVEYOR_REPO_TAG_NAME)-$($Env:TARGET).zip" 12 | 13 | Copy-Item "$SRC_DIR\target\release\brotli.exe" '.\' 14 | 15 | 7z a "$ZIP" * 16 | 17 | Push-AppveyorArtifact "$ZIP" 18 | 19 | Remove-Item *.* -Force 20 | Set-Location .. 21 | Remove-Item $STAGE 22 | Set-Location $SRC_DIR 23 | -------------------------------------------------------------------------------- /ci/before_deploy.sh: -------------------------------------------------------------------------------- 1 | # This script takes care of building your crate and packaging it for release 2 | 3 | set -ex 4 | 5 | main() { 6 | local src=$(pwd) \ 7 | stage= 8 | 9 | case $TRAVIS_OS_NAME in 10 | linux) 11 | stage=$(mktemp -d) 12 | ;; 13 | osx) 14 | stage=$(mktemp -d -t tmp) 15 | ;; 16 | esac 17 | 18 | test -f Cargo.lock || cargo generate-lockfile 19 | 20 | # TODO Update this to build the artifacts that matter to you 21 | cross rustc --release --features=validation 22 | 23 | # TODO Update this to package the right artifacts 24 | cp target/$TARGET/brotli $stage/ 25 | 26 | cd $stage 27 | tar czf $src/$CRATE_NAME-$TRAVIS_TAG-$TARGET.tar.gz * 28 | cd $src 29 | 30 | rm -rf $stage 31 | } 32 | 33 | main 34 | -------------------------------------------------------------------------------- /ci/install.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | main() { 4 | local target= 5 | if [ $TRAVIS_OS_NAME = linux ]; then 6 | target=x86_64-unknown-linux-musl 7 | sort=sort 8 | else 9 | target=x86_64-apple-darwin 10 | sort=gsort # for `sort --sort-version`, from brew's coreutils. 11 | fi 12 | 13 | # Builds for iOS are done on OSX, but require the specific target to be 14 | # installed. 15 | case $TARGET in 16 | aarch64-apple-ios) 17 | rustup target install aarch64-apple-ios 18 | ;; 19 | armv7-apple-ios) 20 | rustup target install armv7-apple-ios 21 | ;; 22 | armv7s-apple-ios) 23 | rustup target install armv7s-apple-ios 24 | ;; 25 | i386-apple-ios) 26 | rustup target install i386-apple-ios 27 | ;; 28 | x86_64-apple-ios) 29 | rustup target install x86_64-apple-ios 30 | ;; 31 | esac 32 | 33 | # This fetches latest stable release 34 | local tag=$(git ls-remote --tags --refs --exit-code https://github.com/japaric/cross \ 35 | | cut -d/ -f3 \ 36 | | grep -E '^v[0.1.0-9.]+$' \ 37 | | $sort --version-sort \ 38 | | tail -n1) 39 | curl -LSfs https://japaric.github.io/trust/install.sh | \ 40 | sh -s -- \ 41 | --force \ 42 | --git japaric/cross \ 43 | --tag $tag \ 44 | --target $target 45 | } 46 | 47 | main 48 | -------------------------------------------------------------------------------- /ci/script.sh: -------------------------------------------------------------------------------- 1 | # This script takes care of testing your crate 2 | 3 | set -ex 4 | 5 | # TODO This is the "test phase", tweak it as you see fit 6 | main() { 7 | cross build --target $TARGET 8 | cross build --target $TARGET --release 9 | 10 | if [ ! -z $DISABLE_TESTS ]; then 11 | return 12 | fi 13 | 14 | cross test --target $TARGET 15 | cross test --target $TARGET --release 16 | 17 | cross test --target $TARGET --release --features=no-stdlib 18 | } 19 | 20 | # we don't run the "test phase" when doing deploys 21 | if [ -z $TRAVIS_TAG ]; then 22 | main 23 | fi 24 | -------------------------------------------------------------------------------- /examples/compress.rs: -------------------------------------------------------------------------------- 1 | extern crate brotli; 2 | #[cfg(not(feature = "std"))] 3 | fn main() { 4 | panic!("For no-stdlib examples please see the tests") 5 | } 6 | #[cfg(feature = "std")] 7 | fn main() { 8 | use std::io; 9 | use std::io::{Read, Write}; 10 | let stdout = &mut io::stdout(); 11 | { 12 | let mut writer = brotli::CompressorWriter::new(stdout, 4096, 11, 22); 13 | let mut buf = [0u8; 4096]; 14 | loop { 15 | match io::stdin().read(&mut buf[..]) { 16 | Err(e) => { 17 | if let io::ErrorKind::Interrupted = e.kind() { 18 | continue; 19 | } 20 | panic!("{}", e); 21 | } 22 | Ok(size) => { 23 | if size == 0 { 24 | match writer.flush() { 25 | Err(e) => { 26 | if let io::ErrorKind::Interrupted = e.kind() { 27 | continue; 28 | } 29 | panic!("{}", e) 30 | } 31 | Ok(_) => break, 32 | } 33 | } 34 | match writer.write_all(&buf[..size]) { 35 | Err(e) => panic!("{}", e), 36 | Ok(_) => {} 37 | } 38 | } 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /examples/decompress.rs: -------------------------------------------------------------------------------- 1 | extern crate brotli; 2 | #[cfg(not(feature = "std"))] 3 | fn main() { 4 | panic!("For no-stdlib examples please see the tests") 5 | } 6 | #[cfg(feature = "std")] 7 | fn main() { 8 | use std::io; 9 | let stdin = &mut io::stdin(); 10 | { 11 | use std::io::{Read, Write}; 12 | let mut reader = brotli::Decompressor::new( 13 | stdin, 4096, // buffer size 14 | ); 15 | let mut buf = [0u8; 4096]; 16 | loop { 17 | match reader.read(&mut buf[..]) { 18 | Err(e) => { 19 | if let io::ErrorKind::Interrupted = e.kind() { 20 | continue; 21 | } 22 | panic!("{}", e); 23 | } 24 | Ok(size) => { 25 | if size == 0 { 26 | break; 27 | } 28 | match io::stdout().write_all(&buf[..size]) { 29 | Err(e) => panic!("{}", e), 30 | Ok(_) => {} 31 | } 32 | } 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env just --justfile 2 | 3 | @_default: 4 | just --list --unsorted 5 | 6 | # Clean all build artifacts 7 | clean: 8 | cargo clean 9 | 10 | # Build everything 11 | build: build-brotli build-simd build-ffi 12 | 13 | # Build the main crate 14 | build-brotli: 15 | RUSTFLAGS='-D warnings' cargo build --workspace --all-targets --bins --tests --lib --benches --examples 16 | 17 | # Build simd with nightly 18 | build-simd: 19 | RUSTFLAGS='-D warnings' cargo +nightly build --features simd 20 | 21 | # Build the brotli-ffi crate (in ./c dir) 22 | build-ffi: 23 | # TODO: The c/Cargo.toml does not depend on the **unpublished** main crate, so its build never actually gets tested 24 | RUSTFLAGS='-D warnings' cargo build --features ffi-api 25 | RUSTFLAGS='-D warnings' cargo build --workspace --all-targets --bins --tests --lib --benches --examples --manifest-path c/Cargo.toml 26 | # For now, use original make file for building/testing the FFI crate 27 | cd c && make 28 | 29 | # Run cargo fmt with optional params 30 | fmt *ARGS: 31 | cargo fmt --all -- {{ ARGS }} 32 | cd c && cargo fmt --all -- {{ ARGS }} 33 | 34 | # Run Nightly cargo fmt, ordering imports by groups 35 | fmt2: 36 | cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate 37 | cd c && cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate 38 | 39 | # Run cargo clippy 40 | clippy: 41 | cargo clippy -- -D warnings 42 | cargo clippy --workspace --all-targets --bins --tests --lib --benches --examples -- -D warnings 43 | cd c && cargo clippy -- -D warnings 44 | cd c && cargo clippy --workspace --all-targets --bins --tests --lib --benches --examples -- -D warnings 45 | 46 | # Build and open code documentation 47 | docs: 48 | cargo doc --no-deps --open 49 | cd c && cargo doc --no-deps --open 50 | 51 | # Test documentation 52 | test-doc: 53 | cargo test --doc 54 | RUSTDOCFLAGS="-D warnings" cargo doc --no-deps 55 | cd c && cargo test --doc 56 | cd c && RUSTDOCFLAGS="-D warnings" cargo doc --no-deps 57 | 58 | # Test using cargo test with optional params 59 | test *ARGS: 60 | cargo test {{ ARGS }} 61 | cd c && cargo test {{ ARGS }} 62 | 63 | # Report current versions of rustc, cargo, and other utils 64 | sys-info: 65 | rustc --version 66 | cargo --version 67 | {{ just_executable() }} --version 68 | 69 | # Get MSRV (Minimum Supported Rust Version) for the brotli crate 70 | read-msrv: 71 | cargo metadata --no-deps --format-version 1 | jq -r -e '.packages[] | select(.name == "brotli").rust_version' 72 | 73 | # All tests to run for CI (TODO: add clippy) 74 | ci-test: sys-info (fmt "--check") build test test-doc 75 | 76 | # All stable tests to run for CI with the earliest supported Rust version. Assumes the Rust version is already set by rustup. 77 | ci-test-msrv: sys-info build-brotli build-ffi test 78 | 79 | # Test if changes are backwards compatible (patch), or need a new minor/major version 80 | semver-checks: 81 | cargo semver-checks 82 | -------------------------------------------------------------------------------- /research/concatenate_some.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import random as insecure_random 4 | import subprocess 5 | import hashlib 6 | import traceback 7 | 8 | def get_goal(): 9 | ret = insecure_random.randrange(1,16) 10 | if ret == 15: 11 | ret = insecure_random.randrange(15,64) 12 | return ret 13 | 14 | def shasum(work): 15 | sha = hashlib.sha256() 16 | for file in work: 17 | with open(file) as ff: 18 | while True: 19 | r = ff.read(65536) 20 | if r: 21 | sha.update(r) 22 | else: 23 | break 24 | return sha.digest() 25 | def file_size(filename): 26 | return os.stat(filename).st_size 27 | uncatable_stats = 0 28 | catable_stats = 0 29 | def process(work, brotli, cat, prefix): 30 | sys.stdout.flush() 31 | global uncatable_stats 32 | global catable_stats 33 | work = [filename for filename in work if filename.find(".jpg") != -1 or filename.find(".avi") != -1 or filename.find(".mp4") != -1 or filename.find(".mov") != -1 or insecure_random.randrange(1,17) == 16] 34 | if len(work) == 0: 35 | return 36 | try: 37 | fullsum = shasum(work) 38 | except Exception: 39 | traceback.print_exc() 40 | print 'ok early exit' 41 | return 42 | quality = "-q" + str(insecure_random.randrange(2,12 if len(work) < 8 else 10)) 43 | if insecure_random.randrange(0,16) == 0: 44 | quality = '-q9.5' 45 | append = insecure_random.randrange(0,2) == 0 46 | frivolous_procs = [] 47 | procs = [] 48 | print 'processing',work,'at',quality,append 49 | for index, filename in enumerate(work): 50 | thread_count = insecure_random.randrange(1,17) 51 | if quality > 8 and thread_count == 1: 52 | thread_count = 2 53 | magic = insecure_random.randrange(0,2) == 0 54 | buffer_size = insecure_random.randrange(1,18) 55 | if buffer_size == 17: 56 | buffer_size = 65536 57 | elif buffer_size > 3: 58 | buffer_size *= 1024 59 | bs = "-bs" + str(buffer_size) 60 | args = [brotli, "-c", bs, quality] 61 | if magic: 62 | args.append("-magic") 63 | frivolous_procs.append(subprocess.Popen(args + [filename, prefix + quality+"-" + str(index)+".compressed"])) 64 | if append and index == 0: 65 | args.append("-appendable") 66 | else: 67 | args.append("-catable") 68 | args.append('-j' + str(thread_count)) 69 | args.append(filename) 70 | args.append(prefix + quality+"-" + str(index)+".br") 71 | procs.append(subprocess.Popen(args)) 72 | for index, proc in enumerate(procs): 73 | ret = proc.wait() 74 | if ret: 75 | print 'failure at ' + work[index],quality,append, bs 76 | assert not ret 77 | buffer_size = insecure_random.randrange(1,18) 78 | if buffer_size == 17: 79 | buffer_size = 65536 80 | elif buffer_size > 3: 81 | buffer_size *= 1024 82 | bs = "-bs" + str(buffer_size) 83 | args = [cat, bs] 84 | for index, filename in enumerate(work): 85 | args.append(prefix + quality+"-" + str(index)+".br") 86 | stdout, _stderr = subprocess.Popen(args, stdout=subprocess.PIPE).communicate() 87 | with open(prefix+".br", 'w') as f: 88 | f.write(stdout) 89 | procs[0] = subprocess.Popen([brotli, prefix +'.br', prefix]) 90 | ret = procs[0].wait() 91 | if ret: 92 | print 'failure at ',work,quality,append 93 | assert not ret 94 | rtsum = shasum([prefix]) 95 | if rtsum != fullsum: 96 | print 'failure at ',work,quality,append 97 | assert rtsum == fullsum 98 | print 'ok',rtsum.encode('hex') 99 | for (index,proc) in enumerate(frivolous_procs): 100 | ret = proc.wait() 101 | assert not ret 102 | uncatable_stats += file_size(prefix + quality+"-" + str(index)+".compressed") 103 | 104 | catable_stats += len(stdout) 105 | print uncatable_stats,'/',catable_stats,(10000*uncatable_stats/catable_stats)/100.0 106 | try: 107 | for index, filename in enumerate(work): 108 | os.remove(prefix + quality+"-" + str(index)+".br") 109 | os.remove(prefix + quality+"-" + str(index)+".compressed") 110 | os.remove(prefix + ".br") 111 | os.remove(prefix) 112 | except Exception: 113 | traceback.print_exc() 114 | 115 | def main(): 116 | start = sys.argv[1] 117 | brotli = sys.argv[2] 118 | catbrotli = sys.argv[3] 119 | goal = get_goal() 120 | work =[] 121 | prefix="/tmp/cat-" + os.urandom(16).encode('hex') 122 | for root, dirnames, filenames in os.walk(start): 123 | for filename in filenames: 124 | try: 125 | if file_size(os.path.join(root,filename)): 126 | work.append(os.path.join(root,filename)) 127 | except Exception: 128 | continue 129 | if len(work) >= goal: 130 | goal = get_goal() 131 | process(work, brotli, catbrotli, prefix) 132 | work = [] 133 | if len(work): 134 | process(work, brotli, catbrotli, prefix) 135 | if __name__=="__main__": 136 | main() 137 | 138 | -------------------------------------------------------------------------------- /research/frombase2.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | result = [] 4 | cur_count = 0 5 | cur_val = 0 6 | for byte in sys.stdin.read(): 7 | if byte == '1': 8 | cur_val |= (1<] filename0 filename1 filename2..." 13 | ) 14 | .unwrap(); 15 | } 16 | fn read_no_interrupt(r: &mut R, buf: &mut [u8]) -> Result { 17 | loop { 18 | match r.read(buf) { 19 | Err(e) => match e.kind() { 20 | io::ErrorKind::Interrupted => continue, 21 | _ => return Err(e), 22 | }, 23 | Ok(cur_read) => return Ok(cur_read), 24 | } 25 | } 26 | } 27 | 28 | fn write_no_interrupt(w: &mut W, mut buf: &[u8]) -> Result { 29 | let mut total_read = 0usize; 30 | loop { 31 | match w.write(buf) { 32 | Err(e) => match e.kind() { 33 | io::ErrorKind::Interrupted => continue, 34 | _ => return Err(e), 35 | }, 36 | Ok(cur_read) => { 37 | buf = &buf[cur_read..]; 38 | total_read += cur_read; 39 | if buf.is_empty() { 40 | return Ok(total_read); 41 | } 42 | } 43 | } 44 | } 45 | } 46 | 47 | fn main() { 48 | let mut window_size: Option = None; 49 | let mut double_dash = false; 50 | let mut buffer_size = 4096usize; 51 | let mut filenames = Vec::::new(); 52 | let mut ostream = io::stdout(); 53 | if env::args_os().len() > 1 { 54 | for argument in env::args().skip(1) { 55 | if argument.starts_with("-w") && !double_dash { 56 | window_size = Some( 57 | argument 58 | .trim_matches('-') 59 | .trim_matches('w') 60 | .parse::() 61 | .unwrap() as u8, 62 | ); 63 | continue; 64 | } 65 | if argument.starts_with("-bs") && !double_dash { 66 | buffer_size = argument 67 | .trim_matches('-') 68 | .trim_matches('b') 69 | .trim_matches('s') 70 | .parse::() 71 | .unwrap(); 72 | continue; 73 | } 74 | if argument == "--" { 75 | double_dash = true; 76 | continue; 77 | } 78 | filenames.push(argument); 79 | } 80 | } else { 81 | usage(); 82 | return; 83 | } 84 | let mut ibuffer = vec![0u8; buffer_size]; 85 | let mut obuffer = vec![0u8; buffer_size]; 86 | let mut ooffset = 0; 87 | let mut ioffset; 88 | let mut bro_cat_li = match window_size { 89 | Some(ws) => BroCatli::new_with_window_size(ws), 90 | None => BroCatli::new(), 91 | }; 92 | for filename in filenames { 93 | bro_cat_li.new_brotli_file(); 94 | let mut input_file = match File::open(Path::new(&filename)) { 95 | Err(why) => panic!("couldn't open {:}\n{:}", filename, why), 96 | Ok(file) => file, 97 | }; 98 | loop { 99 | ioffset = 0; 100 | match read_no_interrupt(&mut input_file, &mut ibuffer[..]) { 101 | Err(e) => panic!("{}", e), 102 | Ok(cur_read) => { 103 | if cur_read == 0 { 104 | break; 105 | } 106 | loop { 107 | match bro_cat_li.stream( 108 | &ibuffer[..cur_read], 109 | &mut ioffset, 110 | &mut obuffer[..], 111 | &mut ooffset, 112 | ) { 113 | BroCatliResult::NeedsMoreOutput => { 114 | match write_no_interrupt(&mut ostream, &obuffer[..ooffset]) { 115 | Err(why) => panic!("couldn't write: {:}", why), 116 | Ok(count) => { 117 | assert_eq!(count, ooffset); 118 | } 119 | } 120 | ooffset = 0; 121 | } 122 | BroCatliResult::NeedsMoreInput => { 123 | break; 124 | } 125 | BroCatliResult::Success => { 126 | panic!("Unexpected state: Success when streaming before finish"); 127 | } 128 | failure => { 129 | panic!( 130 | "Failed to concatenate files on {:} {:?}", 131 | filename, failure 132 | ); 133 | } 134 | } 135 | } 136 | } 137 | } 138 | } 139 | } 140 | loop { 141 | match bro_cat_li.finish(&mut obuffer[..], &mut ooffset) { 142 | BroCatliResult::NeedsMoreOutput => { 143 | match write_no_interrupt(&mut ostream, &obuffer[..ooffset]) { 144 | Err(why) => panic!("couldn't write: {:}", why), 145 | Ok(count) => { 146 | assert_eq!(count, ooffset); 147 | } 148 | } 149 | ooffset = 0; 150 | } 151 | BroCatliResult::NeedsMoreInput => { 152 | panic!("Unexpected EOF"); 153 | } 154 | BroCatliResult::Success => { 155 | if ooffset != 0 { 156 | match write_no_interrupt(&mut ostream, &obuffer[..ooffset]) { 157 | Err(why) => panic!("couldn't write: {:}", why), 158 | Ok(count) => { 159 | assert_eq!(count, ooffset); 160 | } 161 | } 162 | } 163 | break; 164 | } 165 | failure => { 166 | panic!("{:?}", failure) 167 | } 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/bin/tests.rs: -------------------------------------------------------------------------------- 1 | #![cfg(test)] 2 | 3 | extern crate core; 4 | 5 | use core::cmp::min; 6 | use std::io; 7 | 8 | struct Buffer { 9 | data: Vec, 10 | read_offset: usize, 11 | } 12 | 13 | impl Buffer { 14 | pub fn new(buf: &[u8]) -> Buffer { 15 | let mut ret = Buffer { 16 | data: Vec::::new(), 17 | read_offset: 0, 18 | }; 19 | ret.data.extend(buf); 20 | ret 21 | } 22 | } 23 | impl io::Read for Buffer { 24 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 25 | let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); 26 | if bytes_to_read > 0 { 27 | buf[0..bytes_to_read] 28 | .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); 29 | } 30 | self.read_offset += bytes_to_read; 31 | Ok(bytes_to_read) 32 | } 33 | } 34 | impl io::Write for Buffer { 35 | fn write(&mut self, buf: &[u8]) -> io::Result { 36 | self.data.extend(buf); 37 | Ok(buf.len()) 38 | } 39 | fn flush(&mut self) -> io::Result<()> { 40 | Ok(()) 41 | } 42 | } 43 | 44 | fn copy_from_to(mut r: R, mut w: W) -> io::Result { 45 | let mut buffer: [u8; 65536] = [0; 65536]; 46 | let mut out_size: usize = 0; 47 | loop { 48 | match r.read(&mut buffer[..]) { 49 | Err(e) => { 50 | match e.kind() { 51 | io::ErrorKind::Interrupted => continue, 52 | _ => {} 53 | } 54 | return Err(e); 55 | } 56 | Ok(size) => { 57 | if size == 0 { 58 | break; 59 | } else { 60 | match w.write_all(&buffer[..size]) { 61 | Err(e) => { 62 | match e.kind() { 63 | io::ErrorKind::Interrupted => continue, 64 | _ => {} 65 | } 66 | return Err(e); 67 | } 68 | Ok(_) => out_size += size, 69 | } 70 | } 71 | } 72 | } 73 | } 74 | Ok(out_size) 75 | } 76 | 77 | #[test] 78 | fn test_10x_10y() { 79 | let in_buf: [u8; 12] = [ 80 | 0x1b, 0x13, 0x00, 0x00, 0xa4, 0xb0, 0xb2, 0xea, 0x81, 0x47, 0x02, 0x8a, 81 | ]; 82 | 83 | let mut output = Buffer::new(&[]); 84 | let mut input = super::BrotliDecompressor::new(Buffer::new(&in_buf), 4096); 85 | match copy_from_to(&mut input, &mut output) { 86 | Ok(_) => {} 87 | Err(e) => panic!("Error {:?}", e), 88 | } 89 | let mut i: usize = 0; 90 | while i < 10 { 91 | assert_eq!(output.data[i], b'X'); 92 | assert_eq!(output.data[i + 10], b'Y'); 93 | i += 1; 94 | } 95 | assert_eq!(output.data.len(), 20); 96 | } 97 | 98 | #[test] 99 | fn test_alice() { 100 | let in_buf = include_bytes!("../../testdata/alice29.txt.compressed"); 101 | 102 | let mut output = Buffer::new(&[]); 103 | let mut input = super::BrotliDecompressor::new(Buffer::new(in_buf), 1); 104 | match copy_from_to(&mut input, &mut output) { 105 | Ok(_) => {} 106 | Err(e) => panic!("Error {:?}", e), 107 | } 108 | let mut i: usize = 0; 109 | let truth = include_bytes!("../../testdata/alice29.txt"); 110 | while i < truth.len() { 111 | assert_eq!(output.data[i], truth[i]); 112 | i += 1; 113 | } 114 | assert_eq!(truth.len(), output.data.len()); 115 | } 116 | -------------------------------------------------------------------------------- /src/bin/validate.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "validation")] 2 | use core; 3 | use std::io::{self, Error, ErrorKind, Read, Write}; 4 | 5 | use alloc_no_stdlib::{Allocator, SliceWrapper}; 6 | use brotli::enc::BrotliEncoderParams; 7 | use brotli::{CustomWrite, DecompressorWriterCustomIo}; 8 | #[cfg(feature = "validation")] 9 | use sha2::{Digest, Sha256}; 10 | 11 | use super::{HeapAllocator, IoWriterWrapper, Rebox}; 12 | #[cfg(feature = "validation")] 13 | type Checksum = Sha256; 14 | 15 | struct Tee(OutputA, OutputB); 16 | impl Write for Tee { 17 | fn write(&mut self, data: &[u8]) -> Result { 18 | match self.0.write(data) { 19 | Err(err) => Err(err), 20 | Ok(size) => match self.1.write_all(&data[..size]) { 21 | Ok(_) => Ok(size), 22 | Err(err) => Err(err), 23 | }, 24 | } 25 | } 26 | fn flush(&mut self) -> Result<(), io::Error> { 27 | match self.0.flush() { 28 | Err(err) => Err(err), 29 | Ok(_) => loop { 30 | match self.1.flush() { 31 | Err(e) => match e.kind() { 32 | ErrorKind::Interrupted => continue, 33 | _ => return Err(e), 34 | }, 35 | Ok(e) => return Ok(e), 36 | } 37 | }, 38 | } 39 | } 40 | } 41 | 42 | struct DecompressAndValidate<'a, OutputType: Write + 'a>( 43 | DecompressorWriterCustomIo< 44 | io::Error, 45 | IoWriterWrapper<'a, OutputType>, 46 | Rebox, // buffer type 47 | HeapAllocator, 48 | HeapAllocator, 49 | HeapAllocator, 50 | >, 51 | ); 52 | 53 | impl<'a, OutputType: Write> Write for DecompressAndValidate<'a, OutputType> { 54 | fn write(&mut self, data: &[u8]) -> Result { 55 | self.0.write(data) 56 | } 57 | fn flush(&mut self) -> Result<(), io::Error> { 58 | self.0.flush() 59 | } 60 | } 61 | 62 | #[cfg(not(feature = "validation"))] 63 | fn make_sha_writer() -> io::Sink { 64 | io::sink() 65 | } 66 | #[cfg(not(feature = "validation"))] 67 | fn make_sha_reader(r: &mut InputType) -> &mut InputType { 68 | r 69 | } 70 | 71 | #[cfg(not(feature = "validation"))] 72 | fn sha_ok(_writer: &mut io::Sink, _reader: &mut InputType) -> bool { 73 | false 74 | } 75 | 76 | #[cfg(feature = "validation")] 77 | struct ShaReader<'a, InputType: Read + 'a> { 78 | reader: &'a mut InputType, 79 | checksum: Checksum, 80 | } 81 | #[cfg(feature = "validation")] 82 | impl<'a, InputType: Read + 'a> Read for ShaReader<'a, InputType> { 83 | fn read(&mut self, data: &mut [u8]) -> Result { 84 | match self.reader.read(data) { 85 | Err(e) => Err(e), 86 | Ok(size) => { 87 | self.checksum.update(&data[..size]); 88 | Ok(size) 89 | } 90 | } 91 | } 92 | } 93 | #[cfg(feature = "validation")] 94 | fn make_sha_reader(r: &mut InputType) -> ShaReader { 95 | ShaReader { 96 | reader: r, 97 | checksum: Checksum::default(), 98 | } 99 | } 100 | 101 | #[cfg(feature = "validation")] 102 | fn sha_ok(writer: &mut ShaWriter, reader: &mut ShaReader) -> bool { 103 | core::mem::replace(&mut writer.0, Checksum::default()).finalize() 104 | == core::mem::replace(&mut reader.checksum, Checksum::default()).finalize() 105 | } 106 | #[cfg(feature = "validation")] 107 | #[derive(Default)] 108 | struct ShaWriter(Checksum); 109 | #[cfg(feature = "validation")] 110 | impl Write for ShaWriter { 111 | fn write(&mut self, data: &[u8]) -> Result { 112 | self.0.update(data); 113 | Ok(data.len()) 114 | } 115 | fn flush(&mut self) -> Result<(), io::Error> { 116 | Ok(()) 117 | } 118 | } 119 | #[cfg(feature = "validation")] 120 | fn make_sha_writer() -> ShaWriter { 121 | ShaWriter::default() 122 | } 123 | #[cfg(feature = "validation")] 124 | const VALIDATION_FAILED: &'static str = "Validation failed"; 125 | #[cfg(not(feature = "validation"))] 126 | const VALIDATION_FAILED: &str = 127 | "Validation module not enabled: build with cargo build --features=validation"; 128 | 129 | pub fn compress_validate( 130 | r: &mut InputType, 131 | w: &mut OutputType, 132 | buffer_size: usize, 133 | params: &BrotliEncoderParams, 134 | custom_dictionary: Rebox, 135 | num_threads: usize, 136 | ) -> Result<(), io::Error> { 137 | let mut m8 = HeapAllocator::default(); 138 | let buffer = m8.alloc_cell(buffer_size); 139 | // FIXME: could reuse the dictionary to seed the compressor, but that violates the abstraction right now 140 | // also dictionaries are not very popular since they are mostly an internal concept, given their deprecation in 141 | // the standard brotli spec 142 | let mut dict = Vec::::new(); 143 | 144 | dict.extend_from_slice(custom_dictionary.slice()); 145 | let mut sha_writer = make_sha_writer(); 146 | let mut sha_reader = make_sha_reader(r); 147 | let ret; 148 | { 149 | let validate_writer = 150 | DecompressAndValidate(DecompressorWriterCustomIo::new_with_custom_dictionary( 151 | IoWriterWrapper(&mut sha_writer), 152 | buffer, 153 | m8, 154 | HeapAllocator::default(), 155 | HeapAllocator::default(), 156 | custom_dictionary, 157 | Error::new(ErrorKind::InvalidData, "Invalid Data"), 158 | )); 159 | let mut overarching_writer = Tee(validate_writer, w); 160 | ret = super::compress( 161 | &mut sha_reader, 162 | &mut overarching_writer, 163 | buffer_size, 164 | params, 165 | &dict[..], 166 | num_threads, 167 | ); 168 | } 169 | match ret { 170 | Ok(_ret) => { 171 | if sha_ok(&mut sha_writer, &mut sha_reader) { 172 | Ok(()) 173 | } else { 174 | Err(Error::new(ErrorKind::InvalidData, VALIDATION_FAILED)) 175 | } 176 | } 177 | Err(e) => Err(e), 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/enc/block_split.rs: -------------------------------------------------------------------------------- 1 | use super::super::alloc; 2 | use super::super::alloc::{Allocator, SliceWrapper}; 3 | use crate::enc::combined_alloc::alloc_default; 4 | 5 | pub struct BlockSplit + alloc::Allocator> { 6 | pub num_types: usize, 7 | pub num_blocks: usize, 8 | pub types: >::AllocatedMemory, 9 | pub lengths: >::AllocatedMemory, 10 | } 11 | 12 | impl + alloc::Allocator> Default for BlockSplit { 13 | fn default() -> Self { 14 | Self { 15 | num_types: 0, 16 | num_blocks: 0, 17 | types: alloc_default::(), 18 | lengths: alloc_default::(), 19 | } 20 | } 21 | } 22 | 23 | impl + alloc::Allocator> BlockSplit { 24 | pub fn new() -> BlockSplit { 25 | Self::default() 26 | } 27 | pub fn destroy(&mut self, m: &mut Alloc) { 28 | >::free_cell(m, core::mem::take(&mut self.types)); 29 | >::free_cell(m, core::mem::take(&mut self.lengths)); 30 | self.num_blocks = 0; 31 | self.num_types = 0; 32 | } 33 | pub fn types_alloc_size(&self) -> usize { 34 | self.types.slice().len() 35 | } 36 | pub fn lengths_alloc_size(&self) -> usize { 37 | self.lengths.slice().len() 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/enc/fixed_queue.rs: -------------------------------------------------------------------------------- 1 | pub const MAX_THREADS: usize = 16; 2 | 3 | pub struct FixedQueue { 4 | data: [Option; MAX_THREADS], 5 | size: usize, 6 | start: usize, 7 | } 8 | impl Default for FixedQueue { 9 | fn default() -> Self { 10 | Self::new() 11 | } 12 | } 13 | impl FixedQueue { 14 | pub fn new() -> Self { 15 | FixedQueue { 16 | data: [ 17 | None, None, None, None, None, None, None, None, None, None, None, None, None, None, 18 | None, None, 19 | ], 20 | size: 0, 21 | start: 0, 22 | } 23 | } 24 | pub fn can_push(&self) -> bool { 25 | self.size < self.data.len() 26 | } 27 | pub fn size(&self) -> usize { 28 | self.size 29 | } 30 | pub fn push(&mut self, item: T) -> Result<(), ()> { 31 | if self.size == self.data.len() { 32 | return Err(()); 33 | } 34 | let index = (self.start + self.size) % self.data.len(); 35 | self.data[index] = Some(item); 36 | self.size += 1; 37 | Ok(()) 38 | } 39 | pub fn pop(&mut self) -> Option { 40 | if self.size == 0 { 41 | return None; 42 | } 43 | let index = self.start % self.data.len(); 44 | let ret = self.data[index].take(); 45 | self.start += 1; 46 | self.size -= 1; 47 | ret 48 | } 49 | pub fn how_much_free_space(&self) -> usize { 50 | self.data.len() - self.size 51 | } 52 | pub fn remove) -> bool>(&mut self, f: F) -> Option { 53 | if self.size == 0 { 54 | return None; 55 | } 56 | for index in 0..self.size { 57 | if f(&self.data[(self.start + index) % self.data.len()]) { 58 | let start_index = self.start % self.data.len(); 59 | let target_index = (self.start + index) % self.data.len(); 60 | let ret = self.data[target_index].take(); 61 | let replace = self.data[start_index].take(); 62 | let is_none = core::mem::replace(&mut self.data[target_index], replace); 63 | assert!(is_none.is_none()); 64 | self.start += 1; 65 | self.size -= 1; 66 | return ret; 67 | } 68 | } 69 | None 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/enc/input_pair.rs: -------------------------------------------------------------------------------- 1 | use core; 2 | use core::cmp::min; 3 | 4 | use super::super::alloc::{SliceWrapper, SliceWrapperMut}; 5 | use super::interface::Freezable; 6 | #[derive(Copy, Clone, Default, Debug)] 7 | pub struct InputReference<'a> { 8 | pub data: &'a [u8], 9 | pub orig_offset: usize, // offset into the original slice of data 10 | } 11 | impl<'a> SliceWrapper for InputReference<'a> { 12 | fn slice(&self) -> &[u8] { 13 | self.data 14 | } 15 | } 16 | 17 | impl<'a> Freezable for InputReference<'a> { 18 | fn freeze(&self) -> super::interface::SliceOffset { 19 | debug_assert!(self.data.len() <= 0xffff_ffff); 20 | super::interface::SliceOffset(self.orig_offset, self.data.len() as u32) 21 | } 22 | } 23 | 24 | #[derive(Default)] 25 | pub struct InputReferenceMut<'a> { 26 | pub data: &'a mut [u8], 27 | pub orig_offset: usize, // offset into the original slice of data 28 | } 29 | 30 | impl<'a> SliceWrapper for InputReferenceMut<'a> { 31 | fn slice(&self) -> &[u8] { 32 | self.data 33 | } 34 | } 35 | impl<'a> SliceWrapperMut for InputReferenceMut<'a> { 36 | fn slice_mut(&mut self) -> &mut [u8] { 37 | self.data 38 | } 39 | } 40 | 41 | impl<'a> From> for InputReference<'a> { 42 | fn from(val: InputReferenceMut<'a>) -> InputReference<'a> { 43 | InputReference { 44 | data: val.data, 45 | orig_offset: val.orig_offset, 46 | } 47 | } 48 | } 49 | 50 | impl<'a> From<&'a InputReferenceMut<'a>> for InputReference<'a> { 51 | fn from(val: &'a InputReferenceMut<'a>) -> InputReference<'a> { 52 | InputReference { 53 | data: val.data, 54 | orig_offset: val.orig_offset, 55 | } 56 | } 57 | } 58 | 59 | #[derive(Clone, Debug, Copy)] 60 | pub struct InputPair<'a>(pub InputReference<'a>, pub InputReference<'a>); 61 | 62 | impl<'a> PartialEq for InputPair<'a> { 63 | fn eq(&self, other: &InputPair<'_>) -> bool { 64 | if self.0.len() + self.1.len() != other.0.len() + other.1.len() { 65 | return false; 66 | } 67 | for (a_iter, b_iter) in self 68 | .0 69 | .data 70 | .iter() 71 | .chain(self.1.data.iter()) 72 | .zip(other.0.data.iter().chain(other.1.data.iter())) 73 | { 74 | if *a_iter != *b_iter { 75 | return false; 76 | } 77 | } 78 | true 79 | } 80 | } 81 | impl<'a> core::ops::Index for InputPair<'a> { 82 | type Output = u8; 83 | fn index(&self, index: usize) -> &u8 { 84 | if index >= self.0.len() { 85 | &self.1.data[index - self.0.len()] 86 | } else { 87 | &self.0.data[index] 88 | } 89 | } 90 | } 91 | impl<'a> core::fmt::LowerHex for InputPair<'a> { 92 | fn fmt(&self, fmtr: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> { 93 | for item in self.0.data { 94 | fmtr.write_fmt(format_args!("{:02x}", item))? 95 | } 96 | for item in self.1.data { 97 | fmtr.write_fmt(format_args!("{:02x}", item))? 98 | } 99 | Ok(()) 100 | } 101 | } 102 | 103 | impl<'a> InputPair<'a> { 104 | pub fn split_at(&self, loc: usize) -> (InputPair<'a>, InputPair<'a>) { 105 | if loc >= self.0.len() { 106 | let offset_from_self_1 = loc - self.0.len(); 107 | let (first, second) = self.1.data.split_at(min(offset_from_self_1, self.1.len())); 108 | return ( 109 | InputPair::<'a>( 110 | self.0, 111 | InputReference::<'a> { 112 | data: first, 113 | orig_offset: self.1.orig_offset, 114 | }, 115 | ), 116 | InputPair::<'a>( 117 | InputReference::<'a>::default(), 118 | InputReference::<'a> { 119 | data: second, 120 | orig_offset: offset_from_self_1 + self.1.orig_offset, 121 | }, 122 | ), 123 | ); 124 | } 125 | let (first, second) = self.0.data.split_at(min(loc, self.0.len())); 126 | ( 127 | InputPair::<'a>( 128 | InputReference::<'a> { 129 | data: first, 130 | orig_offset: self.0.orig_offset, 131 | }, 132 | InputReference::<'a>::default(), 133 | ), 134 | InputPair::<'a>( 135 | InputReference::<'a> { 136 | data: second, 137 | orig_offset: self.0.orig_offset + loc, 138 | }, 139 | self.1, 140 | ), 141 | ) 142 | } 143 | pub fn len(&self) -> usize { 144 | self.0.len() + self.1.len() 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/enc/ir_interpret.rs: -------------------------------------------------------------------------------- 1 | use super::super::alloc::SliceWrapper; 2 | use super::constants::{kSigned3BitContextLookup, kUTF8ContextLookup}; 3 | use super::histogram::ContextType; 4 | use super::input_pair::InputReference; 5 | use super::interface; 6 | use super::interface::LiteralPredictionModeNibble; 7 | pub trait IRInterpreter { 8 | fn inc_local_byte_offset(&mut self, inc: usize); 9 | fn local_byte_offset(&self) -> usize; 10 | fn update_block_type(&mut self, new_type: u8, new_stride: u8); 11 | fn block_type(&self) -> u8; 12 | fn literal_data_at_offset(&self, index: usize) -> u8; 13 | fn literal_context_map(&self) -> &[u8]; 14 | fn prediction_mode(&self) -> crate::interface::LiteralPredictionModeNibble; 15 | fn update_cost( 16 | &mut self, 17 | stride_prior: [u8; 8], 18 | stride_byte_offset: usize, 19 | selected_bits: u8, 20 | cm_prior: usize, 21 | literal: u8, 22 | ); 23 | } 24 | 25 | pub fn push_base( 26 | xself: &mut Interpreter, 27 | val: interface::Command>, 28 | ) { 29 | match val { 30 | interface::Command::BlockSwitchCommand(_) 31 | | interface::Command::BlockSwitchDistance(_) 32 | | interface::Command::PredictionMode(_) => {} 33 | interface::Command::Copy(ref copy) => { 34 | xself.inc_local_byte_offset(copy.num_bytes as usize); 35 | } 36 | interface::Command::Dict(ref dict) => { 37 | xself.inc_local_byte_offset(dict.final_size as usize); 38 | } 39 | interface::Command::BlockSwitchLiteral(block_type) => { 40 | xself.update_block_type(block_type.block_type(), block_type.stride()) 41 | } 42 | interface::Command::Literal(ref lit) => { 43 | //let stride = xself.get_stride(xself.local_byte_offset()) as usize; 44 | let mut priors = [0u8; 8]; 45 | for poffset in 0..8 { 46 | if xself.local_byte_offset() > poffset { 47 | let input_offset = xself.local_byte_offset() - poffset - 1; 48 | priors[7 - poffset] = xself.literal_data_at_offset(input_offset); 49 | } 50 | } 51 | let mut cur = 0usize; 52 | for literal in lit.data.slice().iter() { 53 | let (huffman_table_index, selected_bits) = 54 | compute_huffman_table_index_for_context_map( 55 | priors[(cur + 7) & 7], 56 | priors[(cur + 6) & 7], 57 | xself.literal_context_map(), 58 | xself.prediction_mode(), 59 | xself.block_type(), 60 | ); 61 | xself.update_cost( 62 | priors, 63 | (cur + 7) & 7, 64 | selected_bits, 65 | huffman_table_index, 66 | *literal, 67 | ); 68 | priors[cur & 7] = *literal; 69 | cur += 1; 70 | cur &= 7; 71 | } 72 | xself.inc_local_byte_offset(lit.data.slice().len()); 73 | } 74 | } 75 | } 76 | 77 | // not sure why this fails 78 | //impl<'a> interface::CommandProcessor<'a> for IRInterpreter { 79 | // fn push])>(&mut self, 80 | // val: interface::Command>, 81 | // callback: &mut Cb) { 82 | // push_base(self, val, callback) 83 | // } 84 | //} 85 | 86 | fn compute_huffman_table_index_for_context_map( 87 | prev_byte: u8, 88 | prev_prev_byte: u8, 89 | literal_context_map: &[u8], //interface::PredictionModeContextMap, 90 | prediction_mode: LiteralPredictionModeNibble, 91 | block_type: u8, 92 | ) -> (usize, u8) { 93 | let prior = Context( 94 | prev_byte, 95 | prev_prev_byte, 96 | prediction_mode.to_context_enum().unwrap(), 97 | ); 98 | assert!(prior < 64); 99 | let context_map_index = ((block_type as usize) << 6) | prior as usize; 100 | if context_map_index < literal_context_map.len() { 101 | (literal_context_map[context_map_index] as usize, prior) 102 | } else { 103 | (prior as usize, prior) 104 | } 105 | } 106 | 107 | pub fn Context(p1: u8, p2: u8, mode: ContextType) -> u8 { 108 | match mode { 109 | ContextType::CONTEXT_LSB6 => p1 & 0x3f, 110 | ContextType::CONTEXT_MSB6 => (p1 as i32 >> 2) as u8, 111 | ContextType::CONTEXT_UTF8 => { 112 | (kUTF8ContextLookup[p1 as usize] as i32 113 | | kUTF8ContextLookup[(p2 as i32 + 256i32) as usize] as i32) as u8 114 | } 115 | ContextType::CONTEXT_SIGNED => { 116 | (((kSigned3BitContextLookup[p1 as usize] as i32) << 3) 117 | + kSigned3BitContextLookup[p2 as usize] as i32) as u8 118 | } 119 | } 120 | // 0u8 121 | } 122 | -------------------------------------------------------------------------------- /src/enc/log_table_8.rs: -------------------------------------------------------------------------------- 1 | use crate::enc::floatX; 2 | 3 | #[allow(clippy::excessive_precision)] 4 | pub static logs_8: [floatX; 256] = [ 5 | 0.0000000000000000, 6 | 0.0000000000000000, 7 | 1.0000000000000000, 8 | 1.5849625007211563, 9 | 2.0000000000000000, 10 | 2.3219280948873622, 11 | 2.5849625007211561, 12 | 2.8073549220576042, 13 | 3.0000000000000000, 14 | 3.1699250014423126, 15 | 3.3219280948873626, 16 | 3.4594316186372978, 17 | 3.5849625007211565, 18 | 3.7004397181410922, 19 | 3.8073549220576037, 20 | 3.9068905956085187, 21 | 4.0000000000000000, 22 | 4.0874628412503400, 23 | 4.1699250014423122, 24 | 4.2479275134435852, 25 | 4.3219280948873626, 26 | 4.3923174227787607, 27 | 4.4594316186372973, 28 | 4.5235619560570131, 29 | 4.5849625007211570, 30 | 4.6438561897747244, 31 | 4.7004397181410926, 32 | 4.7548875021634691, 33 | 4.8073549220576037, 34 | 4.8579809951275728, 35 | 4.9068905956085187, 36 | 4.9541963103868758, 37 | 5.0000000000000000, 38 | 5.0443941193584534, 39 | 5.0874628412503400, 40 | 5.1292830169449664, 41 | 5.1699250014423122, 42 | 5.2094533656289501, 43 | 5.2479275134435852, 44 | 5.2854022188622487, 45 | 5.3219280948873626, 46 | 5.3575520046180838, 47 | 5.3923174227787607, 48 | 5.4262647547020979, 49 | 5.4594316186372973, 50 | 5.4918530963296748, 51 | 5.5235619560570131, 52 | 5.5545888516776376, 53 | 5.5849625007211570, 54 | 5.6147098441152083, 55 | 5.6438561897747244, 56 | 5.6724253419714961, 57 | 5.7004397181410926, 58 | 5.7279204545631996, 59 | 5.7548875021634691, 60 | 5.7813597135246599, 61 | 5.8073549220576046, 62 | 5.8328900141647422, 63 | 5.8579809951275719, 64 | 5.8826430493618416, 65 | 5.9068905956085187, 66 | 5.9307373375628867, 67 | 5.9541963103868758, 68 | 5.9772799234999168, 69 | 6.0000000000000000, 70 | 6.0223678130284544, 71 | 6.0443941193584534, 72 | 6.0660891904577721, 73 | 6.0874628412503400, 74 | 6.1085244567781700, 75 | 6.1292830169449672, 76 | 6.1497471195046822, 77 | 6.1699250014423122, 78 | 6.1898245588800176, 79 | 6.2094533656289510, 80 | 6.2288186904958804, 81 | 6.2479275134435861, 82 | 6.2667865406949019, 83 | 6.2854022188622487, 84 | 6.3037807481771031, 85 | 6.3219280948873617, 86 | 6.3398500028846252, 87 | 6.3575520046180847, 88 | 6.3750394313469254, 89 | 6.3923174227787598, 90 | 6.4093909361377026, 91 | 6.4262647547020979, 92 | 6.4429434958487288, 93 | 6.4594316186372982, 94 | 6.4757334309663976, 95 | 6.4918530963296748, 96 | 6.5077946401986964, 97 | 6.5235619560570131, 98 | 6.5391588111080319, 99 | 6.5545888516776376, 100 | 6.5698556083309478, 101 | 6.5849625007211561, 102 | 6.5999128421871278, 103 | 6.6147098441152092, 104 | 6.6293566200796095, 105 | 6.6438561897747253, 106 | 6.6582114827517955, 107 | 6.6724253419714952, 108 | 6.6865005271832185, 109 | 6.7004397181410917, 110 | 6.7142455176661224, 111 | 6.7279204545631988, 112 | 6.7414669864011465, 113 | 6.7548875021634691, 114 | 6.7681843247769260, 115 | 6.7813597135246599, 116 | 6.7944158663501062, 117 | 6.8073549220576037, 118 | 6.8201789624151887, 119 | 6.8328900141647422, 120 | 6.8454900509443757, 121 | 6.8579809951275719, 122 | 6.8703647195834048, 123 | 6.8826430493618416, 124 | 6.8948177633079437, 125 | 6.9068905956085187, 126 | 6.9188632372745955, 127 | 6.9307373375628867, 128 | 6.9425145053392399, 129 | 6.9541963103868758, 130 | 6.9657842846620879, 131 | 6.9772799234999168, 132 | 6.9886846867721664, 133 | 7.0000000000000000, 134 | 7.0112272554232540, 135 | 7.0223678130284544, 136 | 7.0334230015374501, 137 | 7.0443941193584534, 138 | 7.0552824355011898, 139 | 7.0660891904577721, 140 | 7.0768155970508317, 141 | 7.0874628412503400, 142 | 7.0980320829605272, 143 | 7.1085244567781700, 144 | 7.1189410727235076, 145 | 7.1292830169449664, 146 | 7.1395513523987937, 147 | 7.1497471195046822, 148 | 7.1598713367783891, 149 | 7.1699250014423130, 150 | 7.1799090900149345, 151 | 7.1898245588800176, 152 | 7.1996723448363644, 153 | 7.2094533656289492, 154 | 7.2191685204621621, 155 | 7.2288186904958804, 156 | 7.2384047393250794, 157 | 7.2479275134435861, 158 | 7.2573878426926521, 159 | 7.2667865406949019, 160 | 7.2761244052742384, 161 | 7.2854022188622487, 162 | 7.2946207488916270, 163 | 7.3037807481771031, 164 | 7.3128829552843557, 165 | 7.3219280948873617, 166 | 7.3309168781146177, 167 | 7.3398500028846243, 168 | 7.3487281542310781, 169 | 7.3575520046180847, 170 | 7.3663222142458151, 171 | 7.3750394313469254, 172 | 7.3837042924740528, 173 | 7.3923174227787607, 174 | 7.4008794362821844, 175 | 7.4093909361377026, 176 | 7.4178525148858991, 177 | 7.4262647547020979, 178 | 7.4346282276367255, 179 | 7.4429434958487288, 180 | 7.4512111118323299, 181 | 7.4594316186372973, 182 | 7.4676055500829976, 183 | 7.4757334309663976, 184 | 7.4838157772642564, 185 | 7.4918530963296748, 186 | 7.4998458870832057, 187 | 7.5077946401986964, 188 | 7.5156998382840436, 189 | 7.5235619560570131, 190 | 7.5313814605163119, 191 | 7.5391588111080319, 192 | 7.5468944598876373, 193 | 7.5545888516776376, 194 | 7.5622424242210728, 195 | 7.5698556083309478, 196 | 7.5774288280357487, 197 | 7.5849625007211561, 198 | 7.5924570372680806, 199 | 7.5999128421871278, 200 | 7.6073303137496113, 201 | 7.6147098441152075, 202 | 7.6220518194563764, 203 | 7.6293566200796095, 204 | 7.6366246205436488, 205 | 7.6438561897747244, 206 | 7.6510516911789290, 207 | 7.6582114827517955, 208 | 7.6653359171851765, 209 | 7.6724253419714952, 210 | 7.6794800995054464, 211 | 7.6865005271832185, 212 | 7.6934869574993252, 213 | 7.7004397181410926, 214 | 7.7073591320808825, 215 | 7.7142455176661224, 216 | 7.7210991887071856, 217 | 7.7279204545631996, 218 | 7.7347096202258392, 219 | 7.7414669864011465, 220 | 7.7481928495894596, 221 | 7.7548875021634691, 222 | 7.7615512324444795, 223 | 7.7681843247769260, 224 | 7.7747870596011737, 225 | 7.7813597135246608, 226 | 7.7879025593914317, 227 | 7.7944158663501062, 228 | 7.8008998999203047, 229 | 7.8073549220576037, 230 | 7.8137811912170374, 231 | 7.8201789624151887, 232 | 7.8265484872909159, 233 | 7.8328900141647422, 234 | 7.8392037880969445, 235 | 7.8454900509443757, 236 | 7.8517490414160571, 237 | 7.8579809951275719, 238 | 7.8641861446542798, 239 | 7.8703647195834048, 240 | 7.8765169465650002, 241 | 7.8826430493618425, 242 | 7.8887432488982601, 243 | 7.8948177633079446, 244 | 7.9008668079807496, 245 | 7.9068905956085187, 246 | 7.9128893362299619, 247 | 7.9188632372745955, 248 | 7.9248125036057813, 249 | 7.9307373375628867, 250 | 7.9366379390025719, 251 | 7.9425145053392399, 252 | 7.9483672315846778, 253 | 7.9541963103868758, 254 | 7.9600019320680806, 255 | 7.9657842846620870, 256 | 7.9715435539507720, 257 | 7.9772799234999168, 258 | 7.9829935746943104, 259 | 7.9886846867721664, 260 | 7.9943534368588578, 261 | ]; 262 | -------------------------------------------------------------------------------- /src/enc/multithreading.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "std")] 2 | 3 | use alloc::{Allocator, SliceWrapper}; 4 | use core::marker::PhantomData; 5 | use core::mem; 6 | use std; 7 | // in-place thread create 8 | use std::sync::RwLock; 9 | use std::thread::JoinHandle; 10 | 11 | use crate::enc::backward_references::UnionHasher; 12 | use crate::enc::threading::{ 13 | AnyBoxConstructor, BatchSpawnable, BatchSpawnableLite, BrotliEncoderThreadError, CompressMulti, 14 | CompressionThreadResult, InternalOwned, InternalSendAlloc, Joinable, Owned, OwnedRetriever, 15 | PoisonedThreadError, SendAlloc, 16 | }; 17 | use crate::enc::{BrotliAlloc, BrotliEncoderParams}; 18 | 19 | pub struct MultiThreadedJoinable( 20 | JoinHandle, 21 | PhantomData, 22 | ); 23 | 24 | impl Joinable 25 | for MultiThreadedJoinable 26 | { 27 | fn join(self) -> Result { 28 | match self.0.join() { 29 | Ok(t) => Ok(t), 30 | Err(e) => Err(::new(e)), 31 | } 32 | } 33 | } 34 | 35 | pub struct MultiThreadedOwnedRetriever(RwLock); 36 | 37 | impl OwnedRetriever for MultiThreadedOwnedRetriever { 38 | fn view T>(&self, f: F) -> Result { 39 | match self.0.read() { 40 | Ok(u) => Ok(f(&*u)), 41 | Err(_) => Err(PoisonedThreadError::default()), 42 | } 43 | } 44 | fn unwrap(self) -> Result { 45 | match self.0.into_inner() { 46 | Ok(u) => Ok(u), 47 | Err(_) => Err(PoisonedThreadError::default()), 48 | } 49 | } 50 | } 51 | 52 | #[derive(Default)] 53 | pub struct MultiThreadedSpawner {} 54 | 55 | fn spawn_work< 56 | ReturnValue: Send + 'static, 57 | ExtraInput: Send + 'static, 58 | F: Fn(ExtraInput, usize, usize, &U, Alloc) -> ReturnValue + Send + 'static, 59 | Alloc: BrotliAlloc + Send + 'static, 60 | U: Send + 'static + Sync, 61 | >( 62 | extra_input: ExtraInput, 63 | index: usize, 64 | num_threads: usize, 65 | locked_input: std::sync::Arc>, 66 | alloc: Alloc, 67 | f: F, 68 | ) -> std::thread::JoinHandle 69 | where 70 | >::AllocatedMemory: Send + 'static, 71 | { 72 | std::thread::spawn(move || { 73 | let t: ReturnValue = locked_input 74 | .view(move |guard: &U| -> ReturnValue { 75 | f(extra_input, index, num_threads, guard, alloc) 76 | }) 77 | .unwrap(); 78 | t 79 | }) 80 | } 81 | 82 | impl< 83 | ReturnValue: Send + 'static, 84 | ExtraInput: Send + 'static, 85 | Alloc: BrotliAlloc + Send + 'static, 86 | U: Send + 'static + Sync, 87 | > BatchSpawnable for MultiThreadedSpawner 88 | where 89 | >::AllocatedMemory: Send + 'static, 90 | { 91 | type JoinHandle = MultiThreadedJoinable; 92 | type FinalJoinHandle = std::sync::Arc>; 93 | fn make_spawner(&mut self, input: &mut Owned) -> Self::FinalJoinHandle { 94 | std::sync::Arc::>::new(RwLock::new( 95 | mem::replace(input, Owned(InternalOwned::Borrowed)).unwrap(), 96 | )) 97 | } 98 | fn spawn ReturnValue + Send + 'static + Copy>( 99 | &mut self, 100 | input: &mut Self::FinalJoinHandle, 101 | work: &mut SendAlloc, 102 | index: usize, 103 | num_threads: usize, 104 | f: F, 105 | ) { 106 | let (alloc, extra_input) = work.replace_with_default(); 107 | let ret = spawn_work(extra_input, index, num_threads, input.clone(), alloc, f); 108 | *work = SendAlloc(InternalSendAlloc::Join(MultiThreadedJoinable( 109 | ret, 110 | PhantomData, 111 | ))); 112 | } 113 | } 114 | impl< 115 | ReturnValue: Send + 'static, 116 | ExtraInput: Send + 'static, 117 | Alloc: BrotliAlloc + Send + 'static, 118 | U: Send + 'static + Sync, 119 | > BatchSpawnableLite for MultiThreadedSpawner 120 | where 121 | >::AllocatedMemory: Send + 'static, 122 | >::AllocatedMemory: Send + Sync, 123 | >::AllocatedMemory: Send + Sync, 124 | { 125 | type JoinHandle = 126 | >::JoinHandle; 127 | type FinalJoinHandle = >::FinalJoinHandle; 133 | fn make_spawner(&mut self, input: &mut Owned) -> Self::FinalJoinHandle { 134 | >::make_spawner(self, input) 135 | } 136 | fn spawn( 137 | &mut self, 138 | handle: &mut Self::FinalJoinHandle, 139 | alloc_per_thread: &mut SendAlloc, 140 | index: usize, 141 | num_threads: usize, 142 | f: fn(ExtraInput, usize, usize, &U, Alloc) -> ReturnValue, 143 | ) { 144 | >::spawn( 145 | self, 146 | handle, 147 | alloc_per_thread, 148 | index, 149 | num_threads, 150 | f, 151 | ) 152 | } 153 | } 154 | 155 | pub fn compress_multi< 156 | Alloc: BrotliAlloc + Send + 'static, 157 | SliceW: SliceWrapper + Send + 'static + Sync, 158 | >( 159 | params: &BrotliEncoderParams, 160 | owned_input: &mut Owned, 161 | output: &mut [u8], 162 | alloc_per_thread: &mut [SendAlloc< 163 | CompressionThreadResult, 164 | UnionHasher, 165 | Alloc, 166 | , 168 | UnionHasher, 169 | Alloc, 170 | SliceW, 171 | >>::JoinHandle, 172 | >], 173 | ) -> Result 174 | where 175 | >::AllocatedMemory: Send, 176 | >::AllocatedMemory: Send + Sync, 177 | >::AllocatedMemory: Send + Sync, 178 | { 179 | CompressMulti( 180 | params, 181 | owned_input, 182 | output, 183 | alloc_per_thread, 184 | &mut MultiThreadedSpawner::default(), 185 | ) 186 | } 187 | -------------------------------------------------------------------------------- /src/enc/parameters.rs: -------------------------------------------------------------------------------- 1 | #[derive(PartialEq, Eq, Copy, Clone, Debug)] 2 | #[repr(C)] 3 | pub enum BrotliEncoderParameter { 4 | BROTLI_PARAM_MODE = 0, 5 | BROTLI_PARAM_QUALITY = 1, 6 | BROTLI_PARAM_LGWIN = 2, 7 | BROTLI_PARAM_LGBLOCK = 3, 8 | BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING = 4, 9 | BROTLI_PARAM_SIZE_HINT = 5, 10 | BROTLI_PARAM_LARGE_WINDOW = 6, 11 | BROTLI_PARAM_Q9_5 = 150, 12 | BROTLI_METABLOCK_CALLBACK = 151, 13 | BROTLI_PARAM_STRIDE_DETECTION_QUALITY = 152, 14 | BROTLI_PARAM_HIGH_ENTROPY_DETECTION_QUALITY = 153, 15 | BROTLI_PARAM_LITERAL_BYTE_SCORE = 154, 16 | BROTLI_PARAM_CDF_ADAPTATION_DETECTION = 155, 17 | BROTLI_PARAM_PRIOR_BITMASK_DETECTION = 156, 18 | BROTLI_PARAM_SPEED = 157, 19 | BROTLI_PARAM_SPEED_MAX = 158, 20 | BROTLI_PARAM_CM_SPEED = 159, 21 | BROTLI_PARAM_CM_SPEED_MAX = 160, 22 | BROTLI_PARAM_SPEED_LOW = 161, 23 | BROTLI_PARAM_SPEED_LOW_MAX = 162, 24 | BROTLI_PARAM_CM_SPEED_LOW = 164, 25 | BROTLI_PARAM_CM_SPEED_LOW_MAX = 165, 26 | BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166, 27 | BROTLI_PARAM_CATABLE = 167, 28 | BROTLI_PARAM_APPENDABLE = 168, 29 | BROTLI_PARAM_MAGIC_NUMBER = 169, 30 | BROTLI_PARAM_NO_DICTIONARY = 170, 31 | BROTLI_PARAM_FAVOR_EFFICIENCY = 171, 32 | UNUSED7 = 7, 33 | UNUSED8 = 8, 34 | UNUSED9 = 9, 35 | UNUSED10 = 10, 36 | UNUSED11 = 11, 37 | UNUSED12 = 12, 38 | UNUSED13 = 13, 39 | UNUSED14 = 14, 40 | UNUSED15 = 15, 41 | UNUSED16 = 16, 42 | UNUSED17 = 17, 43 | UNUSED18 = 18, 44 | UNUSED19 = 19, 45 | UNUSED20 = 20, 46 | UNUSED21 = 21, 47 | UNUSED22 = 22, 48 | UNUSED23 = 23, 49 | UNUSED24 = 24, 50 | UNUSED25 = 25, 51 | UNUSED26 = 26, 52 | UNUSED27 = 27, 53 | UNUSED28 = 28, 54 | UNUSED29 = 29, 55 | UNUSED30 = 30, 56 | UNUSED31 = 31, 57 | UNUSED32 = 32, 58 | UNUSED33 = 33, 59 | UNUSED34 = 34, 60 | UNUSED35 = 35, 61 | UNUSED36 = 36, 62 | UNUSED37 = 37, 63 | UNUSED38 = 38, 64 | UNUSED39 = 39, 65 | UNUSED40 = 40, 66 | UNUSED41 = 41, 67 | UNUSED42 = 42, 68 | UNUSED43 = 43, 69 | UNUSED44 = 44, 70 | UNUSED45 = 45, 71 | UNUSED46 = 46, 72 | UNUSED47 = 47, 73 | UNUSED48 = 48, 74 | UNUSED49 = 49, 75 | UNUSED50 = 50, 76 | UNUSED51 = 51, 77 | UNUSED52 = 52, 78 | UNUSED53 = 53, 79 | UNUSED54 = 54, 80 | UNUSED55 = 55, 81 | UNUSED56 = 56, 82 | UNUSED57 = 57, 83 | UNUSED58 = 58, 84 | UNUSED59 = 59, 85 | UNUSED60 = 60, 86 | UNUSED61 = 61, 87 | UNUSED62 = 62, 88 | UNUSED63 = 63, 89 | UNUSED64 = 64, 90 | UNUSED65 = 65, 91 | UNUSED66 = 66, 92 | UNUSED67 = 67, 93 | UNUSED68 = 68, 94 | UNUSED69 = 69, 95 | UNUSED70 = 70, 96 | UNUSED71 = 71, 97 | UNUSED72 = 72, 98 | UNUSED73 = 73, 99 | UNUSED74 = 74, 100 | UNUSED75 = 75, 101 | UNUSED76 = 76, 102 | UNUSED77 = 77, 103 | UNUSED78 = 78, 104 | UNUSED79 = 79, 105 | UNUSED80 = 80, 106 | UNUSED81 = 81, 107 | UNUSED82 = 82, 108 | UNUSED83 = 83, 109 | UNUSED84 = 84, 110 | UNUSED85 = 85, 111 | UNUSED86 = 86, 112 | UNUSED87 = 87, 113 | UNUSED88 = 88, 114 | UNUSED89 = 89, 115 | UNUSED90 = 90, 116 | UNUSED91 = 91, 117 | UNUSED92 = 92, 118 | UNUSED93 = 93, 119 | UNUSED94 = 94, 120 | UNUSED95 = 95, 121 | UNUSED96 = 96, 122 | UNUSED97 = 97, 123 | UNUSED98 = 98, 124 | UNUSED99 = 99, 125 | UNUSED100 = 100, 126 | UNUSED101 = 101, 127 | UNUSED102 = 102, 128 | UNUSED103 = 103, 129 | UNUSED104 = 104, 130 | UNUSED105 = 105, 131 | UNUSED106 = 106, 132 | UNUSED107 = 107, 133 | UNUSED108 = 108, 134 | UNUSED109 = 109, 135 | UNUSED110 = 110, 136 | UNUSED111 = 111, 137 | UNUSED112 = 112, 138 | UNUSED113 = 113, 139 | UNUSED114 = 114, 140 | UNUSED115 = 115, 141 | UNUSED116 = 116, 142 | UNUSED117 = 117, 143 | UNUSED118 = 118, 144 | UNUSED119 = 119, 145 | UNUSED120 = 120, 146 | UNUSED121 = 121, 147 | UNUSED122 = 122, 148 | UNUSED123 = 123, 149 | UNUSED124 = 124, 150 | UNUSED125 = 125, 151 | UNUSED126 = 126, 152 | UNUSED127 = 127, 153 | UNUSED128 = 128, 154 | UNUSED129 = 129, 155 | UNUSED130 = 130, 156 | UNUSED131 = 131, 157 | UNUSED132 = 132, 158 | UNUSED133 = 133, 159 | UNUSED134 = 134, 160 | UNUSED135 = 135, 161 | UNUSED136 = 136, 162 | UNUSED137 = 137, 163 | UNUSED138 = 138, 164 | UNUSED139 = 139, 165 | UNUSED140 = 140, 166 | UNUSED141 = 141, 167 | UNUSED142 = 142, 168 | UNUSED143 = 143, 169 | UNUSED144 = 144, 170 | UNUSED145 = 145, 171 | UNUSED146 = 146, 172 | UNUSED147 = 147, 173 | UNUSED148 = 148, 174 | UNUSED149 = 149, 175 | UNUSED172 = 172, 176 | UNUSED173 = 173, 177 | UNUSED174 = 174, 178 | UNUSED175 = 175, 179 | UNUSED176 = 176, 180 | UNUSED177 = 177, 181 | UNUSED178 = 178, 182 | UNUSED179 = 179, 183 | UNUSED180 = 180, 184 | UNUSED181 = 181, 185 | UNUSED182 = 182, 186 | UNUSED183 = 183, 187 | UNUSED184 = 184, 188 | UNUSED185 = 185, 189 | UNUSED186 = 186, 190 | UNUSED187 = 187, 191 | UNUSED188 = 188, 192 | UNUSED189 = 189, 193 | UNUSED190 = 190, 194 | UNUSED191 = 191, 195 | UNUSED192 = 192, 196 | UNUSED193 = 193, 197 | UNUSED194 = 194, 198 | UNUSED195 = 195, 199 | UNUSED196 = 196, 200 | UNUSED197 = 197, 201 | UNUSED198 = 198, 202 | UNUSED199 = 199, 203 | UNUSED200 = 200, 204 | UNUSED201 = 201, 205 | UNUSED202 = 202, 206 | UNUSED203 = 203, 207 | UNUSED204 = 204, 208 | UNUSED205 = 205, 209 | UNUSED206 = 206, 210 | UNUSED207 = 207, 211 | UNUSED208 = 208, 212 | UNUSED209 = 209, 213 | UNUSED210 = 210, 214 | UNUSED211 = 211, 215 | UNUSED212 = 212, 216 | UNUSED213 = 213, 217 | UNUSED214 = 214, 218 | UNUSED215 = 215, 219 | UNUSED216 = 216, 220 | UNUSED217 = 217, 221 | UNUSED218 = 218, 222 | UNUSED219 = 219, 223 | UNUSED220 = 220, 224 | UNUSED221 = 221, 225 | UNUSED222 = 222, 226 | UNUSED223 = 223, 227 | UNUSED224 = 224, 228 | UNUSED225 = 225, 229 | UNUSED226 = 226, 230 | UNUSED227 = 227, 231 | UNUSED228 = 228, 232 | UNUSED229 = 229, 233 | UNUSED230 = 230, 234 | UNUSED231 = 231, 235 | UNUSED232 = 232, 236 | UNUSED233 = 233, 237 | UNUSED234 = 234, 238 | UNUSED235 = 235, 239 | UNUSED236 = 236, 240 | UNUSED237 = 237, 241 | UNUSED238 = 238, 242 | UNUSED239 = 239, 243 | UNUSED240 = 240, 244 | UNUSED241 = 241, 245 | UNUSED242 = 242, 246 | UNUSED243 = 243, 247 | UNUSED244 = 244, 248 | UNUSED245 = 245, 249 | UNUSED246 = 246, 250 | UNUSED247 = 247, 251 | UNUSED248 = 248, 252 | UNUSED249 = 249, 253 | UNUSED250 = 250, 254 | UNUSED251 = 251, 255 | UNUSED252 = 252, 256 | UNUSED253 = 253, 257 | UNUSED254 = 254, 258 | UNUSED255 = 255, 259 | } 260 | -------------------------------------------------------------------------------- /src/enc/pdf.rs: -------------------------------------------------------------------------------- 1 | //TODO: replace with builtin SIMD type 2 | 3 | // FIXME!!! 4 | #[allow(dead_code)] 5 | #[derive(Copy, Clone, Default, Debug)] 6 | pub struct PDF([i16; 16]); 7 | -------------------------------------------------------------------------------- /src/enc/singlethreading.rs: -------------------------------------------------------------------------------- 1 | use alloc::{Allocator, SliceWrapper}; 2 | use core::marker::PhantomData; 3 | use core::mem; 4 | #[cfg(feature = "std")] 5 | use std; 6 | 7 | use super::backward_references::UnionHasher; 8 | use crate::enc::threading::{ 9 | BatchSpawnable, BatchSpawnableLite, BrotliEncoderThreadError, CompressMulti, 10 | CompressionThreadResult, InternalOwned, InternalSendAlloc, Joinable, Owned, OwnedRetriever, 11 | PoisonedThreadError, SendAlloc, 12 | }; 13 | use crate::enc::{BrotliAlloc, BrotliEncoderParams}; 14 | 15 | pub struct SingleThreadedJoinable { 16 | result: Result, 17 | } 18 | impl Joinable for SingleThreadedJoinable { 19 | fn join(self) -> Result { 20 | self.result 21 | } 22 | } 23 | #[cfg(feature = "std")] 24 | pub struct SingleThreadedOwnedRetriever(std::sync::RwLock); 25 | #[cfg(feature = "std")] 26 | impl OwnedRetriever for SingleThreadedOwnedRetriever { 27 | fn view T>(&self, f: F) -> Result { 28 | Ok(f(&*self.0.read().unwrap())) 29 | } 30 | fn unwrap(self) -> Result { 31 | Ok(self.0.into_inner().unwrap()) 32 | } 33 | } 34 | #[cfg(feature = "std")] 35 | impl SingleThreadedOwnedRetriever { 36 | fn new(u: U) -> Self { 37 | SingleThreadedOwnedRetriever(std::sync::RwLock::new(u)) 38 | } 39 | } 40 | 41 | #[cfg(not(feature = "std"))] 42 | pub struct SingleThreadedOwnedRetriever(U); 43 | #[cfg(not(feature = "std"))] 44 | impl SingleThreadedOwnedRetriever { 45 | fn new(u: U) -> Self { 46 | SingleThreadedOwnedRetriever(u) 47 | } 48 | } 49 | #[cfg(not(feature = "std"))] 50 | impl OwnedRetriever for SingleThreadedOwnedRetriever { 51 | fn view T>(&self, f: F) -> Result { 52 | Ok(f(&self.0)) 53 | } 54 | fn unwrap(self) -> Result { 55 | Ok(self.0) 56 | } 57 | } 58 | 59 | #[derive(Default)] 60 | pub struct SingleThreadedSpawner {} 61 | 62 | impl< 63 | ReturnValue: Send + 'static, 64 | ExtraInput: Send + 'static, 65 | Alloc: BrotliAlloc + Send + 'static, 66 | U: Send + 'static + Sync, 67 | > BatchSpawnable for SingleThreadedSpawner 68 | where 69 | >::AllocatedMemory: Send + 'static, 70 | { 71 | type JoinHandle = SingleThreadedJoinable; 72 | type FinalJoinHandle = SingleThreadedOwnedRetriever; 73 | fn make_spawner(&mut self, input: &mut Owned) -> Self::FinalJoinHandle { 74 | SingleThreadedOwnedRetriever::::new( 75 | mem::replace(input, Owned(InternalOwned::Borrowed)).unwrap(), 76 | ) 77 | } 78 | fn spawn ReturnValue + Send + 'static + Copy>( 79 | &mut self, 80 | handle: &mut Self::FinalJoinHandle, 81 | work: &mut SendAlloc, 82 | index: usize, 83 | num_threads: usize, 84 | f: F, 85 | ) { 86 | let (alloc, extra_input) = work.replace_with_default(); 87 | let ret = handle.view(|sub_view| f(extra_input, index, num_threads, sub_view, alloc)); 88 | *work = SendAlloc(InternalSendAlloc::Join(SingleThreadedJoinable { 89 | result: Ok(ret.unwrap()), 90 | })); 91 | } 92 | } 93 | 94 | impl< 95 | ReturnValue: Send + 'static, 96 | ExtraInput: Send + 'static, 97 | Alloc: BrotliAlloc + Send + 'static, 98 | U: Send + 'static + Sync, 99 | > BatchSpawnableLite for SingleThreadedSpawner 100 | where 101 | >::AllocatedMemory: Send + 'static, 102 | { 103 | type JoinHandle = 104 | >::JoinHandle; 105 | type FinalJoinHandle = >::FinalJoinHandle; 111 | 112 | fn make_spawner(&mut self, input: &mut Owned) -> Self::FinalJoinHandle { 113 | >::make_spawner(self, input) 114 | } 115 | fn spawn( 116 | &mut self, 117 | handle: &mut Self::FinalJoinHandle, 118 | alloc_per_thread: &mut SendAlloc, 119 | index: usize, 120 | num_threads: usize, 121 | f: fn(ExtraInput, usize, usize, &U, Alloc) -> ReturnValue, 122 | ) { 123 | >::spawn( 124 | self, 125 | handle, 126 | alloc_per_thread, 127 | index, 128 | num_threads, 129 | f, 130 | ) 131 | } 132 | } 133 | 134 | pub fn compress_multi< 135 | Alloc: BrotliAlloc + Send + 'static, 136 | SliceW: SliceWrapper + Send + 'static + Sync, 137 | >( 138 | params: &BrotliEncoderParams, 139 | owned_input: &mut Owned, 140 | output: &mut [u8], 141 | alloc_per_thread: &mut [SendAlloc< 142 | CompressionThreadResult, 143 | UnionHasher, 144 | Alloc, 145 | , 147 | UnionHasher, 148 | Alloc, 149 | SliceW, 150 | >>::JoinHandle, 151 | >], 152 | ) -> Result 153 | where 154 | >::AllocatedMemory: Send, 155 | >::AllocatedMemory: Send, 156 | >::AllocatedMemory: Send, 157 | { 158 | CompressMulti( 159 | params, 160 | owned_input, 161 | output, 162 | alloc_per_thread, 163 | &mut SingleThreadedSpawner::default(), 164 | ) 165 | } 166 | 167 | pub struct WorkerPool { 168 | a: PhantomData, 169 | b: PhantomData, 170 | c: PhantomData, 171 | d: PhantomData, 172 | } 173 | pub fn new_work_pool(_num_threads: usize) -> WorkerPool { 174 | WorkerPool:: { 175 | a: PhantomData, 176 | b: PhantomData, 177 | c: PhantomData, 178 | d: PhantomData, 179 | } 180 | } 181 | 182 | pub fn compress_worker_pool< 183 | Alloc: BrotliAlloc + Send + 'static, 184 | SliceW: SliceWrapper + Send + 'static + Sync, 185 | >( 186 | params: &BrotliEncoderParams, 187 | owned_input: &mut Owned, 188 | output: &mut [u8], 189 | alloc_per_thread: &mut [SendAlloc< 190 | CompressionThreadResult, 191 | UnionHasher, 192 | Alloc, 193 | , 195 | UnionHasher, 196 | Alloc, 197 | SliceW, 198 | >>::JoinHandle, 199 | >], 200 | _worker_pool: &mut WorkerPool< 201 | CompressionThreadResult, 202 | UnionHasher, 203 | Alloc, 204 | (SliceW, BrotliEncoderParams), 205 | >, 206 | ) -> Result 207 | where 208 | >::AllocatedMemory: Send, 209 | >::AllocatedMemory: Send, 210 | >::AllocatedMemory: Send, 211 | { 212 | compress_multi(params, owned_input, output, alloc_per_thread) 213 | } 214 | -------------------------------------------------------------------------------- /src/enc/utf8_util.rs: -------------------------------------------------------------------------------- 1 | use crate::enc::floatX; 2 | 3 | fn parse_as_utf8(input: &[u8], size: usize) -> (usize, i32) { 4 | if (input[0] & 0x80) == 0 { 5 | if input[0] > 0 { 6 | return (1, i32::from(input[0])); 7 | } 8 | } 9 | if size > 1 && (input[0] & 0xe0) == 0xc0 && (input[1] & 0xc0) == 0x80 { 10 | let symbol = (input[0] as i32 & 0x1f) << 6 | input[1] as i32 & 0x3f; 11 | if symbol > 0x7f { 12 | return (2, symbol); 13 | } 14 | } 15 | if size > 2 16 | && (input[0] & 0xf0) == 0xe0 17 | && (input[1] & 0xc0) == 0x80 18 | && (input[2] & 0xc0) == 0x80 19 | { 20 | let symbol = (i32::from(input[0]) & 0x0f) << 12 21 | | (i32::from(input[1]) & 0x3f) << 6 22 | | i32::from(input[2]) & 0x3f; 23 | if symbol > 0x7ff { 24 | return (3, symbol); 25 | } 26 | } 27 | if size > 3 28 | && (input[0] & 0xf8) == 0xf0 29 | && (input[1] & 0xc0) == 0x80 30 | && (input[2] & 0xc0) == 0x80 31 | && (input[3] & 0xc0) == 0x80 32 | { 33 | let symbol = (i32::from(input[0]) & 0x07) << 18 34 | | (i32::from(input[1]) & 0x3f) << 12 35 | | (i32::from(input[2]) & 0x3f) << 6 36 | | i32::from(input[3]) & 0x3f; 37 | if symbol > 0xffff && symbol <= 0x10_ffff { 38 | return (4, symbol); 39 | } 40 | } 41 | 42 | (1, 0x11_0000 | i32::from(input[0])) 43 | } 44 | 45 | pub(crate) fn is_mostly_utf8( 46 | data: &[u8], 47 | pos: usize, 48 | mask: usize, 49 | length: usize, 50 | min_fraction: floatX, 51 | ) -> bool { 52 | let mut size_utf8: usize = 0; 53 | let mut i: usize = 0; 54 | while i < length { 55 | let (bytes_read, symbol) = parse_as_utf8(&data[(pos.wrapping_add(i) & mask)..], length - i); 56 | i = i.wrapping_add(bytes_read); 57 | if symbol < 0x11_0000 { 58 | size_utf8 = size_utf8.wrapping_add(bytes_read); 59 | } 60 | } 61 | size_utf8 as floatX > min_fraction * length as floatX 62 | } 63 | -------------------------------------------------------------------------------- /src/enc/util.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::excessive_precision)] 2 | 3 | use crate::enc::log_table_16::logs_16; 4 | use crate::enc::log_table_8::logs_8; 5 | 6 | #[cfg(feature = "float64")] 7 | pub type floatX = f64; 8 | 9 | #[cfg(not(feature = "float64"))] 10 | pub type floatX = f32; 11 | 12 | #[inline(always)] 13 | pub fn FastLog2u16(v: u16) -> floatX { 14 | logs_16[v as usize] 15 | } 16 | 17 | #[cfg(feature = "std")] 18 | #[inline(always)] 19 | pub fn FastLog2(v: u64) -> floatX { 20 | if v < 256 { 21 | logs_8[v as usize] 22 | } else { 23 | (v as f32).log2() as floatX 24 | } 25 | } 26 | 27 | #[cfg(not(feature = "std"))] 28 | #[inline(always)] 29 | pub fn FastLog2(v: u64) -> floatX { 30 | if v < 256 { 31 | logs_8[v as usize] 32 | } else { 33 | FastLog2u64(v) 34 | } 35 | } 36 | 37 | #[cfg(feature = "std")] 38 | #[inline(always)] 39 | pub fn FastLog2f64(v: u64) -> floatX { 40 | if v < 256 { 41 | logs_8[v as usize] 42 | } else { 43 | (v as floatX).log2() 44 | } 45 | } 46 | 47 | #[cfg(not(feature = "std"))] 48 | #[inline(always)] 49 | pub fn FastLog2f64(v: u64) -> floatX { 50 | FastLog2(v) as floatX 51 | } 52 | 53 | #[inline] 54 | pub fn FastLog2u64(v: u64) -> floatX { 55 | let bsr_8 = 56i8 - v.leading_zeros() as i8; 56 | let offset = bsr_8 & -((bsr_8 >= 0) as i8); 57 | (offset as floatX) + logs_8[(v >> offset) as usize] 58 | } 59 | 60 | #[inline(always)] 61 | pub fn FastLog2u32(v: i32) -> floatX { 62 | let bsr_8 = 24i8 - v.leading_zeros() as i8; 63 | let offset = bsr_8 & -((bsr_8 >= 0) as i8); 64 | (offset as floatX) + logs_8[(v >> offset) as usize] 65 | } 66 | 67 | #[inline(always)] 68 | pub fn xFastLog2u16(v: u16) -> floatX { 69 | let bsr_8 = 8i8 - v.leading_zeros() as i8; 70 | let offset = (bsr_8 & -((bsr_8 >= 0) as i8)); 71 | (offset as floatX) + logs_8[(v >> offset) as usize] 72 | } 73 | 74 | #[cfg(feature = "std")] 75 | #[inline(always)] 76 | pub fn FastPow2(v: floatX) -> floatX { 77 | (2 as floatX).powf(v) 78 | } 79 | 80 | #[cfg(not(feature = "std"))] 81 | #[inline(always)] 82 | pub fn FastPow2(v: floatX) -> floatX { 83 | assert!(v >= 0 as floatX); 84 | let round_down = v as i32; 85 | let remainder = v - round_down as floatX; 86 | let mut x = 1 as floatX; 87 | // (1 + (x/n) * ln2) ^ n 88 | // let n = 8 89 | x += remainder * (0.693147180559945309417232121458 / 256.0) as floatX; 90 | x *= x; 91 | x *= x; 92 | x *= x; 93 | x *= x; 94 | x *= x; 95 | x *= x; 96 | x *= x; 97 | x *= x; 98 | 99 | (1 << round_down) as floatX * x 100 | } 101 | 102 | #[inline(always)] 103 | pub fn Log2FloorNonZero(v: u64) -> u32 { 104 | 63u32 ^ v.leading_zeros() 105 | } 106 | 107 | #[cfg(test)] 108 | mod test { 109 | fn baseline_log2_floor_non_zero(mut n: u64) -> u32 { 110 | let mut result: u32 = 0; 111 | while { 112 | n >>= 1i32; 113 | n 114 | } != 0 115 | { 116 | result = result.wrapping_add(1); 117 | } 118 | result 119 | } 120 | 121 | #[test] 122 | fn log2floor_non_zero_works() { 123 | let examples = [ 124 | 4u64, 125 | 254, 126 | 256, 127 | 1428, 128 | 25412509, 129 | 21350891256, 130 | 65536, 131 | 1258912591, 132 | 60968101, 133 | 1, 134 | 12589125190825, 135 | 105912059215091, 136 | 0, 137 | ]; 138 | for example in examples.iter() { 139 | let fast_version = super::Log2FloorNonZero(*example); 140 | let baseline_version = baseline_log2_floor_non_zero(*example); 141 | if *example != 0 { 142 | // make sure we don't panic when computing...but don't care about result 143 | assert_eq!(fast_version, baseline_version); 144 | } 145 | } 146 | } 147 | pub fn approx_eq(a: f64, b: f64, tol: f64) { 148 | let mut t0 = a - b; 149 | let mut t1 = b - a; 150 | if t0 < 0.0 { 151 | t0 = -t0; 152 | } 153 | if t1 < 0.0 { 154 | t1 = -t1; 155 | } 156 | if (!(t1 < tol)) { 157 | assert_eq!(a, b); 158 | } 159 | if (!(t0 < tol)) { 160 | assert_eq!(a, b); 161 | } 162 | } 163 | #[test] 164 | fn fast_log2_works() { 165 | let examples = [ 166 | 4u64, 167 | 254, 168 | 256, 169 | 1428, 170 | 25412509, 171 | 21350891256, 172 | 65536, 173 | 1258912591, 174 | 60968101, 175 | 1, 176 | 12589125190825, 177 | 105912059215091, 178 | 0, 179 | ]; 180 | let tol = [ 181 | 0.00001, 0.0001, 0.0001, 0.005, 0.007, 0.008, 0.01, 0.01, 0.01, 0.000001, 0.01, 0.01, 182 | 0.0001, 183 | ]; 184 | for (index, example) in examples.iter().enumerate() { 185 | let fast_version = super::FastLog2(*example); 186 | if *example != 0 { 187 | // make sure we don't panic when computing...but don't care about result 188 | let baseline_version = (*example as f64).log2(); 189 | approx_eq(fast_version as f64, baseline_version, tol[index]); 190 | } else { 191 | //assert_eq!(fast_version as f64, 0.0 as f64); 192 | } 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/enc/vectorization.rs: -------------------------------------------------------------------------------- 1 | #![allow(unknown_lints)] 2 | #![allow(unused_macros)] 3 | 4 | use crate::enc::util::FastLog2; 5 | use crate::enc::{s8, v8}; 6 | pub type Mem256f = v8; 7 | pub type Mem256i = s8; 8 | pub type v256 = v8; 9 | pub type v256i = s8; 10 | pub fn sum8(x: v256) -> f32 { 11 | x[0] + x[1] + x[2] + x[3] + x[4] + x[5] + x[6] + x[7] 12 | } 13 | 14 | pub fn sum8i(x: v256i) -> i32 { 15 | x[0].wrapping_add(x[1]) 16 | .wrapping_add(x[2]) 17 | .wrapping_add(x[3]) 18 | .wrapping_add(x[4]) 19 | .wrapping_add(x[5]) 20 | .wrapping_add(x[6]) 21 | .wrapping_add(x[7]) 22 | } 23 | 24 | pub fn log2i(x: v256i) -> v256 { 25 | [ 26 | FastLog2(x[0] as u64), 27 | FastLog2(x[1] as u64), 28 | FastLog2(x[2] as u64), 29 | FastLog2(x[3] as u64), 30 | FastLog2(x[4] as u64), 31 | FastLog2(x[5] as u64), 32 | FastLog2(x[6] as u64), 33 | FastLog2(x[7] as u64), 34 | ] 35 | .into() 36 | } 37 | pub fn cast_i32_to_f32(x: v256i) -> v256 { 38 | [ 39 | x[0] as f32, 40 | x[1] as f32, 41 | x[2] as f32, 42 | x[3] as f32, 43 | x[4] as f32, 44 | x[5] as f32, 45 | x[6] as f32, 46 | x[7] as f32, 47 | ] 48 | .into() 49 | } 50 | pub fn cast_f32_to_i32(x: v256) -> v256i { 51 | [ 52 | x[0] as i32, 53 | x[1] as i32, 54 | x[2] as i32, 55 | x[3] as i32, 56 | x[4] as i32, 57 | x[5] as i32, 58 | x[6] as i32, 59 | x[7] as i32, 60 | ] 61 | .into() 62 | } 63 | -------------------------------------------------------------------------------- /src/enc/weights.rs: -------------------------------------------------------------------------------- 1 | use core::cmp::{max, min}; 2 | pub type Prob = u16; 3 | 4 | pub const BLEND_FIXED_POINT_PRECISION: i8 = 15; 5 | #[allow(dead_code)] 6 | pub const LOG2_SCALE: i32 = 15; 7 | #[derive(Debug, Copy, Clone)] 8 | pub struct Weights { 9 | model_weights: [i32; 2], 10 | mixing_param: u8, 11 | normalized_weight: Prob, 12 | } 13 | impl Default for Weights { 14 | fn default() -> Self { 15 | Self::new() 16 | } 17 | } 18 | impl Weights { 19 | pub fn new() -> Self { 20 | Weights { 21 | model_weights: [1; 2], 22 | mixing_param: 1, 23 | normalized_weight: 1 << (BLEND_FIXED_POINT_PRECISION - 1), 24 | } 25 | } 26 | #[allow(unused)] 27 | #[inline(always)] 28 | pub fn update(&mut self, model_probs: [Prob; 2], weighted_prob: Prob) { 29 | debug_assert!(self.mixing_param != 0); 30 | normalize_weights(&mut self.model_weights); 31 | let w0new = compute_new_weight( 32 | model_probs, 33 | weighted_prob, 34 | self.model_weights, 35 | false, 36 | self.mixing_param - 1, 37 | ); 38 | let w1new = compute_new_weight( 39 | model_probs, 40 | weighted_prob, 41 | self.model_weights, 42 | true, 43 | self.mixing_param - 1, 44 | ); 45 | self.model_weights = [w0new, w1new]; 46 | self.normalized_weight = compute_normalized_weight(self.model_weights); 47 | } 48 | #[allow(dead_code)] 49 | #[inline(always)] 50 | pub fn norm_weight(&self) -> Prob { 51 | self.normalized_weight 52 | } 53 | } 54 | 55 | #[allow(dead_code)] 56 | #[inline(always)] 57 | fn compute_normalized_weight(model_weights: [i32; 2]) -> Prob { 58 | let total = i64::from(model_weights[0]) + i64::from(model_weights[1]); 59 | let leading_zeros = total.leading_zeros(); 60 | let shift = max(56 - (leading_zeros as i8), 0); 61 | let total_8bit = total >> shift; 62 | /*::probability::numeric::fast_divide_16bit_by_8bit( 63 | ((model_weights[0] >> shift) as u16)<< 8, 64 | ::probability::numeric::lookup_divisor8(total_8bit as u8)) << (BLEND_FIXED_POINT_PRECISION - 8) 65 | */ 66 | ((((model_weights[0] >> shift) as u16) << 8) / total_8bit as u16/*fixme??*/) 67 | << (BLEND_FIXED_POINT_PRECISION - 8) 68 | } 69 | 70 | #[allow(dead_code)] 71 | #[cold] 72 | fn fix_weights(weights: &mut [i32; 2]) { 73 | let ilog = 32 - min(weights[0].leading_zeros(), weights[1].leading_zeros()); 74 | let max_log = 24; 75 | if ilog >= max_log { 76 | weights[0] >>= ilog - max_log; 77 | weights[1] >>= ilog - max_log; 78 | } 79 | } 80 | 81 | #[allow(dead_code)] 82 | #[inline(always)] 83 | fn normalize_weights(weights: &mut [i32; 2]) { 84 | if ((weights[0] | weights[1]) & 0x7f00_0000) != 0 { 85 | fix_weights(weights); 86 | } 87 | } 88 | 89 | #[allow(dead_code)] 90 | #[cfg(feature = "floating_point_context_mixing")] 91 | fn compute_new_weight( 92 | probs: [Prob; 2], 93 | weighted_prob: Prob, 94 | weights: [i32; 2], 95 | index_equal_1: bool, 96 | _speed: u8, 97 | ) -> i32 { 98 | // speed ranges from 1 to 14 inclusive 99 | let index = index_equal_1 as usize; 100 | let n1i = probs[index] as f64 / ((1i64 << LOG2_SCALE) as f64); 101 | //let n0i = 1.0f64 - n1i; 102 | let ni = 1.0f64; 103 | let s1 = weighted_prob as f64 / ((1i64 << LOG2_SCALE) as f64); 104 | let s0 = 1.0f64 - s1; 105 | let s = 1.0f64; 106 | //let p0 = s0; 107 | let p1 = s1; 108 | let wi = weights[index] as f64 / ((1i64 << LOG2_SCALE) as f64); 109 | let mut wi_new = wi + (1.0 - p1) * (s * n1i - s1 * ni) / (s0 * s1); 110 | let eps = 0.00001f64; 111 | if !(wi_new > eps) { 112 | wi_new = eps; 113 | } 114 | (wi_new * ((1i64 << LOG2_SCALE) as f64)) as i32 115 | } 116 | 117 | #[allow(dead_code)] 118 | #[cfg(not(feature = "floating_point_context_mixing"))] 119 | #[inline(always)] 120 | fn compute_new_weight( 121 | probs: [Prob; 2], 122 | weighted_prob: Prob, 123 | weights: [i32; 2], 124 | index_equal_1: bool, 125 | _speed: u8, 126 | ) -> i32 { 127 | // speed ranges from 1 to 14 inclusive 128 | let index = index_equal_1 as usize; 129 | let full_model_sum_p1 = i64::from(weighted_prob); 130 | let full_model_total = 1i64 << LOG2_SCALE; 131 | let full_model_sum_p0 = full_model_total.wrapping_sub(i64::from(weighted_prob)); 132 | let n1i = i64::from(probs[index]); 133 | let ni = 1i64 << LOG2_SCALE; 134 | let error = full_model_total.wrapping_sub(full_model_sum_p1); 135 | let wi = i64::from(weights[index]); 136 | let efficacy = full_model_total.wrapping_mul(n1i) - full_model_sum_p1.wrapping_mul(ni); 137 | //let geometric_probabilities = full_model_sum_p1 * full_model_sum_p0; 138 | let log_geometric_probabilities = 139 | 64 - (full_model_sum_p1.wrapping_mul(full_model_sum_p0)).leading_zeros(); 140 | //let scaled_geometric_probabilities = geometric_probabilities * S; 141 | //let new_weight_adj = (error * efficacy) >> log_geometric_probabilities;// / geometric_probabilities; 142 | //let new_weight_adj = (error * efficacy)/(full_model_sum_p1 * full_model_sum_p0); 143 | let new_weight_adj = (error.wrapping_mul(efficacy)) >> log_geometric_probabilities; 144 | // assert!(wi + new_weight_adj < (1i64 << 31)); 145 | //print!("{} -> {} due to {:?} vs {}\n", wi as f64 / (weights[0] + weights[1]) as f64, (wi + new_weight_adj) as f64 /(weights[0] as i64 + new_weight_adj as i64 + weights[1] as i64) as f64, probs[index], weighted_prob); 146 | max(1, wi.wrapping_add(new_weight_adj) as i32) 147 | } 148 | -------------------------------------------------------------------------------- /src/ffi/alloc_util.rs: -------------------------------------------------------------------------------- 1 | use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; 2 | 3 | use brotli_decompressor::ffi::alloc_util::SubclassableAllocator; 4 | 5 | use crate::enc::BrotliAlloc; 6 | 7 | pub struct BrotliSubclassableAllocator(SubclassableAllocator); 8 | 9 | impl BrotliSubclassableAllocator { 10 | pub fn new(s: SubclassableAllocator) -> BrotliSubclassableAllocator { 11 | BrotliSubclassableAllocator(s) 12 | } 13 | } 14 | 15 | #[derive(Default)] 16 | pub struct SendableMemoryBlock( 17 | >::AllocatedMemory, 18 | ); 19 | impl SliceWrapperMut for SendableMemoryBlock { 20 | fn slice_mut(&mut self) -> &mut [T] { 21 | self.0.slice_mut() 22 | } 23 | } 24 | impl SliceWrapper for SendableMemoryBlock { 25 | fn slice(&self) -> &[T] { 26 | self.0.slice() 27 | } 28 | } 29 | impl Allocator for BrotliSubclassableAllocator { 30 | type AllocatedMemory = SendableMemoryBlock; 31 | fn alloc_cell(&mut self, s: usize) -> Self::AllocatedMemory { 32 | SendableMemoryBlock(self.0.alloc_cell(s)) 33 | } 34 | fn free_cell(&mut self, data: Self::AllocatedMemory) { 35 | self.0.free_cell(data.0) 36 | } 37 | } 38 | 39 | impl BrotliAlloc for BrotliSubclassableAllocator {} 40 | #[cfg(not(feature = "safe"))] 41 | unsafe impl Send for BrotliSubclassableAllocator {} 42 | 43 | #[cfg(not(feature = "safe"))] 44 | unsafe impl Send for SendableMemoryBlock {} 45 | 46 | #[cfg(not(feature = "std"))] 47 | #[cfg(feature = "no-stdlib-ffi-binding")] 48 | #[panic_handler] 49 | extern "C" fn panic_impl(_: &::core::panic::PanicInfo) -> ! { 50 | loop {} 51 | } 52 | 53 | #[cfg(not(feature = "std"))] 54 | #[cfg(feature = "no-stdlib-ffi-binding")] 55 | #[lang = "eh_personality"] 56 | extern "C" fn eh_personality() {} 57 | -------------------------------------------------------------------------------- /src/ffi/broccoli.rs: -------------------------------------------------------------------------------- 1 | use core; 2 | 3 | pub use brotli_decompressor::ffi::interface::c_void; 4 | use brotli_decompressor::ffi::{slice_from_raw_parts_or_nil, slice_from_raw_parts_or_nil_mut}; 5 | 6 | use crate::concat::BroCatli; 7 | pub use crate::concat::BroCatliResult; 8 | pub type BroccoliResult = BroCatliResult; 9 | // a tool to concatenate brotli files together 10 | 11 | #[repr(C)] 12 | pub struct BroccoliState { 13 | more_data: *mut c_void, 14 | current_data: [u8; 120], 15 | } 16 | 17 | impl Clone for BroccoliState { 18 | fn clone(&self) -> BroccoliState { 19 | let mut cd = [0u8; 120]; 20 | cd.clone_from_slice(&self.current_data[..]); 21 | BroccoliState { 22 | more_data: self.more_data, 23 | current_data: cd, 24 | } 25 | } 26 | } 27 | 28 | impl Copy for BroccoliState {} 29 | 30 | impl Default for BroccoliState { 31 | fn default() -> BroccoliState { 32 | BroCatli::new().into() 33 | } 34 | } 35 | impl From for BroccoliState { 36 | fn from(data: BroCatli) -> BroccoliState { 37 | let mut buffer = [0u8; 120]; 38 | data.serialize_to_buffer(&mut buffer[..]).unwrap(); 39 | BroccoliState { 40 | more_data: core::ptr::null_mut(), 41 | current_data: buffer, 42 | } 43 | } 44 | } 45 | impl From for BroCatli { 46 | fn from(val: BroccoliState) -> Self { 47 | BroCatli::deserialize_from_buffer(&val.current_data[..]).unwrap() 48 | } 49 | } 50 | 51 | #[no_mangle] 52 | pub extern "C" fn BroccoliCreateInstance() -> BroccoliState { 53 | BroCatli::new().into() 54 | } 55 | #[no_mangle] 56 | pub extern "C" fn BroccoliCreateInstanceWithWindowSize(window_size: u8) -> BroccoliState { 57 | BroCatli::new_with_window_size(window_size).into() 58 | } 59 | #[no_mangle] 60 | pub extern "C" fn BroccoliDestroyInstance(_state: BroccoliState) {} 61 | 62 | #[no_mangle] 63 | pub unsafe extern "C" fn BroccoliNewBrotliFile(state: *mut BroccoliState) { 64 | let mut bro_catli: BroCatli = (*state).into(); 65 | bro_catli.new_brotli_file(); 66 | *state = BroccoliState::from(bro_catli); 67 | } 68 | 69 | #[no_mangle] 70 | pub unsafe extern "C" fn BroccoliConcatStream( 71 | state: *mut BroccoliState, 72 | available_in: *mut usize, 73 | input_buf_ptr: *mut *const u8, 74 | available_out: *mut usize, 75 | output_buf_ptr: *mut *mut u8, 76 | ) -> BroccoliResult { 77 | let input_buf = slice_from_raw_parts_or_nil(*input_buf_ptr, *available_in); 78 | let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); 79 | let mut input_offset = 0usize; 80 | let mut output_offset = 0usize; 81 | let mut bro_catli: BroCatli = (*state).into(); 82 | let ret = bro_catli.stream(input_buf, &mut input_offset, output_buf, &mut output_offset); 83 | *input_buf_ptr = (*input_buf_ptr).add(input_offset); 84 | *output_buf_ptr = (*output_buf_ptr).add(output_offset); 85 | *available_in -= input_offset; 86 | *available_out -= output_offset; 87 | *state = BroccoliState::from(bro_catli); 88 | ret 89 | } 90 | 91 | #[no_mangle] 92 | pub unsafe extern "C" fn BroccoliConcatStreaming( 93 | state: *mut BroccoliState, 94 | available_in: *mut usize, 95 | mut input_buf: *const u8, 96 | available_out: *mut usize, 97 | mut output_buf: *mut u8, 98 | ) -> BroccoliResult { 99 | BroccoliConcatStream( 100 | state, 101 | available_in, 102 | &mut input_buf, 103 | available_out, 104 | &mut output_buf, 105 | ) 106 | } 107 | 108 | #[no_mangle] 109 | pub unsafe extern "C" fn BroccoliConcatFinish( 110 | state: *mut BroccoliState, 111 | available_out: *mut usize, 112 | output_buf_ptr: *mut *mut u8, 113 | ) -> BroCatliResult { 114 | let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); 115 | let mut output_offset = 0usize; 116 | let mut bro_catli: BroCatli = (*state).into(); 117 | let ret = bro_catli.finish(output_buf, &mut output_offset); 118 | *output_buf_ptr = (*output_buf_ptr).add(output_offset); 119 | *available_out -= output_offset; 120 | *state = BroccoliState::from(bro_catli); 121 | ret 122 | } 123 | 124 | // exactly the same as BrotliConcatFinish but without the indirect 125 | #[no_mangle] 126 | pub unsafe extern "C" fn BroccoliConcatFinished( 127 | state: *mut BroccoliState, 128 | available_out: *mut usize, 129 | mut output_buf: *mut u8, 130 | ) -> BroCatliResult { 131 | BroccoliConcatFinish(state, available_out, &mut output_buf) 132 | } 133 | -------------------------------------------------------------------------------- /src/ffi/decompressor.rs: -------------------------------------------------------------------------------- 1 | pub use brotli_decompressor::ffi::interface::{brotli_alloc_func, brotli_free_func, c_void}; 2 | pub use brotli_decompressor::{ffi, BrotliDecoderReturnInfo, HuffmanCode}; 3 | 4 | pub unsafe extern "C" fn CBrotliDecoderCreateInstance( 5 | alloc_func: brotli_alloc_func, 6 | free_func: brotli_free_func, 7 | opaque: *mut c_void, 8 | ) -> *mut ffi::BrotliDecoderState { 9 | ffi::BrotliDecoderCreateInstance(alloc_func, free_func, opaque) 10 | } 11 | 12 | pub unsafe extern "C" fn CBrotliDecoderSetParameter( 13 | state_ptr: *mut ffi::BrotliDecoderState, 14 | selector: ffi::interface::BrotliDecoderParameter, 15 | value: u32, 16 | ) { 17 | ffi::BrotliDecoderSetParameter(state_ptr, selector, value) 18 | } 19 | 20 | #[cfg(feature = "std")] // this requires a default allocator 21 | pub unsafe extern "C" fn CBrotliDecoderDecompress( 22 | encoded_size: usize, 23 | encoded_buffer: *const u8, 24 | decoded_size: *mut usize, 25 | decoded_buffer: *mut u8, 26 | ) -> ffi::interface::BrotliDecoderResult { 27 | ffi::BrotliDecoderDecompress(encoded_size, encoded_buffer, decoded_size, decoded_buffer) 28 | } 29 | 30 | pub unsafe extern "C" fn CBrotliDecoderDecompressStream( 31 | state_ptr: *mut ffi::BrotliDecoderState, 32 | available_in: *mut usize, 33 | input_buf_ptr: *mut *const u8, 34 | available_out: *mut usize, 35 | output_buf_ptr: *mut *mut u8, 36 | total_out: *mut usize, 37 | ) -> ffi::interface::BrotliDecoderResult { 38 | ffi::BrotliDecoderDecompressStream( 39 | state_ptr, 40 | available_in, 41 | input_buf_ptr, 42 | available_out, 43 | output_buf_ptr, 44 | total_out, 45 | ) 46 | } 47 | 48 | pub unsafe extern "C" fn CBrotliDecoderDecompressStreaming( 49 | state_ptr: *mut ffi::BrotliDecoderState, 50 | available_in: *mut usize, 51 | input_buf_ptr: *const u8, 52 | available_out: *mut usize, 53 | output_buf_ptr: *mut u8, 54 | ) -> ffi::interface::BrotliDecoderResult { 55 | ffi::BrotliDecoderDecompressStreaming( 56 | state_ptr, 57 | available_in, 58 | input_buf_ptr, 59 | available_out, 60 | output_buf_ptr, 61 | ) 62 | } 63 | 64 | pub unsafe extern "C" fn CBrotliDecoderDecompressWithReturnInfo( 65 | available_in: usize, 66 | input_buf_ptr: *const u8, 67 | available_out_and_scratch: usize, 68 | output_buf_and_scratch: *mut u8, 69 | ) -> BrotliDecoderReturnInfo { 70 | ffi::BrotliDecoderDecompressWithReturnInfo( 71 | available_in, 72 | input_buf_ptr, 73 | available_out_and_scratch, 74 | output_buf_and_scratch, 75 | ) 76 | } 77 | 78 | pub unsafe extern "C" fn CBrotliDecoderDecompressPrealloc( 79 | available_in: usize, 80 | input_buf_ptr: *const u8, 81 | available_out: usize, 82 | output_buf_ptr: *mut u8, 83 | available_u8: usize, 84 | u8_ptr: *mut u8, 85 | available_u32: usize, 86 | u32_ptr: *mut u32, 87 | available_hc: usize, 88 | hc_ptr: *mut HuffmanCode, 89 | ) -> BrotliDecoderReturnInfo { 90 | ffi::BrotliDecoderDecompressPrealloc( 91 | available_in, 92 | input_buf_ptr, 93 | available_out, 94 | output_buf_ptr, 95 | available_u8, 96 | u8_ptr, 97 | available_u32, 98 | u32_ptr, 99 | available_hc, 100 | hc_ptr, 101 | ) 102 | } 103 | 104 | pub unsafe extern "C" fn CBrotliDecoderMallocU8( 105 | state_ptr: *mut ffi::BrotliDecoderState, 106 | size: usize, 107 | ) -> *mut u8 { 108 | ffi::BrotliDecoderMallocU8(state_ptr, size) 109 | } 110 | 111 | pub unsafe extern "C" fn CBrotliDecoderFreeU8( 112 | state_ptr: *mut ffi::BrotliDecoderState, 113 | data: *mut u8, 114 | size: usize, 115 | ) { 116 | ffi::BrotliDecoderFreeU8(state_ptr, data, size) 117 | } 118 | 119 | pub unsafe extern "C" fn CBrotliDecoderMallocUsize( 120 | state_ptr: *mut ffi::BrotliDecoderState, 121 | size: usize, 122 | ) -> *mut usize { 123 | ffi::BrotliDecoderMallocUsize(state_ptr, size) 124 | } 125 | 126 | pub unsafe extern "C" fn CBrotliDecoderFreeUsize( 127 | state_ptr: *mut ffi::BrotliDecoderState, 128 | data: *mut usize, 129 | size: usize, 130 | ) { 131 | ffi::BrotliDecoderFreeUsize(state_ptr, data, size) 132 | } 133 | 134 | pub unsafe extern "C" fn CBrotliDecoderDestroyInstance(state_ptr: *mut ffi::BrotliDecoderState) { 135 | ffi::BrotliDecoderDestroyInstance(state_ptr) 136 | } 137 | 138 | pub extern "C" fn CBrotliDecoderVersion() -> u32 { 139 | ffi::BrotliDecoderVersion() 140 | } 141 | 142 | #[no_mangle] 143 | pub extern "C" fn CBrotliDecoderErrorString(c: ffi::BrotliDecoderErrorCode) -> *const u8 { 144 | ffi::BrotliDecoderErrorString(c) 145 | } 146 | 147 | #[no_mangle] 148 | pub unsafe extern "C" fn CBrotliDecoderHasMoreOutput( 149 | state_ptr: *const ffi::BrotliDecoderState, 150 | ) -> i32 { 151 | ffi::BrotliDecoderHasMoreOutput(state_ptr) 152 | } 153 | 154 | #[no_mangle] 155 | pub unsafe extern "C" fn CBrotliDecoderTakeOutput( 156 | state_ptr: *mut ffi::BrotliDecoderState, 157 | size: *mut usize, 158 | ) -> *const u8 { 159 | ffi::BrotliDecoderTakeOutput(state_ptr, size) 160 | } 161 | 162 | #[no_mangle] 163 | pub unsafe extern "C" fn CBrotliDecoderIsUsed(state_ptr: *const ffi::BrotliDecoderState) -> i32 { 164 | ffi::BrotliDecoderIsUsed(state_ptr) 165 | } 166 | #[no_mangle] 167 | pub unsafe extern "C" fn CBrotliDecoderIsFinished( 168 | state_ptr: *const ffi::BrotliDecoderState, 169 | ) -> i32 { 170 | ffi::BrotliDecoderIsFinished(state_ptr) 171 | } 172 | #[no_mangle] 173 | pub unsafe extern "C" fn CBrotliDecoderGetErrorCode( 174 | state_ptr: *const ffi::BrotliDecoderState, 175 | ) -> ffi::BrotliDecoderErrorCode { 176 | ffi::BrotliDecoderGetErrorCode(state_ptr) 177 | } 178 | #[no_mangle] 179 | pub unsafe extern "C" fn CBrotliDecoderGetErrorString( 180 | state_ptr: *const ffi::BrotliDecoderState, 181 | ) -> *const u8 { 182 | ffi::BrotliDecoderGetErrorString(state_ptr) 183 | } 184 | -------------------------------------------------------------------------------- /src/ffi/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod alloc_util; 2 | pub mod broccoli; 3 | pub mod compressor; 4 | pub mod decompressor; 5 | pub mod multicompress; 6 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![allow(non_snake_case)] 3 | #![allow(unused_parens)] 4 | #![allow(non_camel_case_types)] 5 | #![allow(non_snake_case)] 6 | #![allow(non_upper_case_globals)] 7 | #![cfg_attr(feature = "benchmark", feature(test))] 8 | #![cfg_attr(feature = "simd", feature(portable_simd))] 9 | #![cfg_attr( 10 | feature = "no-stdlib-ffi-binding", 11 | cfg_attr(not(feature = "std"), feature(lang_items)) 12 | )] 13 | 14 | #[macro_use] 15 | // <-- for debugging, remove xprintln from bit_reader and replace with println 16 | #[cfg(feature = "std")] 17 | extern crate std; 18 | extern crate alloc_no_stdlib as alloc; 19 | #[cfg(feature = "std")] 20 | extern crate alloc_stdlib; 21 | extern crate brotli_decompressor; 22 | 23 | pub mod concat; 24 | pub mod enc; 25 | #[cfg(feature = "ffi-api")] 26 | pub mod ffi; 27 | 28 | pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator}; 29 | 30 | #[cfg(feature = "std")] 31 | pub use alloc_stdlib::HeapAlloc; 32 | #[cfg(feature = "std")] 33 | pub use brotli_decompressor::copy_from_to; 34 | pub use brotli_decompressor::io_wrappers::{CustomRead, CustomWrite}; 35 | #[cfg(feature = "std")] 36 | pub use brotli_decompressor::io_wrappers::{IntoIoReader, IoReaderWrapper, IoWriterWrapper}; 37 | #[cfg(feature = "std")] 38 | pub use brotli_decompressor::reader::Decompressor; 39 | pub use brotli_decompressor::reader::DecompressorCustomIo; 40 | pub use brotli_decompressor::transform::TransformDictionaryWord; 41 | #[cfg(feature = "std")] 42 | pub use brotli_decompressor::writer::DecompressorWriter; 43 | pub use brotli_decompressor::writer::DecompressorWriterCustomIo; 44 | #[cfg(feature = "std")] 45 | pub use brotli_decompressor::BrotliDecompress; 46 | #[cfg(feature = "std")] 47 | pub use brotli_decompressor::BrotliDecompressCustomAlloc; 48 | pub use brotli_decompressor::HuffmanCode; // so we can make custom allocator for decompression 49 | pub use brotli_decompressor::{ 50 | dictionary, reader, transform, writer, BrotliDecompressCustomIo, 51 | BrotliDecompressCustomIoCustomDict, BrotliDecompressStream, BrotliResult, BrotliState, 52 | }; 53 | 54 | pub use self::enc::combined_alloc::CombiningAllocator; 55 | pub use crate::enc::input_pair::{InputPair, InputReference, InputReferenceMut}; 56 | pub use crate::enc::interface::SliceOffset; 57 | #[cfg(feature = "std")] 58 | pub use crate::enc::reader::CompressorReader; 59 | pub use crate::enc::reader::CompressorReaderCustomIo; 60 | #[cfg(feature = "std")] 61 | pub use crate::enc::writer::CompressorWriter; 62 | pub use crate::enc::writer::CompressorWriterCustomIo; 63 | pub use crate::enc::{interface, BrotliCompressCustomIo, BrotliCompressCustomIoCustomDict}; 64 | #[cfg(feature = "std")] 65 | pub use crate::enc::{BrotliCompress, BrotliCompressCustomAlloc}; 66 | 67 | pub const VERSION: u8 = 1; 68 | 69 | // interface 70 | // pub fn BrotliDecompressStream(mut available_in: &mut usize, 71 | // input_offset: &mut usize, 72 | // input: &[u8], 73 | // mut available_out: &mut usize, 74 | // mut output_offset: &mut usize, 75 | // mut output: &mut [u8], 76 | // mut total_out: &mut usize, 77 | // mut s: &mut BrotliState); 78 | -------------------------------------------------------------------------------- /testdata/.gitattributes: -------------------------------------------------------------------------------- 1 | * binary 2 | -------------------------------------------------------------------------------- /testdata/10x10y: -------------------------------------------------------------------------------- 1 | XXXXXXXXXXYYYYYYYYYY -------------------------------------------------------------------------------- /testdata/64x: -------------------------------------------------------------------------------- 1 | XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -------------------------------------------------------------------------------- /testdata/64x.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/64x.compressed -------------------------------------------------------------------------------- /testdata/alice29.txt.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/alice29.txt.compressed -------------------------------------------------------------------------------- /testdata/asyoulik.txt.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/asyoulik.txt.compressed -------------------------------------------------------------------------------- /testdata/backward65536.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/backward65536.compressed -------------------------------------------------------------------------------- /testdata/compressed_file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/compressed_file -------------------------------------------------------------------------------- /testdata/compressed_file.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/compressed_file.compressed -------------------------------------------------------------------------------- /testdata/compressed_repeated: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/compressed_repeated -------------------------------------------------------------------------------- /testdata/compressed_repeated.compressed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/compressed_repeated.compressed -------------------------------------------------------------------------------- /testdata/empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty -------------------------------------------------------------------------------- /testdata/empty.compressed: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/empty.compressed.00: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/empty.compressed.01: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.01 -------------------------------------------------------------------------------- /testdata/empty.compressed.02: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.02 -------------------------------------------------------------------------------- /testdata/empty.compressed.03: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.03 -------------------------------------------------------------------------------- /testdata/empty.compressed.04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.04 -------------------------------------------------------------------------------- /testdata/empty.compressed.05: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.05 -------------------------------------------------------------------------------- /testdata/empty.compressed.06: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.06 -------------------------------------------------------------------------------- /testdata/empty.compressed.07: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.07 -------------------------------------------------------------------------------- /testdata/empty.compressed.08: -------------------------------------------------------------------------------- 1 | 3 -------------------------------------------------------------------------------- /testdata/empty.compressed.09: -------------------------------------------------------------------------------- 1 | 5 -------------------------------------------------------------------------------- /testdata/empty.compressed.10: -------------------------------------------------------------------------------- 1 | 7 -------------------------------------------------------------------------------- /testdata/empty.compressed.11: -------------------------------------------------------------------------------- 1 | 9 -------------------------------------------------------------------------------- /testdata/empty.compressed.12: -------------------------------------------------------------------------------- 1 | ; -------------------------------------------------------------------------------- /testdata/empty.compressed.13: -------------------------------------------------------------------------------- 1 | = -------------------------------------------------------------------------------- /testdata/empty.compressed.14: -------------------------------------------------------------------------------- 1 | ? -------------------------------------------------------------------------------- /testdata/empty.compressed.15: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/empty.compressed.16: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/empty.compressed.16 -------------------------------------------------------------------------------- /testdata/ends_with_truncated_dictionary: -------------------------------------------------------------------------------- 1 | often referred to as -------------------------------------------------------------------------------- /testdata/monkey: -------------------------------------------------------------------------------- 1 | znxcvnmz,xvnm.,zxcnv.,xcn.z,vn.zvn.zxcvn.,zxcn.vn.v,znm.,vnzx.,vnzxc.vn.z,vnz.,nv.z,nvmzxc,nvzxcvcnm.,vczxvnzxcnvmxc.zmcnvzm.,nvmc,nzxmc,vn.mnnmzxc,vnxcnmv,znvzxcnmv,.xcnvm,zxcnzxv.zx,qweryweurqioweupropqwutioweupqrioweutiopweuriopweuriopqwurioputiopqwuriowuqerioupqweropuweropqwurweuqriopuropqwuriopuqwriopuqweopruioqweurqweuriouqweopruioupqiytioqtyiowtyqptypryoqweutioioqtweqruowqeytiowquiourowetyoqwupiotweuqiorweuqroipituqwiorqwtioweuriouytuioerytuioweryuitoweytuiweyuityeruirtyuqriqweuropqweiruioqweurioqwuerioqwyuituierwotueryuiotweyrtuiwertyioweryrueioqptyioruyiopqwtjkasdfhlafhlasdhfjklashjkfhasjklfhklasjdfhklasdhfjkalsdhfklasdhjkflahsjdkfhklasfhjkasdfhasfjkasdhfklsdhalghhaf;hdklasfhjklashjklfasdhfasdjklfhsdjklafsd;hkldadfjjklasdhfjasddfjklfhakjklasdjfkl;asdjfasfljasdfhjklasdfhjkaghjkashf;djfklasdjfkljasdklfjklasdjfkljasdfkljaklfj -------------------------------------------------------------------------------- /testdata/quickfox: -------------------------------------------------------------------------------- 1 | The quick brown fox jumps over the lazy dog -------------------------------------------------------------------------------- /testdata/random_org_10k.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/random_org_10k.bin -------------------------------------------------------------------------------- /testdata/random_then_unicode: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/random_then_unicode -------------------------------------------------------------------------------- /testdata/ukkonooa: -------------------------------------------------------------------------------- 1 | ukko nooa, ukko nooa oli kunnon mies, kun han meni saunaan, pisti laukun naulaan, ukko nooa, ukko nooa oli kunnon mies. -------------------------------------------------------------------------------- /testdata/x: -------------------------------------------------------------------------------- 1 | X -------------------------------------------------------------------------------- /testdata/x.compressed.00: -------------------------------------------------------------------------------- 1 | X -------------------------------------------------------------------------------- /testdata/x.compressed.01: -------------------------------------------------------------------------------- 1 | ,XX -------------------------------------------------------------------------------- /testdata/x.compressed.02: -------------------------------------------------------------------------------- 1 | X -------------------------------------------------------------------------------- /testdata/x.compressed.03: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/rust-brotli/a017e2d0c02ac9539be8594aaa821f864bb69669/testdata/x.compressed.03 -------------------------------------------------------------------------------- /testdata/xyzzy: -------------------------------------------------------------------------------- 1 | Xyzzy --------------------------------------------------------------------------------