├── .chglog ├── CHANGELOG.tpl.md └── config.yml ├── .github ├── labeler.yml └── workflows │ ├── changelog.yml │ ├── erlang.yml │ └── git.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── c_src ├── Makefile ├── brotli_nif.c ├── common │ ├── constants.c │ ├── constants.h │ ├── context.c │ ├── context.h │ ├── dictionary.c │ ├── dictionary.h │ ├── platform.c │ ├── platform.h │ ├── transform.c │ ├── transform.h │ └── version.h ├── dec │ ├── bit_reader.c │ ├── bit_reader.h │ ├── decode.c │ ├── huffman.c │ ├── huffman.h │ ├── prefix.h │ ├── state.c │ └── state.h ├── enc │ ├── backward_references.c │ ├── backward_references.h │ ├── backward_references_hq.c │ ├── backward_references_hq.h │ ├── backward_references_inc.h │ ├── bit_cost.c │ ├── bit_cost.h │ ├── bit_cost_inc.h │ ├── block_encoder_inc.h │ ├── block_splitter.c │ ├── block_splitter.h │ ├── block_splitter_inc.h │ ├── brotli_bit_stream.c │ ├── brotli_bit_stream.h │ ├── cluster.c │ ├── cluster.h │ ├── cluster_inc.h │ ├── command.c │ ├── command.h │ ├── compress_fragment.c │ ├── compress_fragment.h │ ├── compress_fragment_two_pass.c │ ├── compress_fragment_two_pass.h │ ├── dictionary_hash.c │ ├── dictionary_hash.h │ ├── encode.c │ ├── encoder_dict.c │ ├── encoder_dict.h │ ├── entropy_encode.c │ ├── entropy_encode.h │ ├── entropy_encode_static.h │ ├── fast_log.c │ ├── fast_log.h │ ├── find_match_length.h │ ├── hash.h │ ├── hash_composite_inc.h │ ├── hash_forgetful_chain_inc.h │ ├── hash_longest_match64_inc.h │ ├── hash_longest_match_inc.h │ ├── hash_longest_match_quickly_inc.h │ ├── hash_rolling_inc.h │ ├── hash_to_binary_tree_inc.h │ ├── histogram.c │ ├── histogram.h │ ├── histogram_inc.h │ ├── literal_cost.c │ ├── literal_cost.h │ ├── memory.c │ ├── memory.h │ ├── metablock.c │ ├── metablock.h │ ├── metablock_inc.h │ ├── params.h │ ├── prefix.h │ ├── quality.h │ ├── ringbuffer.h │ ├── static_dict.c │ ├── static_dict.h │ ├── static_dict_lut.h │ ├── utf8_util.c │ ├── utf8_util.h │ └── write_bits.h ├── include │ └── brotli │ │ ├── decode.h │ │ ├── encode.h │ │ ├── port.h │ │ └── types.h └── tools │ ├── brotli.c │ └── brotli.md ├── priv └── .gitignore ├── rebar.config ├── rebar.lock ├── src ├── brotli.app.src ├── brotli.erl ├── brotli_decoder.erl ├── brotli_encoder.erl └── brotli_nif.erl └── test ├── brotli_decoder_test.erl ├── brotli_encoder_SUITE.erl ├── brotli_encoder_SUITE_data ├── huge ├── huge.br ├── in │ ├── random_1024 │ ├── random_16 │ ├── random_512 │ └── simple ├── large ├── large.br └── out │ ├── random_1024 │ ├── random_16 │ ├── random_512 │ └── simple ├── brotli_test.erl └── prop_brotli.erl /.chglog/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ if .Versions -}} 2 | 3 | ## [Unreleased] 4 | 5 | {{ if .Unreleased.CommitGroups -}} 6 | {{ range .Unreleased.CommitGroups -}} 7 | ### {{ .Title }} 8 | {{ range .Commits -}} 9 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 10 | {{ end }} 11 | {{ end -}} 12 | {{ end -}} 13 | {{ end -}} 14 | 15 | {{ range .Versions }} 16 | 17 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} 18 | {{ range .CommitGroups -}} 19 | ### {{ .Title }} 20 | {{ range .Commits -}} 21 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} ([`{{ .Hash.Short }}`]({{ $.Info.RepositoryURL }}/commit/{{ .Hash.Long }})) 22 | {{ end }} 23 | {{ end -}} 24 | 25 | {{- if .NoteGroups -}} 26 | {{ range .NoteGroups -}} 27 | ### {{ .Title }} 28 | {{ range .Notes }} 29 | {{ .Body }} 30 | {{ end }} 31 | {{ end -}} 32 | {{ end -}} 33 | {{ end -}} 34 | 35 | {{- if .Versions }} 36 | [Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD 37 | {{ range .Versions -}} 38 | {{ if .Tag.Previous -}} 39 | [{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} 40 | {{ end -}} 41 | {{ end -}} 42 | {{ end -}} 43 | -------------------------------------------------------------------------------- /.chglog/config.yml: -------------------------------------------------------------------------------- 1 | style: github 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: https://github.com/yjh0502/erl-brotli 6 | options: 7 | commits: 8 | filters: 9 | Type: 10 | - ft 11 | - fix 12 | - docs 13 | commit_groups: 14 | title_maps: 15 | ft: Features 16 | fix: Bug Fixes 17 | docs: Documentation 18 | header: 19 | pattern: "^(\\w*)\\:\\s(.*)$" 20 | pattern_maps: 21 | - Type 22 | - Subject 23 | notes: 24 | keywords: 25 | - BREAKING CHANGE 26 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | scope/nif: 2 | - any: ['c_src/brotli_nif.c', 'c_src/Makefile'] 3 | 4 | scope/ci: 5 | - any: ['.github/workflows/*'] 6 | 7 | scope/test: 8 | - any: ['test/*.erl'] 9 | -------------------------------------------------------------------------------- /.github/workflows/changelog.yml: -------------------------------------------------------------------------------- 1 | name: Changelog 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | jobs: 8 | update-changelog: 9 | name: Update changelog 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | ref: master 16 | - uses: Bpazy/setup-git-chglog@v1 17 | - name: git-chglog version 18 | run: git chglog --version 19 | - name: Generate changelog 20 | run: git chglog | tee CHANGELOG.md 21 | - name: Commit 22 | run: | 23 | git config user.email "" 24 | git config user.name "GitHub Action Bot" 25 | git diff -- CHANGELOG.md 26 | git commit -m "chore: update CHANGELOG [skip ci]" CHANGELOG.md && git push origin master || true 27 | -------------------------------------------------------------------------------- /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - 'master' 7 | push: 8 | branches: 9 | - 'master' 10 | 11 | jobs: 12 | build: 13 | name: Test on OTP ${{ matrix.otp_version }} and ${{ matrix.os }} 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | otp_version: ['25.1.1', '24.3.4.6', '23.3.4.2'] 18 | rebar3_version: ['3.16.1'] 19 | os: [ubuntu-18.04] 20 | include: 21 | - otp_version: '22.3.4.20' 22 | rebar3_version: '3.15.2' 23 | os: ubuntu-18.04 24 | - otp_version: '26.0.2' 25 | rebar3_version: '3.22.0' 26 | os: ubuntu-18.04 27 | env: 28 | OTP_VERSION: ${{ matrix.otp_version }} 29 | steps: 30 | - uses: actions/checkout@v2 31 | - uses: erlef/setup-beam@v1 32 | with: 33 | otp-version: ${{ matrix.otp_version }} 34 | rebar3-version: ${{ matrix.rebar3_version }} 35 | elixir-version: '1.13.4' 36 | - uses: actions/cache@v2 37 | name: Cache 38 | with: 39 | path: | 40 | _build 41 | key: ${{ runner.os }}-build-${{ matrix.otp_version }}-${{ hashFiles(format('rebar.lock')) }}-2 42 | restore-keys: | 43 | ${{ runner.os }}-build-${{ matrix.otp_version }}-2- 44 | - name: Compile 45 | run: rebar3 compile 46 | - name: EUnit tests 47 | run: rebar3 eunit --cover 48 | - name: Common Test tests 49 | run: rebar3 ct --cover 50 | - name: PropEr tests 51 | run: rebar3 proper --cover 52 | - name: XRef 53 | run: rebar3 xref 54 | - name: Covertool 55 | if: ${{ always() }} 56 | run: rebar3 covertool generate 57 | - uses: codecov/codecov-action@v1 58 | if: ${{ always() }} 59 | with: 60 | file: _build/test/covertool/brotli.covertool.xml 61 | env_vars: OTP_VERSION 62 | 63 | lint: 64 | name: Lint 65 | runs-on: ${{ matrix.os }} 66 | strategy: 67 | matrix: 68 | otp_version: ['24.3.4.6'] 69 | rebar3_version: ['3.16.1'] 70 | os: [ubuntu-18.04] 71 | env: 72 | OTP_VERSION: ${{ matrix.otp_version }} 73 | steps: 74 | - uses: actions/checkout@v2 75 | - uses: erlef/setup-beam@v1 76 | with: 77 | otp-version: ${{ matrix.otp_version }} 78 | rebar3-version: ${{ matrix.rebar3_version }} 79 | elixir-version: '1.13.4' 80 | - uses: actions/cache@v2 81 | name: Cache 82 | with: 83 | path: | 84 | _build 85 | key: ${{ runner.os }}-build-${{ matrix.otp_version }}-${{ hashFiles(format('rebar.lock')) }}-2 86 | restore-keys: | 87 | ${{ runner.os }}-build-${{ matrix.otp_version }}-2- 88 | - name: Erlfmt Check 89 | run: rebar3 fmt --check 90 | 91 | dialyzer: 92 | name: Dialyze on OTP ${{ matrix.otp_version }} and ${{ matrix.os }} 93 | runs-on: ${{ matrix.os }} 94 | strategy: 95 | matrix: 96 | otp_version: ['25.1.1', '24.3.4.6', '23.3.4.2'] 97 | rebar3_version: ['3.16.1'] 98 | os: [ubuntu-18.04] 99 | include: 100 | - otp_version: '22.3.4.20' 101 | rebar3_version: '3.15.2' 102 | os: ubuntu-18.04 103 | steps: 104 | - uses: actions/checkout@v2 105 | - uses: erlef/setup-beam@v1 106 | with: 107 | otp-version: ${{ matrix.otp_version }} 108 | rebar3-version: ${{ matrix.rebar3_version }} 109 | - uses: actions/cache@v2 110 | name: Cache 111 | with: 112 | path: | 113 | _build 114 | key: ${{ runner.os }}-build-${{ matrix.otp_version }}-${{ hashFiles('rebar.lock') }}-2 115 | restore-keys: | 116 | ${{ runner.os }}-dialyzer-${{ matrix.otp_version }}-2- 117 | - name: Compile 118 | run: rebar3 compile 119 | - name: Dialyzer 120 | run: rebar3 dialyzer 121 | -------------------------------------------------------------------------------- /.github/workflows/git.yml: -------------------------------------------------------------------------------- 1 | name: Git 2 | 3 | on: 4 | pull_request_target: 5 | branches: [ master ] 6 | 7 | jobs: 8 | commit-messages: 9 | name: Check commit messages 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | ref: ${{ github.event.pull_request.head.sha }} 16 | - name: Get PR URL 17 | uses: Dovyski/payload-info-action@master 18 | id: issue_url 19 | with: 20 | filter_pull_request: '.pull_request._links.issue.href' 21 | - name: Check 22 | env: 23 | TYPES: ft|fix|docs|chore|test 24 | run: | 25 | ! git log --oneline -E --invert-grep --grep="^($TYPES):" --pretty=format:"::error title=Invalid commit message::%h %s" origin/$GITHUB_BASE_REF... | grep "." 26 | - name: Unlabel correct PR 27 | if: ${{ success() }} 28 | run: | 29 | curl -X DELETE \ 30 | --header 'authorization: Bearer ${{ github.token }}' \ 31 | --header 'content-type: application/json' \ 32 | ${{ steps.issue_url.outputs.value }}/labels/invalid/commit-messages 33 | - name: Label PR with invalid commit messages 34 | if: ${{ failure() }} 35 | run: | 36 | curl -X POST \ 37 | --header 'authorization: Bearer ${{ github.token }}' \ 38 | --header 'content-type: application/json' \ 39 | --data '["invalid/commit-messages"]' \ 40 | ${{ steps.issue_url.outputs.value }}/labels 41 | 42 | labeler: 43 | name: Label PRs depending on changes 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v2 47 | with: 48 | fetch-depth: 0 49 | ref: ${{ github.event.pull_request.head.sha }} 50 | - uses: actions/labeler@v3 51 | with: 52 | repo-token: "${{ github.token }}" 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .rebar3 2 | _* 3 | .eunit 4 | *.o 5 | *.a 6 | *.beam 7 | *.plt 8 | *.swp 9 | *.swo 10 | .erlang.cookie 11 | ebin 12 | log 13 | erl_crash.dump 14 | .rebar 15 | _rel 16 | _deps 17 | _plugins 18 | _tdeps 19 | logs 20 | _build 21 | rebar3.crashdump 22 | /doc 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | ## [Unreleased] 3 | 4 | 5 | 6 | ## [v0.3.2] - 2023-07-25 7 | ### Bug Fixes 8 | - build on otp 26 ([`909192f`](https://github.com/yjh0502/erl-brotli/commit/909192fc18c102ff7323b9daa7b73d856c0b8796)) 9 | 10 | 11 | 12 | ## [v0.3.1] - 2022-01-06 13 | ### Bug Fixes 14 | - chunk input for brotli:encode/{1,2} ([`eca29c4`](https://github.com/yjh0502/erl-brotli/commit/eca29c44e2d6298c0dc86a0c8e9145fd49fb2449)) 15 | 16 | 17 | 18 | ## v0.3.0 - 2021-10-04 19 | ### Bug Fixes 20 | - remove maps:foreach/2 and use maps:fold/3 instead ([`129dd55`](https://github.com/yjh0502/erl-brotli/commit/129dd557b87360af447adabae079fc58811ce488)) 21 | - change the HEAD branch name ([`47957c8`](https://github.com/yjh0502/erl-brotli/commit/47957c81058e2d4ceacd51bb2594ff36908c7fe1)) 22 | 23 | ### Features 24 | - add decoder API ([`401ff99`](https://github.com/yjh0502/erl-brotli/commit/401ff99e8bbc6ec3b4258d568835d1f673493644)) 25 | - update library to use stream API ([`bc09c59`](https://github.com/yjh0502/erl-brotli/commit/bc09c5969ae4c5b45c07abdd3eb55cc8652fa0ec)) 26 | 27 | 28 | [Unreleased]: https://github.com/yjh0502/erl-brotli/compare/v0.3.2...HEAD 29 | [v0.3.2]: https://github.com/yjh0502/erl-brotli/compare/v0.3.1...v0.3.2 30 | [v0.3.1]: https://github.com/yjh0502/erl-brotli/compare/v0.3.0...v0.3.1 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Jihyun Yu . 2 | Copyright (c) 2021, Łukasz Niemier . 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are 7 | met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | * The names of its contributors may not be used to endorse or promote 17 | products derived from this software without specific prior written 18 | permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | brotli 2 | ===== 3 | 4 | A library providing Erlang API for Brotli compression library. 5 | 6 | Build 7 | ----- 8 | 9 | $ rebar3 compile 10 | 11 | Example Usage (from Elixir) 12 | ----- 13 | Start `iex` and then: 14 | 15 | ``` 16 | iex(1)> c ("src/brotli.erl") 17 | [:brotli] 18 | 19 | iex(2)> c ("src/brotli_nif.erl") 20 | [:brotli_nif] 21 | 22 | iex(3)> c ("src/brotli_encoder.erl") 23 | [:brotli_encoder] 24 | 25 | iex(4)> data = File.read!("README.md") 26 | "brotli\n=====\n\nAn OTP library\n\nBuild\n-----\n\n $ rebar3 compile\n..." 27 | 28 | iex(5)> :brotli.encode(data) 29 | {:ok, 30 | <<27, 26, 3, 0, 140, 146, 28, 142, 124, 217, 200, 164, 20, 156, 211, 199, 168, 31 | 156, 219, 19, 176, 219, 248, 140, 58, 157, 210, 144, 133, 150, 2, 76, 94, 32 | 201, 231, 55, 179, 243, 125, 215, 180, 141, 235, 59, 213, 185, 57, 61, ...>>} 33 | ``` 34 | 35 | ### License 36 | 37 | Library and most tests are licensed on [BSD-3-Clause](LICENSE) License. 38 | Some tests, in files matching glob `prop_*.erl` are GPL-3.0 licensed. 39 | -------------------------------------------------------------------------------- /c_src/Makefile: -------------------------------------------------------------------------------- 1 | # Based on c_src.mk from erlang.mk by Loic Hoguin 2 | 3 | CURDIR := $(shell pwd) 4 | BASEDIR := $(abspath $(CURDIR)/..) 5 | 6 | PROJECT = brotli 7 | 8 | ERTS_INCLUDE_DIR ?= $(shell erl -noshell -eval "io:format(\"~s/erts-~s/include/\", [code:root_dir(), erlang:system_info(version)])." -s init stop) 9 | 10 | ifndef REBAR_BARE_COMPILER_OUTPUT_DIR 11 | OUT ?= $(BASEDIR)/priv 12 | else 13 | OUT ?= $(REBAR_BARE_COMPILER_OUTPUT_DIR)/priv 14 | endif 15 | 16 | # System type and C compiler/flags. 17 | # 18 | 19 | UNAME_SYS := $(shell uname -s) 20 | ifeq ($(UNAME_SYS), Darwin) 21 | CC ?= cc 22 | CFLAGS += -O3 -std=c99 -finline-functions -Wall 23 | LDFLAGS ?= -flat_namespace -undefined suppress 24 | else ifeq ($(UNAME_SYS), FreeBSD) 25 | CC = /usr/bin/clang 26 | CFLAGS += -O3 -std=c99 -finline-functions -Wall 27 | else ifeq ($(UNAME_SYS), Linux) 28 | CC ?= gcc 29 | CFLAGS += -O3 -std=c99 -finline-functions -Wall 30 | endif 31 | 32 | CFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I ./include/ 33 | 34 | LDFLAGS += -shared -rdynamic 35 | 36 | BROTLI_SRCS = $(wildcard common/*.c) $(wildcard dec/*.c) $(wildcard enc/*.c) 37 | BROTLI_OBJS = $(patsubst %.c,%.o,$(BROTLI_SRCS)) 38 | 39 | # Verbosity. 40 | 41 | c_verbose_0 = @echo " C " $(?F); 42 | c_verbose = $(c_verbose_$(V)) 43 | 44 | cpp_verbose_0 = @echo " CPP " $(?F); 45 | cpp_verbose = $(cpp_verbose_$(V)) 46 | 47 | link_verbose_0 = @echo " LD " $(@F); 48 | link_verbose = $(link_verbose_$(V)) 49 | 50 | lib: $(OUT)/$(PROJECT).so 51 | 52 | $(OUT)/$(PROJECT).so: brotli_nif.o $(BROTLI_OBJS) 53 | @mkdir -p $(@D) 54 | $(link_verbose) $(CC) $^ $(LDFLAGS) $(LDLIBS) -o $@ 55 | 56 | tool: $(OUT)/brotli 57 | 58 | $(OUT)/brotli: $(BROTLI_OBJS) tools/brotli.o 59 | $(link_verbose) $(CC) $^ -lm $(LDLIBS) -o $@ 60 | 61 | tools/brotli.o: tools/brotli.c 62 | $(c_verbose) $(CC) -O2 -I ./include/ $(CPPFLAGS) -c $< -o $@ 63 | 64 | %.o: %.c 65 | $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ 66 | 67 | clean: 68 | @rm -f $(OUT)/$(PROJECT).so \ 69 | $(OUT)/brotli $(BROTLI_OBJS) \ 70 | tools/brotli.o brotli_nif.o 71 | 72 | .PHONY: lib tool 73 | -------------------------------------------------------------------------------- /c_src/common/constants.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./constants.h" 8 | 9 | const BrotliPrefixCodeRange 10 | _kBrotliPrefixCodeRanges[BROTLI_NUM_BLOCK_LEN_SYMBOLS] = { 11 | {1, 2}, {5, 2}, {9, 2}, {13, 2}, {17, 3}, {25, 3}, 12 | {33, 3}, {41, 3}, {49, 4}, {65, 4}, {81, 4}, {97, 4}, 13 | {113, 5}, {145, 5}, {177, 5}, {209, 5}, {241, 6}, {305, 6}, 14 | {369, 7}, {497, 8}, {753, 9}, {1265, 10}, {2289, 11}, {4337, 12}, 15 | {8433, 13}, {16625, 24}}; 16 | -------------------------------------------------------------------------------- /c_src/common/context.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Lookup table to map the previous two bytes to a context id. 8 | 9 | There are four different context modeling modes defined here: 10 | CONTEXT_LSB6: context id is the least significant 6 bits of the last byte, 11 | CONTEXT_MSB6: context id is the most significant 6 bits of the last byte, 12 | CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text, 13 | CONTEXT_SIGNED: second-order context model tuned for signed integers. 14 | 15 | If |p1| and |p2| are the previous two bytes, and |mode| is current context 16 | mode, we calculate the context as: 17 | 18 | context = ContextLut(mode)[p1] | ContextLut(mode)[p2 + 256]. 19 | 20 | For CONTEXT_UTF8 mode, if the previous two bytes are ASCII characters 21 | (i.e. < 128), this will be equivalent to 22 | 23 | context = 4 * context1(p1) + context2(p2), 24 | 25 | where context1 is based on the previous byte in the following way: 26 | 27 | 0 : non-ASCII control 28 | 1 : \t, \n, \r 29 | 2 : space 30 | 3 : other punctuation 31 | 4 : " ' 32 | 5 : % 33 | 6 : ( < [ { 34 | 7 : ) > ] } 35 | 8 : , ; : 36 | 9 : . 37 | 10 : = 38 | 11 : number 39 | 12 : upper-case vowel 40 | 13 : upper-case consonant 41 | 14 : lower-case vowel 42 | 15 : lower-case consonant 43 | 44 | and context2 is based on the second last byte: 45 | 46 | 0 : control, space 47 | 1 : punctuation 48 | 2 : upper-case letter, number 49 | 3 : lower-case letter 50 | 51 | If the last byte is ASCII, and the second last byte is not (in a valid UTF8 52 | stream it will be a continuation byte, value between 128 and 191), the 53 | context is the same as if the second last byte was an ASCII control or space. 54 | 55 | If the last byte is a UTF8 lead byte (value >= 192), then the next byte will 56 | be a continuation byte and the context id is 2 or 3 depending on the LSB of 57 | the last byte and to a lesser extent on the second last byte if it is ASCII. 58 | 59 | If the last byte is a UTF8 continuation byte, the second last byte can be: 60 | - continuation byte: the next byte is probably ASCII or lead byte (assuming 61 | 4-byte UTF8 characters are rare) and the context id is 0 or 1. 62 | - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1 63 | - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3 64 | 65 | The possible value combinations of the previous two bytes, the range of 66 | context ids and the type of the next byte is summarized in the table below: 67 | 68 | |--------\-----------------------------------------------------------------| 69 | | \ Last byte | 70 | | Second \---------------------------------------------------------------| 71 | | last byte \ ASCII | cont. byte | lead byte | 72 | | \ (0-127) | (128-191) | (192-) | 73 | |=============|===================|=====================|==================| 74 | | ASCII | next: ASCII/lead | not valid | next: cont. | 75 | | (0-127) | context: 4 - 63 | | context: 2 - 3 | 76 | |-------------|-------------------|---------------------|------------------| 77 | | cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. | 78 | | (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 | 79 | |-------------|-------------------|---------------------|------------------| 80 | | lead byte | not valid | next: ASCII/lead | not valid | 81 | | (192-207) | | context: 0 - 1 | | 82 | |-------------|-------------------|---------------------|------------------| 83 | | lead byte | not valid | next: cont. | not valid | 84 | | (208-) | | context: 2 - 3 | | 85 | |-------------|-------------------|---------------------|------------------| 86 | */ 87 | 88 | #ifndef BROTLI_COMMON_CONTEXT_H_ 89 | #define BROTLI_COMMON_CONTEXT_H_ 90 | 91 | #include 92 | #include 93 | 94 | typedef enum ContextType { 95 | CONTEXT_LSB6 = 0, 96 | CONTEXT_MSB6 = 1, 97 | CONTEXT_UTF8 = 2, 98 | CONTEXT_SIGNED = 3 99 | } ContextType; 100 | 101 | /* "Soft-private", it is exported, but not "advertised" as API. */ 102 | /* Common context lookup table for all context modes. */ 103 | BROTLI_COMMON_API extern const uint8_t _kBrotliContextLookupTable[2048]; 104 | 105 | typedef const uint8_t* ContextLut; 106 | 107 | /* typeof(MODE) == ContextType; returns ContextLut */ 108 | #define BROTLI_CONTEXT_LUT(MODE) (&_kBrotliContextLookupTable[(MODE) << 9]) 109 | 110 | /* typeof(LUT) == ContextLut */ 111 | #define BROTLI_CONTEXT(P1, P2, LUT) ((LUT)[P1] | ((LUT) + 256)[P2]) 112 | 113 | #endif /* BROTLI_COMMON_CONTEXT_H_ */ 114 | -------------------------------------------------------------------------------- /c_src/common/dictionary.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Collection of static dictionary words. */ 8 | 9 | #ifndef BROTLI_COMMON_DICTIONARY_H_ 10 | #define BROTLI_COMMON_DICTIONARY_H_ 11 | 12 | #include 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | typedef struct BrotliDictionary { 20 | /** 21 | * Number of bits to encode index of dictionary word in a bucket. 22 | * 23 | * Specification: Appendix A. Static Dictionary Data 24 | * 25 | * Words in a dictionary are bucketed by length. 26 | * @c 0 means that there are no words of a given length. 27 | * Dictionary consists of words with length of [4..24] bytes. 28 | * Values at [0..3] and [25..31] indices should not be addressed. 29 | */ 30 | uint8_t size_bits_by_length[32]; 31 | 32 | /* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */ 33 | uint32_t offsets_by_length[32]; 34 | 35 | /* assert(data_size == offsets_by_length[31]) */ 36 | size_t data_size; 37 | 38 | /* Data array is not bound, and should obey to size_bits_by_length values. 39 | Specified size matches default (RFC 7932) dictionary. Its size is 40 | defined by data_size */ 41 | const uint8_t* data; 42 | } BrotliDictionary; 43 | 44 | BROTLI_COMMON_API const BrotliDictionary* BrotliGetDictionary(void); 45 | 46 | /** 47 | * Sets dictionary data. 48 | * 49 | * When dictionary data is already set / present, this method is no-op. 50 | * 51 | * Dictionary data MUST be provided before BrotliGetDictionary is invoked. 52 | * This method is used ONLY in multi-client environment (e.g. C + Java), 53 | * to reduce storage by sharing single dictionary between implementations. 54 | */ 55 | BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data); 56 | 57 | #define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4 58 | #define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24 59 | 60 | #if defined(__cplusplus) || defined(c_plusplus) 61 | } /* extern "C" */ 62 | #endif 63 | 64 | #endif /* BROTLI_COMMON_DICTIONARY_H_ */ 65 | -------------------------------------------------------------------------------- /c_src/common/platform.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include 8 | 9 | #include "./platform.h" 10 | #include 11 | 12 | /* Default brotli_alloc_func */ 13 | void* BrotliDefaultAllocFunc(void* opaque, size_t size) { 14 | BROTLI_UNUSED(opaque); 15 | return malloc(size); 16 | } 17 | 18 | /* Default brotli_free_func */ 19 | void BrotliDefaultFreeFunc(void* opaque, void* address) { 20 | BROTLI_UNUSED(opaque); 21 | free(address); 22 | } 23 | -------------------------------------------------------------------------------- /c_src/common/transform.h: -------------------------------------------------------------------------------- 1 | /* transforms is a part of ABI, but not API. 2 | 3 | It means that there are some functions that are supposed to be in "common" 4 | library, but header itself is not placed into include/brotli. This way, 5 | aforementioned functions will be available only to brotli internals. 6 | */ 7 | 8 | #ifndef BROTLI_COMMON_TRANSFORM_H_ 9 | #define BROTLI_COMMON_TRANSFORM_H_ 10 | 11 | #include 12 | #include 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | enum BrotliWordTransformType { 19 | BROTLI_TRANSFORM_IDENTITY = 0, 20 | BROTLI_TRANSFORM_OMIT_LAST_1 = 1, 21 | BROTLI_TRANSFORM_OMIT_LAST_2 = 2, 22 | BROTLI_TRANSFORM_OMIT_LAST_3 = 3, 23 | BROTLI_TRANSFORM_OMIT_LAST_4 = 4, 24 | BROTLI_TRANSFORM_OMIT_LAST_5 = 5, 25 | BROTLI_TRANSFORM_OMIT_LAST_6 = 6, 26 | BROTLI_TRANSFORM_OMIT_LAST_7 = 7, 27 | BROTLI_TRANSFORM_OMIT_LAST_8 = 8, 28 | BROTLI_TRANSFORM_OMIT_LAST_9 = 9, 29 | BROTLI_TRANSFORM_UPPERCASE_FIRST = 10, 30 | BROTLI_TRANSFORM_UPPERCASE_ALL = 11, 31 | BROTLI_TRANSFORM_OMIT_FIRST_1 = 12, 32 | BROTLI_TRANSFORM_OMIT_FIRST_2 = 13, 33 | BROTLI_TRANSFORM_OMIT_FIRST_3 = 14, 34 | BROTLI_TRANSFORM_OMIT_FIRST_4 = 15, 35 | BROTLI_TRANSFORM_OMIT_FIRST_5 = 16, 36 | BROTLI_TRANSFORM_OMIT_FIRST_6 = 17, 37 | BROTLI_TRANSFORM_OMIT_FIRST_7 = 18, 38 | BROTLI_TRANSFORM_OMIT_FIRST_8 = 19, 39 | BROTLI_TRANSFORM_OMIT_FIRST_9 = 20, 40 | BROTLI_TRANSFORM_SHIFT_FIRST = 21, 41 | BROTLI_TRANSFORM_SHIFT_ALL = 22, 42 | BROTLI_NUM_TRANSFORM_TYPES /* Counts transforms, not a transform itself. */ 43 | }; 44 | 45 | #define BROTLI_TRANSFORMS_MAX_CUT_OFF BROTLI_TRANSFORM_OMIT_LAST_9 46 | 47 | typedef struct BrotliTransforms { 48 | uint16_t prefix_suffix_size; 49 | /* Last character must be null, so prefix_suffix_size must be at least 1. */ 50 | const uint8_t* prefix_suffix; 51 | const uint16_t* prefix_suffix_map; 52 | uint32_t num_transforms; 53 | /* Each entry is a [prefix_id, transform, suffix_id] triplet. */ 54 | const uint8_t* transforms; 55 | /* Shift for BROTLI_TRANSFORM_SHIFT_FIRST and BROTLI_TRANSFORM_SHIFT_ALL, 56 | must be NULL if and only if no such transforms are present. */ 57 | const uint8_t* params; 58 | /* Indices of transforms like ["", BROTLI_TRANSFORM_OMIT_LAST_#, ""]. 59 | 0-th element corresponds to ["", BROTLI_TRANSFORM_IDENTITY, ""]. 60 | -1, if cut-off transform does not exist. */ 61 | int16_t cutOffTransforms[BROTLI_TRANSFORMS_MAX_CUT_OFF + 1]; 62 | } BrotliTransforms; 63 | 64 | /* T is BrotliTransforms*; result is uint8_t. */ 65 | #define BROTLI_TRANSFORM_PREFIX_ID(T, I) ((T)->transforms[((I) * 3) + 0]) 66 | #define BROTLI_TRANSFORM_TYPE(T, I) ((T)->transforms[((I) * 3) + 1]) 67 | #define BROTLI_TRANSFORM_SUFFIX_ID(T, I) ((T)->transforms[((I) * 3) + 2]) 68 | 69 | /* T is BrotliTransforms*; result is const uint8_t*. */ 70 | #define BROTLI_TRANSFORM_PREFIX(T, I) (&(T)->prefix_suffix[ \ 71 | (T)->prefix_suffix_map[BROTLI_TRANSFORM_PREFIX_ID(T, I)]]) 72 | #define BROTLI_TRANSFORM_SUFFIX(T, I) (&(T)->prefix_suffix[ \ 73 | (T)->prefix_suffix_map[BROTLI_TRANSFORM_SUFFIX_ID(T, I)]]) 74 | 75 | BROTLI_COMMON_API const BrotliTransforms* BrotliGetTransforms(void); 76 | 77 | BROTLI_COMMON_API int BrotliTransformDictionaryWord( 78 | uint8_t* dst, const uint8_t* word, int len, 79 | const BrotliTransforms* transforms, int transform_idx); 80 | 81 | #if defined(__cplusplus) || defined(c_plusplus) 82 | } /* extern "C" */ 83 | #endif 84 | 85 | #endif /* BROTLI_COMMON_TRANSFORM_H_ */ 86 | -------------------------------------------------------------------------------- /c_src/common/version.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Version definition. */ 8 | 9 | #ifndef BROTLI_COMMON_VERSION_H_ 10 | #define BROTLI_COMMON_VERSION_H_ 11 | 12 | /* This macro should only be used when library is compiled together with client. 13 | If library is dynamically linked, use BrotliDecoderVersion and 14 | BrotliEncoderVersion methods. */ 15 | 16 | /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */ 17 | #define BROTLI_VERSION 0x1000009 18 | 19 | /* This macro is used by build system to produce Libtool-friendly soname. See 20 | https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html 21 | */ 22 | 23 | /* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */ 24 | #define BROTLI_ABI_VERSION 0x1009000 25 | 26 | #endif /* BROTLI_COMMON_VERSION_H_ */ 27 | -------------------------------------------------------------------------------- /c_src/dec/bit_reader.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Bit reading helpers */ 8 | 9 | #include "./bit_reader.h" 10 | 11 | #include "../common/platform.h" 12 | #include 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | const uint32_t kBrotliBitMask[33] = { 0x00000000, 19 | 0x00000001, 0x00000003, 0x00000007, 0x0000000F, 20 | 0x0000001F, 0x0000003F, 0x0000007F, 0x000000FF, 21 | 0x000001FF, 0x000003FF, 0x000007FF, 0x00000FFF, 22 | 0x00001FFF, 0x00003FFF, 0x00007FFF, 0x0000FFFF, 23 | 0x0001FFFF, 0x0003FFFF, 0x0007FFFF, 0x000FFFFF, 24 | 0x001FFFFF, 0x003FFFFF, 0x007FFFFF, 0x00FFFFFF, 25 | 0x01FFFFFF, 0x03FFFFFF, 0x07FFFFFF, 0x0FFFFFFF, 26 | 0x1FFFFFFF, 0x3FFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF 27 | }; 28 | 29 | void BrotliInitBitReader(BrotliBitReader* const br) { 30 | br->val_ = 0; 31 | br->bit_pos_ = sizeof(br->val_) << 3; 32 | } 33 | 34 | BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) { 35 | size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1; 36 | /* Fixing alignment after unaligned BrotliFillWindow would result accumulator 37 | overflow. If unalignment is caused by BrotliSafeReadBits, then there is 38 | enough space in accumulator to fix alignment. */ 39 | if (!BROTLI_ALIGNED_READ) { 40 | aligned_read_mask = 0; 41 | } 42 | if (BrotliGetAvailableBits(br) == 0) { 43 | if (!BrotliPullByte(br)) { 44 | return BROTLI_FALSE; 45 | } 46 | } 47 | 48 | while ((((size_t)br->next_in) & aligned_read_mask) != 0) { 49 | if (!BrotliPullByte(br)) { 50 | /* If we consumed all the input, we don't care about the alignment. */ 51 | return BROTLI_TRUE; 52 | } 53 | } 54 | return BROTLI_TRUE; 55 | } 56 | 57 | BROTLI_BOOL BrotliSafeReadBits32Slow(BrotliBitReader* const br, 58 | uint32_t n_bits, uint32_t* val) { 59 | uint32_t low_val; 60 | uint32_t high_val; 61 | BrotliBitReaderState memento; 62 | BROTLI_DCHECK(n_bits <= 32); 63 | BROTLI_DCHECK(n_bits > 24); 64 | BrotliBitReaderSaveState(br, &memento); 65 | if (!BrotliSafeReadBits(br, 16, &low_val) || 66 | !BrotliSafeReadBits(br, n_bits - 16, &high_val)) { 67 | BrotliBitReaderRestoreState(br, &memento); 68 | return BROTLI_FALSE; 69 | } 70 | *val = low_val | (high_val << 16); 71 | return BROTLI_TRUE; 72 | } 73 | 74 | #if defined(__cplusplus) || defined(c_plusplus) 75 | } /* extern "C" */ 76 | #endif 77 | -------------------------------------------------------------------------------- /c_src/dec/huffman.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Utilities for building Huffman decoding tables. */ 8 | 9 | #ifndef BROTLI_DEC_HUFFMAN_H_ 10 | #define BROTLI_DEC_HUFFMAN_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | #define BROTLI_HUFFMAN_MAX_CODE_LENGTH 15 20 | 21 | /* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */ 22 | #define BROTLI_HUFFMAN_MAX_SIZE_26 396 23 | /* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */ 24 | #define BROTLI_HUFFMAN_MAX_SIZE_258 632 25 | /* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */ 26 | #define BROTLI_HUFFMAN_MAX_SIZE_272 646 27 | 28 | #define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5 29 | 30 | #if ((defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_32)) && \ 31 | BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)) 32 | #define BROTLI_HUFFMAN_CODE_FAST_LOAD 33 | #endif 34 | 35 | #if !defined(BROTLI_HUFFMAN_CODE_FAST_LOAD) 36 | /* Do not create this struct directly - use the ConstructHuffmanCode 37 | * constructor below! */ 38 | typedef struct { 39 | uint8_t bits; /* number of bits used for this symbol */ 40 | uint16_t value; /* symbol value or table offset */ 41 | } HuffmanCode; 42 | 43 | static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits, 44 | const uint16_t value) { 45 | HuffmanCode h; 46 | h.bits = bits; 47 | h.value = value; 48 | return h; 49 | } 50 | 51 | /* Please use the following macros to optimize HuffmanCode accesses in hot 52 | * paths. 53 | * 54 | * For example, assuming |table| contains a HuffmanCode pointer: 55 | * 56 | * BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table); 57 | * BROTLI_HC_ADJUST_TABLE_INDEX(table, index_into_table); 58 | * *bits = BROTLI_HC_GET_BITS(table); 59 | * *value = BROTLI_HC_GET_VALUE(table); 60 | * BROTLI_HC_ADJUST_TABLE_INDEX(table, offset); 61 | * *bits2 = BROTLI_HC_GET_BITS(table); 62 | * *value2 = BROTLI_HC_GET_VALUE(table); 63 | * 64 | */ 65 | 66 | #define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) 67 | #define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V) 68 | 69 | /* These must be given a HuffmanCode pointer! */ 70 | #define BROTLI_HC_FAST_LOAD_BITS(H) (H->bits) 71 | #define BROTLI_HC_FAST_LOAD_VALUE(H) (H->value) 72 | 73 | #else /* BROTLI_HUFFMAN_CODE_FAST_LOAD */ 74 | 75 | typedef BROTLI_ALIGNED(4) uint32_t HuffmanCode; 76 | 77 | static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits, 78 | const uint16_t value) { 79 | return (HuffmanCode) ((value & 0xFFFF) << 16) | (bits & 0xFF); 80 | } 81 | 82 | #define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) uint32_t __fastload_##H = (*H) 83 | #define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V); __fastload_##H = (*H) 84 | 85 | /* These must be given a HuffmanCode pointer! */ 86 | #define BROTLI_HC_FAST_LOAD_BITS(H) ((__fastload_##H) & 0xFF) 87 | #define BROTLI_HC_FAST_LOAD_VALUE(H) ((__fastload_##H) >> 16) 88 | #endif /* BROTLI_HUFFMAN_CODE_FAST_LOAD */ 89 | 90 | /* Builds Huffman lookup table assuming code lengths are in symbol order. */ 91 | BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table, 92 | const uint8_t* const code_lengths, uint16_t* count); 93 | 94 | /* Builds Huffman lookup table assuming code lengths are in symbol order. 95 | Returns size of resulting table. */ 96 | BROTLI_INTERNAL uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table, 97 | int root_bits, const uint16_t* const symbol_lists, uint16_t* count); 98 | 99 | /* Builds a simple Huffman table. The |num_symbols| parameter is to be 100 | interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, 101 | 2 means 3 symbols, 3 means 4 symbols with lengths [2, 2, 2, 2], 102 | 4 means 4 symbols with lengths [1, 2, 3, 3]. */ 103 | BROTLI_INTERNAL uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table, 104 | int root_bits, uint16_t* symbols, uint32_t num_symbols); 105 | 106 | /* Contains a collection of Huffman trees with the same alphabet size. */ 107 | /* alphabet_size_limit is needed due to simple codes, since 108 | log2(alphabet_size_max) could be greater than log2(alphabet_size_limit). */ 109 | typedef struct { 110 | HuffmanCode** htrees; 111 | HuffmanCode* codes; 112 | uint16_t alphabet_size_max; 113 | uint16_t alphabet_size_limit; 114 | uint16_t num_htrees; 115 | } HuffmanTreeGroup; 116 | 117 | #if defined(__cplusplus) || defined(c_plusplus) 118 | } /* extern "C" */ 119 | #endif 120 | 121 | #endif /* BROTLI_DEC_HUFFMAN_H_ */ 122 | -------------------------------------------------------------------------------- /c_src/dec/state.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./state.h" 8 | 9 | #include /* free, malloc */ 10 | 11 | #include 12 | #include "./huffman.h" 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | BROTLI_BOOL BrotliDecoderStateInit(BrotliDecoderState* s, 19 | brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) { 20 | if (!alloc_func) { 21 | s->alloc_func = BrotliDefaultAllocFunc; 22 | s->free_func = BrotliDefaultFreeFunc; 23 | s->memory_manager_opaque = 0; 24 | } else { 25 | s->alloc_func = alloc_func; 26 | s->free_func = free_func; 27 | s->memory_manager_opaque = opaque; 28 | } 29 | 30 | s->error_code = 0; /* BROTLI_DECODER_NO_ERROR */ 31 | 32 | BrotliInitBitReader(&s->br); 33 | s->state = BROTLI_STATE_UNINITED; 34 | s->large_window = 0; 35 | s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE; 36 | s->substate_uncompressed = BROTLI_STATE_UNCOMPRESSED_NONE; 37 | s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE; 38 | s->substate_read_block_length = BROTLI_STATE_READ_BLOCK_LENGTH_NONE; 39 | 40 | s->buffer_length = 0; 41 | s->loop_counter = 0; 42 | s->pos = 0; 43 | s->rb_roundtrips = 0; 44 | s->partial_pos_out = 0; 45 | 46 | s->block_type_trees = NULL; 47 | s->block_len_trees = NULL; 48 | s->ringbuffer = NULL; 49 | s->ringbuffer_size = 0; 50 | s->new_ringbuffer_size = 0; 51 | s->ringbuffer_mask = 0; 52 | 53 | s->context_map = NULL; 54 | s->context_modes = NULL; 55 | s->dist_context_map = NULL; 56 | s->context_map_slice = NULL; 57 | s->dist_context_map_slice = NULL; 58 | 59 | s->literal_hgroup.codes = NULL; 60 | s->literal_hgroup.htrees = NULL; 61 | s->insert_copy_hgroup.codes = NULL; 62 | s->insert_copy_hgroup.htrees = NULL; 63 | s->distance_hgroup.codes = NULL; 64 | s->distance_hgroup.htrees = NULL; 65 | 66 | s->is_last_metablock = 0; 67 | s->is_uncompressed = 0; 68 | s->is_metadata = 0; 69 | s->should_wrap_ringbuffer = 0; 70 | s->canny_ringbuffer_allocation = 1; 71 | 72 | s->window_bits = 0; 73 | s->max_distance = 0; 74 | s->dist_rb[0] = 16; 75 | s->dist_rb[1] = 15; 76 | s->dist_rb[2] = 11; 77 | s->dist_rb[3] = 4; 78 | s->dist_rb_idx = 0; 79 | s->block_type_trees = NULL; 80 | s->block_len_trees = NULL; 81 | 82 | s->mtf_upper_bound = 63; 83 | 84 | s->dictionary = BrotliGetDictionary(); 85 | s->transforms = BrotliGetTransforms(); 86 | 87 | return BROTLI_TRUE; 88 | } 89 | 90 | void BrotliDecoderStateMetablockBegin(BrotliDecoderState* s) { 91 | s->meta_block_remaining_len = 0; 92 | s->block_length[0] = 1U << 24; 93 | s->block_length[1] = 1U << 24; 94 | s->block_length[2] = 1U << 24; 95 | s->num_block_types[0] = 1; 96 | s->num_block_types[1] = 1; 97 | s->num_block_types[2] = 1; 98 | s->block_type_rb[0] = 1; 99 | s->block_type_rb[1] = 0; 100 | s->block_type_rb[2] = 1; 101 | s->block_type_rb[3] = 0; 102 | s->block_type_rb[4] = 1; 103 | s->block_type_rb[5] = 0; 104 | s->context_map = NULL; 105 | s->context_modes = NULL; 106 | s->dist_context_map = NULL; 107 | s->context_map_slice = NULL; 108 | s->literal_htree = NULL; 109 | s->dist_context_map_slice = NULL; 110 | s->dist_htree_index = 0; 111 | s->context_lookup = NULL; 112 | s->literal_hgroup.codes = NULL; 113 | s->literal_hgroup.htrees = NULL; 114 | s->insert_copy_hgroup.codes = NULL; 115 | s->insert_copy_hgroup.htrees = NULL; 116 | s->distance_hgroup.codes = NULL; 117 | s->distance_hgroup.htrees = NULL; 118 | } 119 | 120 | void BrotliDecoderStateCleanupAfterMetablock(BrotliDecoderState* s) { 121 | BROTLI_DECODER_FREE(s, s->context_modes); 122 | BROTLI_DECODER_FREE(s, s->context_map); 123 | BROTLI_DECODER_FREE(s, s->dist_context_map); 124 | BROTLI_DECODER_FREE(s, s->literal_hgroup.htrees); 125 | BROTLI_DECODER_FREE(s, s->insert_copy_hgroup.htrees); 126 | BROTLI_DECODER_FREE(s, s->distance_hgroup.htrees); 127 | } 128 | 129 | void BrotliDecoderStateCleanup(BrotliDecoderState* s) { 130 | BrotliDecoderStateCleanupAfterMetablock(s); 131 | 132 | BROTLI_DECODER_FREE(s, s->ringbuffer); 133 | BROTLI_DECODER_FREE(s, s->block_type_trees); 134 | } 135 | 136 | BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s, 137 | HuffmanTreeGroup* group, uint32_t alphabet_size_max, 138 | uint32_t alphabet_size_limit, uint32_t ntrees) { 139 | /* 376 = 256 (1-st level table) + 4 + 7 + 15 + 31 + 63 (2-nd level mix-tables) 140 | This number is discovered "unlimited" "enough" calculator; it is actually 141 | a wee bigger than required in several cases (especially for alphabets with 142 | less than 16 symbols). */ 143 | const size_t max_table_size = alphabet_size_limit + 376; 144 | const size_t code_size = sizeof(HuffmanCode) * ntrees * max_table_size; 145 | const size_t htree_size = sizeof(HuffmanCode*) * ntrees; 146 | /* Pointer alignment is, hopefully, wider than sizeof(HuffmanCode). */ 147 | HuffmanCode** p = (HuffmanCode**)BROTLI_DECODER_ALLOC(s, 148 | code_size + htree_size); 149 | group->alphabet_size_max = (uint16_t)alphabet_size_max; 150 | group->alphabet_size_limit = (uint16_t)alphabet_size_limit; 151 | group->num_htrees = (uint16_t)ntrees; 152 | group->htrees = p; 153 | group->codes = (HuffmanCode*)(&p[ntrees]); 154 | return !!p; 155 | } 156 | 157 | #if defined(__cplusplus) || defined(c_plusplus) 158 | } /* extern "C" */ 159 | #endif 160 | -------------------------------------------------------------------------------- /c_src/enc/backward_references.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function to find backward reference copies. */ 8 | 9 | #include "./backward_references.h" 10 | 11 | #include "../common/constants.h" 12 | #include "../common/context.h" 13 | #include "../common/dictionary.h" 14 | #include "../common/platform.h" 15 | #include 16 | #include "./command.h" 17 | #include "./dictionary_hash.h" 18 | #include "./memory.h" 19 | #include "./quality.h" 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | extern "C" { 23 | #endif 24 | 25 | static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance, 26 | size_t max_distance, 27 | const int* dist_cache) { 28 | if (distance <= max_distance) { 29 | size_t distance_plus_3 = distance + 3; 30 | size_t offset0 = distance_plus_3 - (size_t)dist_cache[0]; 31 | size_t offset1 = distance_plus_3 - (size_t)dist_cache[1]; 32 | if (distance == (size_t)dist_cache[0]) { 33 | return 0; 34 | } else if (distance == (size_t)dist_cache[1]) { 35 | return 1; 36 | } else if (offset0 < 7) { 37 | return (0x9750468 >> (4 * offset0)) & 0xF; 38 | } else if (offset1 < 7) { 39 | return (0xFDB1ACE >> (4 * offset1)) & 0xF; 40 | } else if (distance == (size_t)dist_cache[2]) { 41 | return 2; 42 | } else if (distance == (size_t)dist_cache[3]) { 43 | return 3; 44 | } 45 | } 46 | return distance + BROTLI_NUM_DISTANCE_SHORT_CODES - 1; 47 | } 48 | 49 | #define EXPAND_CAT(a, b) CAT(a, b) 50 | #define CAT(a, b) a ## b 51 | #define FN(X) EXPAND_CAT(X, HASHER()) 52 | #define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER())) 53 | 54 | #define PREFIX() N 55 | 56 | #define HASHER() H2 57 | /* NOLINTNEXTLINE(build/include) */ 58 | #include "./backward_references_inc.h" 59 | #undef HASHER 60 | 61 | #define HASHER() H3 62 | /* NOLINTNEXTLINE(build/include) */ 63 | #include "./backward_references_inc.h" 64 | #undef HASHER 65 | 66 | #define HASHER() H4 67 | /* NOLINTNEXTLINE(build/include) */ 68 | #include "./backward_references_inc.h" 69 | #undef HASHER 70 | 71 | #define HASHER() H5 72 | /* NOLINTNEXTLINE(build/include) */ 73 | #include "./backward_references_inc.h" 74 | #undef HASHER 75 | 76 | #define HASHER() H6 77 | /* NOLINTNEXTLINE(build/include) */ 78 | #include "./backward_references_inc.h" 79 | #undef HASHER 80 | 81 | #define HASHER() H40 82 | /* NOLINTNEXTLINE(build/include) */ 83 | #include "./backward_references_inc.h" 84 | #undef HASHER 85 | 86 | #define HASHER() H41 87 | /* NOLINTNEXTLINE(build/include) */ 88 | #include "./backward_references_inc.h" 89 | #undef HASHER 90 | 91 | #define HASHER() H42 92 | /* NOLINTNEXTLINE(build/include) */ 93 | #include "./backward_references_inc.h" 94 | #undef HASHER 95 | 96 | #define HASHER() H54 97 | /* NOLINTNEXTLINE(build/include) */ 98 | #include "./backward_references_inc.h" 99 | #undef HASHER 100 | 101 | #define HASHER() H35 102 | /* NOLINTNEXTLINE(build/include) */ 103 | #include "./backward_references_inc.h" 104 | #undef HASHER 105 | 106 | #define HASHER() H55 107 | /* NOLINTNEXTLINE(build/include) */ 108 | #include "./backward_references_inc.h" 109 | #undef HASHER 110 | 111 | #define HASHER() H65 112 | /* NOLINTNEXTLINE(build/include) */ 113 | #include "./backward_references_inc.h" 114 | #undef HASHER 115 | 116 | #undef PREFIX 117 | 118 | #undef EXPORT_FN 119 | #undef FN 120 | #undef CAT 121 | #undef EXPAND_CAT 122 | 123 | void BrotliCreateBackwardReferences(size_t num_bytes, 124 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 125 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 126 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 127 | Command* commands, size_t* num_commands, size_t* num_literals) { 128 | switch (params->hasher.type) { 129 | #define CASE_(N) \ 130 | case N: \ 131 | CreateBackwardReferencesNH ## N(num_bytes, \ 132 | position, ringbuffer, ringbuffer_mask, \ 133 | literal_context_lut, params, hasher, dist_cache, \ 134 | last_insert_len, commands, num_commands, num_literals); \ 135 | return; 136 | FOR_GENERIC_HASHERS(CASE_) 137 | #undef CASE_ 138 | default: 139 | break; 140 | } 141 | } 142 | 143 | #if defined(__cplusplus) || defined(c_plusplus) 144 | } /* extern "C" */ 145 | #endif 146 | -------------------------------------------------------------------------------- /c_src/enc/backward_references.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function to find backward reference copies. */ 8 | 9 | #ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_ 10 | #define BROTLI_ENC_BACKWARD_REFERENCES_H_ 11 | 12 | #include "../common/constants.h" 13 | #include "../common/context.h" 14 | #include "../common/dictionary.h" 15 | #include "../common/platform.h" 16 | #include 17 | #include "./command.h" 18 | #include "./hash.h" 19 | #include "./quality.h" 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* "commands" points to the next output command to write to, "*num_commands" is 26 | initially the total amount of commands output by previous 27 | CreateBackwardReferences calls, and must be incremented by the amount written 28 | by this call. */ 29 | BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes, 30 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 31 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 32 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 33 | Command* commands, size_t* num_commands, size_t* num_literals); 34 | 35 | #if defined(__cplusplus) || defined(c_plusplus) 36 | } /* extern "C" */ 37 | #endif 38 | 39 | #endif /* BROTLI_ENC_BACKWARD_REFERENCES_H_ */ 40 | -------------------------------------------------------------------------------- /c_src/enc/backward_references_hq.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function to find backward reference copies. */ 8 | 9 | #ifndef BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ 10 | #define BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ 11 | 12 | #include "../common/constants.h" 13 | #include "../common/context.h" 14 | #include "../common/dictionary.h" 15 | #include "../common/platform.h" 16 | #include 17 | #include "./command.h" 18 | #include "./hash.h" 19 | #include "./memory.h" 20 | #include "./quality.h" 21 | 22 | #if defined(__cplusplus) || defined(c_plusplus) 23 | extern "C" { 24 | #endif 25 | 26 | BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m, 27 | size_t num_bytes, 28 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 29 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 30 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 31 | Command* commands, size_t* num_commands, size_t* num_literals); 32 | 33 | BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, 34 | size_t num_bytes, 35 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 36 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 37 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 38 | Command* commands, size_t* num_commands, size_t* num_literals); 39 | 40 | typedef struct ZopfliNode { 41 | /* Best length to get up to this byte (not including this byte itself) 42 | highest 7 bit is used to reconstruct the length code. */ 43 | uint32_t length; 44 | /* Distance associated with the length. */ 45 | uint32_t distance; 46 | /* Number of literal inserts before this copy; highest 5 bits contain 47 | distance short code + 1 (or zero if no short code). */ 48 | uint32_t dcode_insert_length; 49 | 50 | /* This union holds information used by dynamic-programming. During forward 51 | pass |cost| it used to store the goal function. When node is processed its 52 | |cost| is invalidated in favor of |shortcut|. On path back-tracing pass 53 | |next| is assigned the offset to next node on the path. */ 54 | union { 55 | /* Smallest cost to get to this byte from the beginning, as found so far. */ 56 | float cost; 57 | /* Offset to the next node on the path. Equals to command_length() of the 58 | next node on the path. For last node equals to BROTLI_UINT32_MAX */ 59 | uint32_t next; 60 | /* Node position that provides next distance for distance cache. */ 61 | uint32_t shortcut; 62 | } u; 63 | } ZopfliNode; 64 | 65 | BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length); 66 | 67 | /* Computes the shortest path of commands from position to at most 68 | position + num_bytes. 69 | 70 | On return, path->size() is the number of commands found and path[i] is the 71 | length of the i-th command (copy length plus insert length). 72 | Note that the sum of the lengths of all commands can be less than num_bytes. 73 | 74 | On return, the nodes[0..num_bytes] array will have the following 75 | "ZopfliNode array invariant": 76 | For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then 77 | (1) nodes[i].copy_length() >= 2 78 | (2) nodes[i].command_length() <= i and 79 | (3) nodes[i - nodes[i].command_length()].cost < kInfinity */ 80 | BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath( 81 | MemoryManager* m, size_t num_bytes, 82 | size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, 83 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 84 | const int* dist_cache, Hasher* hasher, ZopfliNode* nodes); 85 | 86 | BROTLI_INTERNAL void BrotliZopfliCreateCommands( 87 | const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes, 88 | int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params, 89 | Command* commands, size_t* num_literals); 90 | 91 | #if defined(__cplusplus) || defined(c_plusplus) 92 | } /* extern "C" */ 93 | #endif 94 | 95 | #endif /* BROTLI_ENC_BACKWARD_REFERENCES_HQ_H_ */ 96 | -------------------------------------------------------------------------------- /c_src/enc/backward_references_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2013 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: EXPORT_FN, FN */ 9 | 10 | static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( 11 | size_t num_bytes, size_t position, 12 | const uint8_t* ringbuffer, size_t ringbuffer_mask, 13 | ContextLut literal_context_lut, const BrotliEncoderParams* params, 14 | Hasher* hasher, int* dist_cache, size_t* last_insert_len, 15 | Command* commands, size_t* num_commands, size_t* num_literals) { 16 | HASHER()* privat = &hasher->privat.FN(_); 17 | /* Set maximum distance, see section 9.1. of the spec. */ 18 | const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); 19 | const size_t position_offset = params->stream_offset; 20 | 21 | const Command* const orig_commands = commands; 22 | size_t insert_length = *last_insert_len; 23 | const size_t pos_end = position + num_bytes; 24 | const size_t store_end = num_bytes >= FN(StoreLookahead)() ? 25 | position + num_bytes - FN(StoreLookahead)() + 1 : position; 26 | 27 | /* For speed up heuristics for random data. */ 28 | const size_t random_heuristics_window_size = 29 | LiteralSpreeLengthForSparseSearch(params); 30 | size_t apply_random_heuristics = position + random_heuristics_window_size; 31 | const size_t gap = 0; 32 | 33 | /* Minimum score to accept a backward reference. */ 34 | const score_t kMinScore = BROTLI_SCORE_BASE + 100; 35 | 36 | BROTLI_UNUSED(literal_context_lut); 37 | 38 | FN(PrepareDistanceCache)(privat, dist_cache); 39 | 40 | while (position + FN(HashTypeLength)() < pos_end) { 41 | size_t max_length = pos_end - position; 42 | size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit); 43 | size_t dictionary_start = BROTLI_MIN(size_t, 44 | position + position_offset, max_backward_limit); 45 | HasherSearchResult sr; 46 | sr.len = 0; 47 | sr.len_code_delta = 0; 48 | sr.distance = 0; 49 | sr.score = kMinScore; 50 | FN(FindLongestMatch)(privat, ¶ms->dictionary, 51 | ringbuffer, ringbuffer_mask, dist_cache, position, max_length, 52 | max_distance, dictionary_start + gap, params->dist.max_distance, &sr); 53 | if (sr.score > kMinScore) { 54 | /* Found a match. Let's look for something even better ahead. */ 55 | int delayed_backward_references_in_row = 0; 56 | --max_length; 57 | for (;; --max_length) { 58 | const score_t cost_diff_lazy = 175; 59 | HasherSearchResult sr2; 60 | sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ? 61 | BROTLI_MIN(size_t, sr.len - 1, max_length) : 0; 62 | sr2.len_code_delta = 0; 63 | sr2.distance = 0; 64 | sr2.score = kMinScore; 65 | max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit); 66 | dictionary_start = BROTLI_MIN(size_t, 67 | position + 1 + position_offset, max_backward_limit); 68 | FN(FindLongestMatch)(privat, 69 | ¶ms->dictionary, 70 | ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length, 71 | max_distance, dictionary_start + gap, params->dist.max_distance, 72 | &sr2); 73 | if (sr2.score >= sr.score + cost_diff_lazy) { 74 | /* Ok, let's just write one byte for now and start a match from the 75 | next byte. */ 76 | ++position; 77 | ++insert_length; 78 | sr = sr2; 79 | if (++delayed_backward_references_in_row < 4 && 80 | position + FN(HashTypeLength)() < pos_end) { 81 | continue; 82 | } 83 | } 84 | break; 85 | } 86 | apply_random_heuristics = 87 | position + 2 * sr.len + random_heuristics_window_size; 88 | dictionary_start = BROTLI_MIN(size_t, 89 | position + position_offset, max_backward_limit); 90 | { 91 | /* The first 16 codes are special short-codes, 92 | and the minimum offset is 1. */ 93 | size_t distance_code = ComputeDistanceCode( 94 | sr.distance, dictionary_start + gap, dist_cache); 95 | if ((sr.distance <= (dictionary_start + gap)) && distance_code > 0) { 96 | dist_cache[3] = dist_cache[2]; 97 | dist_cache[2] = dist_cache[1]; 98 | dist_cache[1] = dist_cache[0]; 99 | dist_cache[0] = (int)sr.distance; 100 | FN(PrepareDistanceCache)(privat, dist_cache); 101 | } 102 | InitCommand(commands++, ¶ms->dist, insert_length, 103 | sr.len, sr.len_code_delta, distance_code); 104 | } 105 | *num_literals += insert_length; 106 | insert_length = 0; 107 | /* Put the hash keys into the table, if there are enough bytes left. 108 | Depending on the hasher implementation, it can push all positions 109 | in the given range or only a subset of them. 110 | Avoid hash poisoning with RLE data. */ 111 | { 112 | size_t range_start = position + 2; 113 | size_t range_end = BROTLI_MIN(size_t, position + sr.len, store_end); 114 | if (sr.distance < (sr.len >> 2)) { 115 | range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t, 116 | range_start, position + sr.len - (sr.distance << 2))); 117 | } 118 | FN(StoreRange)(privat, ringbuffer, ringbuffer_mask, range_start, 119 | range_end); 120 | } 121 | position += sr.len; 122 | } else { 123 | ++insert_length; 124 | ++position; 125 | /* If we have not seen matches for a long time, we can skip some 126 | match lookups. Unsuccessful match lookups are very very expensive 127 | and this kind of a heuristic speeds up compression quite 128 | a lot. */ 129 | if (position > apply_random_heuristics) { 130 | /* Going through uncompressible data, jump. */ 131 | if (position > 132 | apply_random_heuristics + 4 * random_heuristics_window_size) { 133 | /* It is quite a long time since we saw a copy, so we assume 134 | that this data is not compressible, and store hashes less 135 | often. Hashes of non compressible data are less likely to 136 | turn out to be useful in the future, too, so we store less of 137 | them to not to flood out the hash table of good compressible 138 | data. */ 139 | const size_t kMargin = 140 | BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 4); 141 | size_t pos_jump = 142 | BROTLI_MIN(size_t, position + 16, pos_end - kMargin); 143 | for (; position < pos_jump; position += 4) { 144 | FN(Store)(privat, ringbuffer, ringbuffer_mask, position); 145 | insert_length += 4; 146 | } 147 | } else { 148 | const size_t kMargin = 149 | BROTLI_MAX(size_t, FN(StoreLookahead)() - 1, 2); 150 | size_t pos_jump = 151 | BROTLI_MIN(size_t, position + 8, pos_end - kMargin); 152 | for (; position < pos_jump; position += 2) { 153 | FN(Store)(privat, ringbuffer, ringbuffer_mask, position); 154 | insert_length += 2; 155 | } 156 | } 157 | } 158 | } 159 | } 160 | insert_length += pos_end - position; 161 | *last_insert_len = insert_length; 162 | *num_commands += (size_t)(commands - orig_commands); 163 | } 164 | -------------------------------------------------------------------------------- /c_src/enc/bit_cost.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions to estimate the bit cost of Huffman trees. */ 8 | 9 | #include "./bit_cost.h" 10 | 11 | #include "../common/constants.h" 12 | #include "../common/platform.h" 13 | #include 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | #define FN(X) X ## Literal 22 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 23 | #undef FN 24 | 25 | #define FN(X) X ## Command 26 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 27 | #undef FN 28 | 29 | #define FN(X) X ## Distance 30 | #include "./bit_cost_inc.h" /* NOLINT(build/include) */ 31 | #undef FN 32 | 33 | #if defined(__cplusplus) || defined(c_plusplus) 34 | } /* extern "C" */ 35 | #endif 36 | -------------------------------------------------------------------------------- /c_src/enc/bit_cost.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions to estimate the bit cost of Huffman trees. */ 8 | 9 | #ifndef BROTLI_ENC_BIT_COST_H_ 10 | #define BROTLI_ENC_BIT_COST_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | static BROTLI_INLINE double ShannonEntropy( 22 | const uint32_t* population, size_t size, size_t* total) { 23 | size_t sum = 0; 24 | double retval = 0; 25 | const uint32_t* population_end = population + size; 26 | size_t p; 27 | if (size & 1) { 28 | goto odd_number_of_elements_left; 29 | } 30 | while (population < population_end) { 31 | p = *population++; 32 | sum += p; 33 | retval -= (double)p * FastLog2(p); 34 | odd_number_of_elements_left: 35 | p = *population++; 36 | sum += p; 37 | retval -= (double)p * FastLog2(p); 38 | } 39 | if (sum) retval += (double)sum * FastLog2(sum); 40 | *total = sum; 41 | return retval; 42 | } 43 | 44 | static BROTLI_INLINE double BitsEntropy( 45 | const uint32_t* population, size_t size) { 46 | size_t sum; 47 | double retval = ShannonEntropy(population, size, &sum); 48 | if (retval < sum) { 49 | /* At least one bit per literal is needed. */ 50 | retval = (double)sum; 51 | } 52 | return retval; 53 | } 54 | 55 | BROTLI_INTERNAL double BrotliPopulationCostLiteral(const HistogramLiteral*); 56 | BROTLI_INTERNAL double BrotliPopulationCostCommand(const HistogramCommand*); 57 | BROTLI_INTERNAL double BrotliPopulationCostDistance(const HistogramDistance*); 58 | 59 | #if defined(__cplusplus) || defined(c_plusplus) 60 | } /* extern "C" */ 61 | #endif 62 | 63 | #endif /* BROTLI_ENC_BIT_COST_H_ */ 64 | -------------------------------------------------------------------------------- /c_src/enc/bit_cost_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2013 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: FN */ 9 | 10 | #define HistogramType FN(Histogram) 11 | 12 | double FN(BrotliPopulationCost)(const HistogramType* histogram) { 13 | static const double kOneSymbolHistogramCost = 12; 14 | static const double kTwoSymbolHistogramCost = 20; 15 | static const double kThreeSymbolHistogramCost = 28; 16 | static const double kFourSymbolHistogramCost = 37; 17 | const size_t data_size = FN(HistogramDataSize)(); 18 | int count = 0; 19 | size_t s[5]; 20 | double bits = 0.0; 21 | size_t i; 22 | if (histogram->total_count_ == 0) { 23 | return kOneSymbolHistogramCost; 24 | } 25 | for (i = 0; i < data_size; ++i) { 26 | if (histogram->data_[i] > 0) { 27 | s[count] = i; 28 | ++count; 29 | if (count > 4) break; 30 | } 31 | } 32 | if (count == 1) { 33 | return kOneSymbolHistogramCost; 34 | } 35 | if (count == 2) { 36 | return (kTwoSymbolHistogramCost + (double)histogram->total_count_); 37 | } 38 | if (count == 3) { 39 | const uint32_t histo0 = histogram->data_[s[0]]; 40 | const uint32_t histo1 = histogram->data_[s[1]]; 41 | const uint32_t histo2 = histogram->data_[s[2]]; 42 | const uint32_t histomax = 43 | BROTLI_MAX(uint32_t, histo0, BROTLI_MAX(uint32_t, histo1, histo2)); 44 | return (kThreeSymbolHistogramCost + 45 | 2 * (histo0 + histo1 + histo2) - histomax); 46 | } 47 | if (count == 4) { 48 | uint32_t histo[4]; 49 | uint32_t h23; 50 | uint32_t histomax; 51 | for (i = 0; i < 4; ++i) { 52 | histo[i] = histogram->data_[s[i]]; 53 | } 54 | /* Sort */ 55 | for (i = 0; i < 4; ++i) { 56 | size_t j; 57 | for (j = i + 1; j < 4; ++j) { 58 | if (histo[j] > histo[i]) { 59 | BROTLI_SWAP(uint32_t, histo, j, i); 60 | } 61 | } 62 | } 63 | h23 = histo[2] + histo[3]; 64 | histomax = BROTLI_MAX(uint32_t, h23, histo[0]); 65 | return (kFourSymbolHistogramCost + 66 | 3 * h23 + 2 * (histo[0] + histo[1]) - histomax); 67 | } 68 | 69 | { 70 | /* In this loop we compute the entropy of the histogram and simultaneously 71 | build a simplified histogram of the code length codes where we use the 72 | zero repeat code 17, but we don't use the non-zero repeat code 16. */ 73 | size_t max_depth = 1; 74 | uint32_t depth_histo[BROTLI_CODE_LENGTH_CODES] = { 0 }; 75 | const double log2total = FastLog2(histogram->total_count_); 76 | for (i = 0; i < data_size;) { 77 | if (histogram->data_[i] > 0) { 78 | /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = 79 | = log2(total_count) - log2(count(symbol)) */ 80 | double log2p = log2total - FastLog2(histogram->data_[i]); 81 | /* Approximate the bit depth by round(-log2(P(symbol))) */ 82 | size_t depth = (size_t)(log2p + 0.5); 83 | bits += histogram->data_[i] * log2p; 84 | if (depth > 15) { 85 | depth = 15; 86 | } 87 | if (depth > max_depth) { 88 | max_depth = depth; 89 | } 90 | ++depth_histo[depth]; 91 | ++i; 92 | } else { 93 | /* Compute the run length of zeros and add the appropriate number of 0 94 | and 17 code length codes to the code length code histogram. */ 95 | uint32_t reps = 1; 96 | size_t k; 97 | for (k = i + 1; k < data_size && histogram->data_[k] == 0; ++k) { 98 | ++reps; 99 | } 100 | i += reps; 101 | if (i == data_size) { 102 | /* Don't add any cost for the last zero run, since these are encoded 103 | only implicitly. */ 104 | break; 105 | } 106 | if (reps < 3) { 107 | depth_histo[0] += reps; 108 | } else { 109 | reps -= 2; 110 | while (reps > 0) { 111 | ++depth_histo[BROTLI_REPEAT_ZERO_CODE_LENGTH]; 112 | /* Add the 3 extra bits for the 17 code length code. */ 113 | bits += 3; 114 | reps >>= 3; 115 | } 116 | } 117 | } 118 | } 119 | /* Add the estimated encoding cost of the code length code histogram. */ 120 | bits += (double)(18 + 2 * max_depth); 121 | /* Add the entropy of the code length code histogram. */ 122 | bits += BitsEntropy(depth_histo, BROTLI_CODE_LENGTH_CODES); 123 | } 124 | return bits; 125 | } 126 | 127 | #undef HistogramType 128 | -------------------------------------------------------------------------------- /c_src/enc/block_encoder_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2014 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: FN */ 9 | 10 | #define HistogramType FN(Histogram) 11 | 12 | /* Creates entropy codes for all block types and stores them to the bit 13 | stream. */ 14 | static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self, 15 | const HistogramType* histograms, const size_t histograms_size, 16 | const size_t alphabet_size, HuffmanTree* tree, 17 | size_t* storage_ix, uint8_t* storage) { 18 | const size_t table_size = histograms_size * self->histogram_length_; 19 | self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size); 20 | self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size); 21 | if (BROTLI_IS_OOM(m)) return; 22 | 23 | { 24 | size_t i; 25 | for (i = 0; i < histograms_size; ++i) { 26 | size_t ix = i * self->histogram_length_; 27 | BuildAndStoreHuffmanTree(&histograms[i].data_[0], self->histogram_length_, 28 | alphabet_size, tree, &self->depths_[ix], &self->bits_[ix], 29 | storage_ix, storage); 30 | } 31 | } 32 | } 33 | 34 | #undef HistogramType 35 | -------------------------------------------------------------------------------- /c_src/enc/block_splitter.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Block split point selection utilities. */ 8 | 9 | #include "./block_splitter.h" 10 | 11 | #include /* memcpy, memset */ 12 | 13 | #include "../common/platform.h" 14 | #include "./bit_cost.h" 15 | #include "./cluster.h" 16 | #include "./command.h" 17 | #include "./fast_log.h" 18 | #include "./histogram.h" 19 | #include "./memory.h" 20 | #include "./quality.h" 21 | 22 | #if defined(__cplusplus) || defined(c_plusplus) 23 | extern "C" { 24 | #endif 25 | 26 | static const size_t kMaxLiteralHistograms = 100; 27 | static const size_t kMaxCommandHistograms = 50; 28 | static const double kLiteralBlockSwitchCost = 28.1; 29 | static const double kCommandBlockSwitchCost = 13.5; 30 | static const double kDistanceBlockSwitchCost = 14.6; 31 | static const size_t kLiteralStrideLength = 70; 32 | static const size_t kCommandStrideLength = 40; 33 | static const size_t kSymbolsPerLiteralHistogram = 544; 34 | static const size_t kSymbolsPerCommandHistogram = 530; 35 | static const size_t kSymbolsPerDistanceHistogram = 544; 36 | static const size_t kMinLengthForBlockSplitting = 128; 37 | static const size_t kIterMulForRefining = 2; 38 | static const size_t kMinItersForRefining = 100; 39 | 40 | static size_t CountLiterals(const Command* cmds, const size_t num_commands) { 41 | /* Count how many we have. */ 42 | size_t total_length = 0; 43 | size_t i; 44 | for (i = 0; i < num_commands; ++i) { 45 | total_length += cmds[i].insert_len_; 46 | } 47 | return total_length; 48 | } 49 | 50 | static void CopyLiteralsToByteArray(const Command* cmds, 51 | const size_t num_commands, 52 | const uint8_t* data, 53 | const size_t offset, 54 | const size_t mask, 55 | uint8_t* literals) { 56 | size_t pos = 0; 57 | size_t from_pos = offset & mask; 58 | size_t i; 59 | for (i = 0; i < num_commands; ++i) { 60 | size_t insert_len = cmds[i].insert_len_; 61 | if (from_pos + insert_len > mask) { 62 | size_t head_size = mask + 1 - from_pos; 63 | memcpy(literals + pos, data + from_pos, head_size); 64 | from_pos = 0; 65 | pos += head_size; 66 | insert_len -= head_size; 67 | } 68 | if (insert_len > 0) { 69 | memcpy(literals + pos, data + from_pos, insert_len); 70 | pos += insert_len; 71 | } 72 | from_pos = (from_pos + insert_len + CommandCopyLen(&cmds[i])) & mask; 73 | } 74 | } 75 | 76 | static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) { 77 | /* Initial seed should be 7. In this case, loop length is (1 << 29). */ 78 | *seed *= 16807U; 79 | return *seed; 80 | } 81 | 82 | static BROTLI_INLINE double BitCost(size_t count) { 83 | return count == 0 ? -2.0 : FastLog2(count); 84 | } 85 | 86 | #define HISTOGRAMS_PER_BATCH 64 87 | #define CLUSTERS_PER_BATCH 16 88 | 89 | #define FN(X) X ## Literal 90 | #define DataType uint8_t 91 | /* NOLINTNEXTLINE(build/include) */ 92 | #include "./block_splitter_inc.h" 93 | #undef DataType 94 | #undef FN 95 | 96 | #define FN(X) X ## Command 97 | #define DataType uint16_t 98 | /* NOLINTNEXTLINE(build/include) */ 99 | #include "./block_splitter_inc.h" 100 | #undef FN 101 | 102 | #define FN(X) X ## Distance 103 | /* NOLINTNEXTLINE(build/include) */ 104 | #include "./block_splitter_inc.h" 105 | #undef DataType 106 | #undef FN 107 | 108 | void BrotliInitBlockSplit(BlockSplit* self) { 109 | self->num_types = 0; 110 | self->num_blocks = 0; 111 | self->types = 0; 112 | self->lengths = 0; 113 | self->types_alloc_size = 0; 114 | self->lengths_alloc_size = 0; 115 | } 116 | 117 | void BrotliDestroyBlockSplit(MemoryManager* m, BlockSplit* self) { 118 | BROTLI_FREE(m, self->types); 119 | BROTLI_FREE(m, self->lengths); 120 | } 121 | 122 | void BrotliSplitBlock(MemoryManager* m, 123 | const Command* cmds, 124 | const size_t num_commands, 125 | const uint8_t* data, 126 | const size_t pos, 127 | const size_t mask, 128 | const BrotliEncoderParams* params, 129 | BlockSplit* literal_split, 130 | BlockSplit* insert_and_copy_split, 131 | BlockSplit* dist_split) { 132 | { 133 | size_t literals_count = CountLiterals(cmds, num_commands); 134 | uint8_t* literals = BROTLI_ALLOC(m, uint8_t, literals_count); 135 | if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(literals)) return; 136 | /* Create a continuous array of literals. */ 137 | CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals); 138 | /* Create the block split on the array of literals. 139 | Literal histograms have alphabet size 256. */ 140 | SplitByteVectorLiteral( 141 | m, literals, literals_count, 142 | kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, 143 | kLiteralStrideLength, kLiteralBlockSwitchCost, params, 144 | literal_split); 145 | if (BROTLI_IS_OOM(m)) return; 146 | BROTLI_FREE(m, literals); 147 | } 148 | 149 | { 150 | /* Compute prefix codes for commands. */ 151 | uint16_t* insert_and_copy_codes = BROTLI_ALLOC(m, uint16_t, num_commands); 152 | size_t i; 153 | if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(insert_and_copy_codes)) return; 154 | for (i = 0; i < num_commands; ++i) { 155 | insert_and_copy_codes[i] = cmds[i].cmd_prefix_; 156 | } 157 | /* Create the block split on the array of command prefixes. */ 158 | SplitByteVectorCommand( 159 | m, insert_and_copy_codes, num_commands, 160 | kSymbolsPerCommandHistogram, kMaxCommandHistograms, 161 | kCommandStrideLength, kCommandBlockSwitchCost, params, 162 | insert_and_copy_split); 163 | if (BROTLI_IS_OOM(m)) return; 164 | /* TODO: reuse for distances? */ 165 | BROTLI_FREE(m, insert_and_copy_codes); 166 | } 167 | 168 | { 169 | /* Create a continuous array of distance prefixes. */ 170 | uint16_t* distance_prefixes = BROTLI_ALLOC(m, uint16_t, num_commands); 171 | size_t j = 0; 172 | size_t i; 173 | if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(distance_prefixes)) return; 174 | for (i = 0; i < num_commands; ++i) { 175 | const Command* cmd = &cmds[i]; 176 | if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) { 177 | distance_prefixes[j++] = cmd->dist_prefix_ & 0x3FF; 178 | } 179 | } 180 | /* Create the block split on the array of distance prefixes. */ 181 | SplitByteVectorDistance( 182 | m, distance_prefixes, j, 183 | kSymbolsPerDistanceHistogram, kMaxCommandHistograms, 184 | kCommandStrideLength, kDistanceBlockSwitchCost, params, 185 | dist_split); 186 | if (BROTLI_IS_OOM(m)) return; 187 | BROTLI_FREE(m, distance_prefixes); 188 | } 189 | } 190 | 191 | 192 | #if defined(__cplusplus) || defined(c_plusplus) 193 | } /* extern "C" */ 194 | #endif 195 | -------------------------------------------------------------------------------- /c_src/enc/block_splitter.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Block split point selection utilities. */ 8 | 9 | #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_ 10 | #define BROTLI_ENC_BLOCK_SPLITTER_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./command.h" 15 | #include "./memory.h" 16 | #include "./quality.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | typedef struct BlockSplit { 23 | size_t num_types; /* Amount of distinct types */ 24 | size_t num_blocks; /* Amount of values in types and length */ 25 | uint8_t* types; 26 | uint32_t* lengths; 27 | 28 | size_t types_alloc_size; 29 | size_t lengths_alloc_size; 30 | } BlockSplit; 31 | 32 | BROTLI_INTERNAL void BrotliInitBlockSplit(BlockSplit* self); 33 | BROTLI_INTERNAL void BrotliDestroyBlockSplit(MemoryManager* m, 34 | BlockSplit* self); 35 | 36 | BROTLI_INTERNAL void BrotliSplitBlock(MemoryManager* m, 37 | const Command* cmds, 38 | const size_t num_commands, 39 | const uint8_t* data, 40 | const size_t offset, 41 | const size_t mask, 42 | const BrotliEncoderParams* params, 43 | BlockSplit* literal_split, 44 | BlockSplit* insert_and_copy_split, 45 | BlockSplit* dist_split); 46 | 47 | #if defined(__cplusplus) || defined(c_plusplus) 48 | } /* extern "C" */ 49 | #endif 50 | 51 | #endif /* BROTLI_ENC_BLOCK_SPLITTER_H_ */ 52 | -------------------------------------------------------------------------------- /c_src/enc/brotli_bit_stream.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2014 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions to convert brotli-related data structures into the 8 | brotli bit stream. The functions here operate under 9 | assumption that there is enough space in the storage, i.e., there are 10 | no out-of-range checks anywhere. 11 | 12 | These functions do bit addressing into a byte array. The byte array 13 | is called "storage" and the index to the bit is called storage_ix 14 | in function arguments. */ 15 | 16 | #ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_ 17 | #define BROTLI_ENC_BROTLI_BIT_STREAM_H_ 18 | 19 | #include "../common/context.h" 20 | #include "../common/platform.h" 21 | #include 22 | #include "./command.h" 23 | #include "./entropy_encode.h" 24 | #include "./memory.h" 25 | #include "./metablock.h" 26 | 27 | #if defined(__cplusplus) || defined(c_plusplus) 28 | extern "C" { 29 | #endif 30 | 31 | /* All Store functions here will use a storage_ix, which is always the bit 32 | position for the current storage. */ 33 | 34 | BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num, 35 | HuffmanTree* tree, size_t* storage_ix, uint8_t* storage); 36 | 37 | BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast( 38 | MemoryManager* m, const uint32_t* histogram, const size_t histogram_total, 39 | const size_t max_bits, uint8_t* depth, uint16_t* bits, size_t* storage_ix, 40 | uint8_t* storage); 41 | 42 | /* REQUIRES: length > 0 */ 43 | /* REQUIRES: length <= (1 << 24) */ 44 | BROTLI_INTERNAL void BrotliStoreMetaBlock(MemoryManager* m, 45 | const uint8_t* input, size_t start_pos, size_t length, size_t mask, 46 | uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last, 47 | const BrotliEncoderParams* params, ContextType literal_context_mode, 48 | const Command* commands, size_t n_commands, const MetaBlockSplit* mb, 49 | size_t* storage_ix, uint8_t* storage); 50 | 51 | /* Stores the meta-block without doing any block splitting, just collects 52 | one histogram per block category and uses that for entropy coding. 53 | REQUIRES: length > 0 54 | REQUIRES: length <= (1 << 24) */ 55 | BROTLI_INTERNAL void BrotliStoreMetaBlockTrivial(MemoryManager* m, 56 | const uint8_t* input, size_t start_pos, size_t length, size_t mask, 57 | BROTLI_BOOL is_last, const BrotliEncoderParams* params, 58 | const Command* commands, size_t n_commands, 59 | size_t* storage_ix, uint8_t* storage); 60 | 61 | /* Same as above, but uses static prefix codes for histograms with a only a few 62 | symbols, and uses static code length prefix codes for all other histograms. 63 | REQUIRES: length > 0 64 | REQUIRES: length <= (1 << 24) */ 65 | BROTLI_INTERNAL void BrotliStoreMetaBlockFast(MemoryManager* m, 66 | const uint8_t* input, size_t start_pos, size_t length, size_t mask, 67 | BROTLI_BOOL is_last, const BrotliEncoderParams* params, 68 | const Command* commands, size_t n_commands, 69 | size_t* storage_ix, uint8_t* storage); 70 | 71 | /* This is for storing uncompressed blocks (simple raw storage of 72 | bytes-as-bytes). 73 | REQUIRES: length > 0 74 | REQUIRES: length <= (1 << 24) */ 75 | BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock( 76 | BROTLI_BOOL is_final_block, const uint8_t* BROTLI_RESTRICT input, 77 | size_t position, size_t mask, size_t len, 78 | size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage); 79 | 80 | #if defined(__cplusplus) || defined(c_plusplus) 81 | } /* extern "C" */ 82 | #endif 83 | 84 | #endif /* BROTLI_ENC_BROTLI_BIT_STREAM_H_ */ 85 | -------------------------------------------------------------------------------- /c_src/enc/cluster.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for clustering similar histograms together. */ 8 | 9 | #include "./cluster.h" 10 | 11 | #include "../common/platform.h" 12 | #include 13 | #include "./bit_cost.h" /* BrotliPopulationCost */ 14 | #include "./fast_log.h" 15 | #include "./histogram.h" 16 | #include "./memory.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | static BROTLI_INLINE BROTLI_BOOL HistogramPairIsLess( 23 | const HistogramPair* p1, const HistogramPair* p2) { 24 | if (p1->cost_diff != p2->cost_diff) { 25 | return TO_BROTLI_BOOL(p1->cost_diff > p2->cost_diff); 26 | } 27 | return TO_BROTLI_BOOL((p1->idx2 - p1->idx1) > (p2->idx2 - p2->idx1)); 28 | } 29 | 30 | /* Returns entropy reduction of the context map when we combine two clusters. */ 31 | static BROTLI_INLINE double ClusterCostDiff(size_t size_a, size_t size_b) { 32 | size_t size_c = size_a + size_b; 33 | return (double)size_a * FastLog2(size_a) + 34 | (double)size_b * FastLog2(size_b) - 35 | (double)size_c * FastLog2(size_c); 36 | } 37 | 38 | #define CODE(X) X 39 | 40 | #define FN(X) X ## Literal 41 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 42 | #undef FN 43 | 44 | #define FN(X) X ## Command 45 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 46 | #undef FN 47 | 48 | #define FN(X) X ## Distance 49 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 50 | #undef FN 51 | 52 | #undef CODE 53 | 54 | #if defined(__cplusplus) || defined(c_plusplus) 55 | } /* extern "C" */ 56 | #endif 57 | -------------------------------------------------------------------------------- /c_src/enc/cluster.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for clustering similar histograms together. */ 8 | 9 | #ifndef BROTLI_ENC_CLUSTER_H_ 10 | #define BROTLI_ENC_CLUSTER_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./histogram.h" 15 | #include "./memory.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | typedef struct HistogramPair { 22 | uint32_t idx1; 23 | uint32_t idx2; 24 | double cost_combo; 25 | double cost_diff; 26 | } HistogramPair; 27 | 28 | #define CODE(X) /* Declaration */; 29 | 30 | #define FN(X) X ## Literal 31 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 32 | #undef FN 33 | 34 | #define FN(X) X ## Command 35 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 36 | #undef FN 37 | 38 | #define FN(X) X ## Distance 39 | #include "./cluster_inc.h" /* NOLINT(build/include) */ 40 | #undef FN 41 | 42 | #undef CODE 43 | 44 | #if defined(__cplusplus) || defined(c_plusplus) 45 | } /* extern "C" */ 46 | #endif 47 | 48 | #endif /* BROTLI_ENC_CLUSTER_H_ */ 49 | -------------------------------------------------------------------------------- /c_src/enc/command.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./command.h" 8 | 9 | #include 10 | 11 | #if defined(__cplusplus) || defined(c_plusplus) 12 | extern "C" { 13 | #endif 14 | 15 | const uint32_t kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES] = { 16 | 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 17 | 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594}; 18 | const uint32_t kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES] = { 19 | 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24}; 20 | const uint32_t kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES] = { 21 | 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22 | 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, 2118}; 23 | const uint32_t kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES] = { 24 | 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24}; 25 | 26 | #if defined(__cplusplus) || defined(c_plusplus) 27 | } /* extern "C" */ 28 | #endif 29 | -------------------------------------------------------------------------------- /c_src/enc/command.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* This class models a sequence of literals and a backward reference copy. */ 8 | 9 | #ifndef BROTLI_ENC_COMMAND_H_ 10 | #define BROTLI_ENC_COMMAND_H_ 11 | 12 | #include "../common/constants.h" 13 | #include "../common/platform.h" 14 | #include 15 | #include "./fast_log.h" 16 | #include "./params.h" 17 | #include "./prefix.h" 18 | 19 | #if defined(__cplusplus) || defined(c_plusplus) 20 | extern "C" { 21 | #endif 22 | 23 | BROTLI_INTERNAL extern const uint32_t 24 | kBrotliInsBase[BROTLI_NUM_INS_COPY_CODES]; 25 | BROTLI_INTERNAL extern const uint32_t 26 | kBrotliInsExtra[BROTLI_NUM_INS_COPY_CODES]; 27 | BROTLI_INTERNAL extern const uint32_t 28 | kBrotliCopyBase[BROTLI_NUM_INS_COPY_CODES]; 29 | BROTLI_INTERNAL extern const uint32_t 30 | kBrotliCopyExtra[BROTLI_NUM_INS_COPY_CODES]; 31 | 32 | static BROTLI_INLINE uint16_t GetInsertLengthCode(size_t insertlen) { 33 | if (insertlen < 6) { 34 | return (uint16_t)insertlen; 35 | } else if (insertlen < 130) { 36 | uint32_t nbits = Log2FloorNonZero(insertlen - 2) - 1u; 37 | return (uint16_t)((nbits << 1) + ((insertlen - 2) >> nbits) + 2); 38 | } else if (insertlen < 2114) { 39 | return (uint16_t)(Log2FloorNonZero(insertlen - 66) + 10); 40 | } else if (insertlen < 6210) { 41 | return 21u; 42 | } else if (insertlen < 22594) { 43 | return 22u; 44 | } else { 45 | return 23u; 46 | } 47 | } 48 | 49 | static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) { 50 | if (copylen < 10) { 51 | return (uint16_t)(copylen - 2); 52 | } else if (copylen < 134) { 53 | uint32_t nbits = Log2FloorNonZero(copylen - 6) - 1u; 54 | return (uint16_t)((nbits << 1) + ((copylen - 6) >> nbits) + 4); 55 | } else if (copylen < 2118) { 56 | return (uint16_t)(Log2FloorNonZero(copylen - 70) + 12); 57 | } else { 58 | return 23u; 59 | } 60 | } 61 | 62 | static BROTLI_INLINE uint16_t CombineLengthCodes( 63 | uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) { 64 | uint16_t bits64 = 65 | (uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u)); 66 | if (use_last_distance && inscode < 8u && copycode < 16u) { 67 | return (copycode < 8u) ? bits64 : (bits64 | 64u); 68 | } else { 69 | /* Specification: 5 Encoding of ... (last table) */ 70 | /* offset = 2 * index, where index is in range [0..8] */ 71 | uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u)); 72 | /* All values in specification are K * 64, 73 | where K = [2, 3, 6, 4, 5, 8, 7, 9, 10], 74 | i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9], 75 | K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D. 76 | All values in D require only 2 bits to encode. 77 | Magic constant is shifted 6 bits left, to avoid final multiplication. */ 78 | offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u); 79 | return (uint16_t)(offset | bits64); 80 | } 81 | } 82 | 83 | static BROTLI_INLINE void GetLengthCode(size_t insertlen, size_t copylen, 84 | BROTLI_BOOL use_last_distance, 85 | uint16_t* code) { 86 | uint16_t inscode = GetInsertLengthCode(insertlen); 87 | uint16_t copycode = GetCopyLengthCode(copylen); 88 | *code = CombineLengthCodes(inscode, copycode, use_last_distance); 89 | } 90 | 91 | static BROTLI_INLINE uint32_t GetInsertBase(uint16_t inscode) { 92 | return kBrotliInsBase[inscode]; 93 | } 94 | 95 | static BROTLI_INLINE uint32_t GetInsertExtra(uint16_t inscode) { 96 | return kBrotliInsExtra[inscode]; 97 | } 98 | 99 | static BROTLI_INLINE uint32_t GetCopyBase(uint16_t copycode) { 100 | return kBrotliCopyBase[copycode]; 101 | } 102 | 103 | static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) { 104 | return kBrotliCopyExtra[copycode]; 105 | } 106 | 107 | typedef struct Command { 108 | uint32_t insert_len_; 109 | /* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */ 110 | uint32_t copy_len_; 111 | /* Stores distance extra bits. */ 112 | uint32_t dist_extra_; 113 | uint16_t cmd_prefix_; 114 | /* Stores distance code in low 10 bits 115 | and number of extra bits in high 6 bits. */ 116 | uint16_t dist_prefix_; 117 | } Command; 118 | 119 | /* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ 120 | static BROTLI_INLINE void InitCommand(Command* self, 121 | const BrotliDistanceParams* dist, size_t insertlen, 122 | size_t copylen, int copylen_code_delta, size_t distance_code) { 123 | /* Don't rely on signed int representation, use honest casts. */ 124 | uint32_t delta = (uint8_t)((int8_t)copylen_code_delta); 125 | self->insert_len_ = (uint32_t)insertlen; 126 | self->copy_len_ = (uint32_t)(copylen | (delta << 25)); 127 | /* The distance prefix and extra bits are stored in this Command as if 128 | npostfix and ndirect were 0, they are only recomputed later after the 129 | clustering if needed. */ 130 | PrefixEncodeCopyDistance( 131 | distance_code, dist->num_direct_distance_codes, 132 | dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_); 133 | GetLengthCode( 134 | insertlen, (size_t)((int)copylen + copylen_code_delta), 135 | TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_); 136 | } 137 | 138 | static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) { 139 | self->insert_len_ = (uint32_t)insertlen; 140 | self->copy_len_ = 4 << 25; 141 | self->dist_extra_ = 0; 142 | self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES; 143 | GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_); 144 | } 145 | 146 | static BROTLI_INLINE uint32_t CommandRestoreDistanceCode( 147 | const Command* self, const BrotliDistanceParams* dist) { 148 | if ((self->dist_prefix_ & 0x3FFu) < 149 | BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) { 150 | return self->dist_prefix_ & 0x3FFu; 151 | } else { 152 | uint32_t dcode = self->dist_prefix_ & 0x3FFu; 153 | uint32_t nbits = self->dist_prefix_ >> 10; 154 | uint32_t extra = self->dist_extra_; 155 | uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U; 156 | uint32_t hcode = (dcode - dist->num_direct_distance_codes - 157 | BROTLI_NUM_DISTANCE_SHORT_CODES) >> 158 | dist->distance_postfix_bits; 159 | uint32_t lcode = (dcode - dist->num_direct_distance_codes - 160 | BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask; 161 | uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U; 162 | return ((offset + extra) << dist->distance_postfix_bits) + lcode + 163 | dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES; 164 | } 165 | } 166 | 167 | static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) { 168 | uint32_t r = self->cmd_prefix_ >> 6; 169 | uint32_t c = self->cmd_prefix_ & 7; 170 | if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) { 171 | return c; 172 | } 173 | return 3; 174 | } 175 | 176 | static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) { 177 | return self->copy_len_ & 0x1FFFFFF; 178 | } 179 | 180 | static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) { 181 | uint32_t modifier = self->copy_len_ >> 25; 182 | int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1))); 183 | return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta); 184 | } 185 | 186 | #if defined(__cplusplus) || defined(c_plusplus) 187 | } /* extern "C" */ 188 | #endif 189 | 190 | #endif /* BROTLI_ENC_COMMAND_H_ */ 191 | -------------------------------------------------------------------------------- /c_src/enc/compress_fragment.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function for fast encoding of an input fragment, independently from the input 8 | history. This function uses one-pass processing: when we find a backward 9 | match, we immediately emit the corresponding command and literal codes to 10 | the bit stream. */ 11 | 12 | #ifndef BROTLI_ENC_COMPRESS_FRAGMENT_H_ 13 | #define BROTLI_ENC_COMPRESS_FRAGMENT_H_ 14 | 15 | #include "../common/platform.h" 16 | #include 17 | #include "./memory.h" 18 | 19 | #if defined(__cplusplus) || defined(c_plusplus) 20 | extern "C" { 21 | #endif 22 | 23 | /* Compresses "input" string to the "*storage" buffer as one or more complete 24 | meta-blocks, and updates the "*storage_ix" bit position. 25 | 26 | If "is_last" is 1, emits an additional empty last meta-block. 27 | 28 | "cmd_depth" and "cmd_bits" contain the command and distance prefix codes 29 | (see comment in encode.h) used for the encoding of this input fragment. 30 | If "is_last" is 0, they are updated to reflect the statistics 31 | of this input fragment, to be used for the encoding of the next fragment. 32 | 33 | "*cmd_code_numbits" is the number of bits of the compressed representation 34 | of the command and distance prefix codes, and "cmd_code" is an array of 35 | at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed 36 | command and distance prefix codes. If "is_last" is 0, these are also 37 | updated to represent the updated "cmd_depth" and "cmd_bits". 38 | 39 | REQUIRES: "input_size" is greater than zero, or "is_last" is 1. 40 | REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). 41 | REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. 42 | REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two 43 | OUTPUT: maximal copy distance <= |input_size| 44 | OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ 45 | BROTLI_INTERNAL void BrotliCompressFragmentFast(MemoryManager* m, 46 | const uint8_t* input, 47 | size_t input_size, 48 | BROTLI_BOOL is_last, 49 | int* table, size_t table_size, 50 | uint8_t cmd_depth[128], 51 | uint16_t cmd_bits[128], 52 | size_t* cmd_code_numbits, 53 | uint8_t* cmd_code, 54 | size_t* storage_ix, 55 | uint8_t* storage); 56 | 57 | #if defined(__cplusplus) || defined(c_plusplus) 58 | } /* extern "C" */ 59 | #endif 60 | 61 | #endif /* BROTLI_ENC_COMPRESS_FRAGMENT_H_ */ 62 | -------------------------------------------------------------------------------- /c_src/enc/compress_fragment_two_pass.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function for fast encoding of an input fragment, independently from the input 8 | history. This function uses two-pass processing: in the first pass we save 9 | the found backward matches and literal bytes into a buffer, and in the 10 | second pass we emit them into the bit stream using prefix codes built based 11 | on the actual command and literal byte histograms. */ 12 | 13 | #ifndef BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_ 14 | #define BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_ 15 | 16 | #include "../common/platform.h" 17 | #include 18 | #include "./memory.h" 19 | 20 | #if defined(__cplusplus) || defined(c_plusplus) 21 | extern "C" { 22 | #endif 23 | 24 | static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17; 25 | 26 | /* Compresses "input" string to the "*storage" buffer as one or more complete 27 | meta-blocks, and updates the "*storage_ix" bit position. 28 | 29 | If "is_last" is 1, emits an additional empty last meta-block. 30 | 31 | REQUIRES: "input_size" is greater than zero, or "is_last" is 1. 32 | REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). 33 | REQUIRES: "command_buf" and "literal_buf" point to at least 34 | kCompressFragmentTwoPassBlockSize long arrays. 35 | REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. 36 | REQUIRES: "table_size" is a power of two 37 | OUTPUT: maximal copy distance <= |input_size| 38 | OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ 39 | BROTLI_INTERNAL void BrotliCompressFragmentTwoPass(MemoryManager* m, 40 | const uint8_t* input, 41 | size_t input_size, 42 | BROTLI_BOOL is_last, 43 | uint32_t* command_buf, 44 | uint8_t* literal_buf, 45 | int* table, 46 | size_t table_size, 47 | size_t* storage_ix, 48 | uint8_t* storage); 49 | 50 | #if defined(__cplusplus) || defined(c_plusplus) 51 | } /* extern "C" */ 52 | #endif 53 | 54 | #endif /* BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_ */ 55 | -------------------------------------------------------------------------------- /c_src/enc/dictionary_hash.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Hash table on the 4-byte prefixes of static dictionary words. */ 8 | 9 | #ifndef BROTLI_ENC_DICTIONARY_HASH_H_ 10 | #define BROTLI_ENC_DICTIONARY_HASH_H_ 11 | 12 | #include 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | extern const uint16_t kStaticDictionaryHashWords[32768]; 19 | extern const uint8_t kStaticDictionaryHashLengths[32768]; 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | } /* extern "C" */ 23 | #endif 24 | 25 | #endif /* BROTLI_ENC_DICTIONARY_HASH_H_ */ 26 | -------------------------------------------------------------------------------- /c_src/enc/encoder_dict.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./encoder_dict.h" 8 | 9 | #include "../common/dictionary.h" 10 | #include "../common/transform.h" 11 | #include "./dictionary_hash.h" 12 | #include "./hash.h" 13 | 14 | #if defined(__cplusplus) || defined(c_plusplus) 15 | extern "C" { 16 | #endif 17 | 18 | void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) { 19 | dict->words = BrotliGetDictionary(); 20 | dict->num_transforms = (uint32_t)BrotliGetTransforms()->num_transforms; 21 | 22 | dict->hash_table_words = kStaticDictionaryHashWords; 23 | dict->hash_table_lengths = kStaticDictionaryHashLengths; 24 | dict->buckets = kStaticDictionaryBuckets; 25 | dict->dict_words = kStaticDictionaryWords; 26 | 27 | dict->cutoffTransformsCount = kCutoffTransformsCount; 28 | dict->cutoffTransforms = kCutoffTransforms; 29 | } 30 | 31 | #if defined(__cplusplus) || defined(c_plusplus) 32 | } /* extern "C" */ 33 | #endif 34 | -------------------------------------------------------------------------------- /c_src/enc/encoder_dict.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #ifndef BROTLI_ENC_ENCODER_DICT_H_ 8 | #define BROTLI_ENC_ENCODER_DICT_H_ 9 | 10 | #include "../common/dictionary.h" 11 | #include "../common/platform.h" 12 | #include 13 | #include "./static_dict_lut.h" 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /* Dictionary data (words and transforms) for 1 possible context */ 20 | typedef struct BrotliEncoderDictionary { 21 | const BrotliDictionary* words; 22 | uint32_t num_transforms; 23 | 24 | /* cut off for fast encoder */ 25 | uint32_t cutoffTransformsCount; 26 | uint64_t cutoffTransforms; 27 | 28 | /* from dictionary_hash.h, for fast encoder */ 29 | const uint16_t* hash_table_words; 30 | const uint8_t* hash_table_lengths; 31 | 32 | /* from static_dict_lut.h, for slow encoder */ 33 | const uint16_t* buckets; 34 | const DictWord* dict_words; 35 | } BrotliEncoderDictionary; 36 | 37 | BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict); 38 | 39 | #if defined(__cplusplus) || defined(c_plusplus) 40 | } /* extern "C" */ 41 | #endif 42 | 43 | #endif /* BROTLI_ENC_ENCODER_DICT_H_ */ 44 | -------------------------------------------------------------------------------- /c_src/enc/entropy_encode.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2010 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Entropy encoding (Huffman) utilities. */ 8 | 9 | #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_ 10 | #define BROTLI_ENC_ENTROPY_ENCODE_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /* A node of a Huffman tree. */ 20 | typedef struct HuffmanTree { 21 | uint32_t total_count_; 22 | int16_t index_left_; 23 | int16_t index_right_or_value_; 24 | } HuffmanTree; 25 | 26 | static BROTLI_INLINE void InitHuffmanTree(HuffmanTree* self, uint32_t count, 27 | int16_t left, int16_t right) { 28 | self->total_count_ = count; 29 | self->index_left_ = left; 30 | self->index_right_or_value_ = right; 31 | } 32 | 33 | /* Returns 1 is assignment of depths succeeded, otherwise 0. */ 34 | BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth( 35 | int p, HuffmanTree* pool, uint8_t* depth, int max_depth); 36 | 37 | /* This function will create a Huffman tree. 38 | 39 | The (data,length) contains the population counts. 40 | The tree_limit is the maximum bit depth of the Huffman codes. 41 | 42 | The depth contains the tree, i.e., how many bits are used for 43 | the symbol. 44 | 45 | The actual Huffman tree is constructed in the tree[] array, which has to 46 | be at least 2 * length + 1 long. 47 | 48 | See http://en.wikipedia.org/wiki/Huffman_coding */ 49 | BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data, 50 | const size_t length, 51 | const int tree_limit, 52 | HuffmanTree* tree, 53 | uint8_t* depth); 54 | 55 | /* Change the population counts in a way that the consequent 56 | Huffman tree compression, especially its RLE-part will be more 57 | likely to compress this data more efficiently. 58 | 59 | length contains the size of the histogram. 60 | counts contains the population counts. 61 | good_for_rle is a buffer of at least length size */ 62 | BROTLI_INTERNAL void BrotliOptimizeHuffmanCountsForRle( 63 | size_t length, uint32_t* counts, uint8_t* good_for_rle); 64 | 65 | /* Write a Huffman tree from bit depths into the bit-stream representation 66 | of a Huffman tree. The generated Huffman tree is to be compressed once 67 | more using a Huffman tree */ 68 | BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth, 69 | size_t num, 70 | size_t* tree_size, 71 | uint8_t* tree, 72 | uint8_t* extra_bits_data); 73 | 74 | /* Get the actual bit values for a tree of bit depths. */ 75 | BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth, 76 | size_t len, 77 | uint16_t* bits); 78 | 79 | BROTLI_INTERNAL extern const size_t kBrotliShellGaps[6]; 80 | /* Input size optimized Shell sort. */ 81 | typedef BROTLI_BOOL (*HuffmanTreeComparator)( 82 | const HuffmanTree*, const HuffmanTree*); 83 | static BROTLI_INLINE void SortHuffmanTreeItems(HuffmanTree* items, 84 | const size_t n, HuffmanTreeComparator comparator) { 85 | if (n < 13) { 86 | /* Insertion sort. */ 87 | size_t i; 88 | for (i = 1; i < n; ++i) { 89 | HuffmanTree tmp = items[i]; 90 | size_t k = i; 91 | size_t j = i - 1; 92 | while (comparator(&tmp, &items[j])) { 93 | items[k] = items[j]; 94 | k = j; 95 | if (!j--) break; 96 | } 97 | items[k] = tmp; 98 | } 99 | return; 100 | } else { 101 | /* Shell sort. */ 102 | int g = n < 57 ? 2 : 0; 103 | for (; g < 6; ++g) { 104 | size_t gap = kBrotliShellGaps[g]; 105 | size_t i; 106 | for (i = gap; i < n; ++i) { 107 | size_t j = i; 108 | HuffmanTree tmp = items[i]; 109 | for (; j >= gap && comparator(&tmp, &items[j - gap]); j -= gap) { 110 | items[j] = items[j - gap]; 111 | } 112 | items[j] = tmp; 113 | } 114 | } 115 | } 116 | } 117 | 118 | #if defined(__cplusplus) || defined(c_plusplus) 119 | } /* extern "C" */ 120 | #endif 121 | 122 | #endif /* BROTLI_ENC_ENTROPY_ENCODE_H_ */ 123 | -------------------------------------------------------------------------------- /c_src/enc/fast_log.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | #include "./fast_log.h" 8 | 9 | #if defined(__cplusplus) || defined(c_plusplus) 10 | extern "C" { 11 | #endif 12 | 13 | /* ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */ 14 | const double kBrotliLog2Table[BROTLI_LOG2_TABLE_SIZE] = { 15 | 0.0000000000000000f, 0.0000000000000000f, 1.0000000000000000f, 16 | 1.5849625007211563f, 2.0000000000000000f, 2.3219280948873622f, 17 | 2.5849625007211561f, 2.8073549220576042f, 3.0000000000000000f, 18 | 3.1699250014423126f, 3.3219280948873626f, 3.4594316186372978f, 19 | 3.5849625007211565f, 3.7004397181410922f, 3.8073549220576037f, 20 | 3.9068905956085187f, 4.0000000000000000f, 4.0874628412503400f, 21 | 4.1699250014423122f, 4.2479275134435852f, 4.3219280948873626f, 22 | 4.3923174227787607f, 4.4594316186372973f, 4.5235619560570131f, 23 | 4.5849625007211570f, 4.6438561897747244f, 4.7004397181410926f, 24 | 4.7548875021634691f, 4.8073549220576037f, 4.8579809951275728f, 25 | 4.9068905956085187f, 4.9541963103868758f, 5.0000000000000000f, 26 | 5.0443941193584534f, 5.0874628412503400f, 5.1292830169449664f, 27 | 5.1699250014423122f, 5.2094533656289501f, 5.2479275134435852f, 28 | 5.2854022188622487f, 5.3219280948873626f, 5.3575520046180838f, 29 | 5.3923174227787607f, 5.4262647547020979f, 5.4594316186372973f, 30 | 5.4918530963296748f, 5.5235619560570131f, 5.5545888516776376f, 31 | 5.5849625007211570f, 5.6147098441152083f, 5.6438561897747244f, 32 | 5.6724253419714961f, 5.7004397181410926f, 5.7279204545631996f, 33 | 5.7548875021634691f, 5.7813597135246599f, 5.8073549220576046f, 34 | 5.8328900141647422f, 5.8579809951275719f, 5.8826430493618416f, 35 | 5.9068905956085187f, 5.9307373375628867f, 5.9541963103868758f, 36 | 5.9772799234999168f, 6.0000000000000000f, 6.0223678130284544f, 37 | 6.0443941193584534f, 6.0660891904577721f, 6.0874628412503400f, 38 | 6.1085244567781700f, 6.1292830169449672f, 6.1497471195046822f, 39 | 6.1699250014423122f, 6.1898245588800176f, 6.2094533656289510f, 40 | 6.2288186904958804f, 6.2479275134435861f, 6.2667865406949019f, 41 | 6.2854022188622487f, 6.3037807481771031f, 6.3219280948873617f, 42 | 6.3398500028846252f, 6.3575520046180847f, 6.3750394313469254f, 43 | 6.3923174227787598f, 6.4093909361377026f, 6.4262647547020979f, 44 | 6.4429434958487288f, 6.4594316186372982f, 6.4757334309663976f, 45 | 6.4918530963296748f, 6.5077946401986964f, 6.5235619560570131f, 46 | 6.5391588111080319f, 6.5545888516776376f, 6.5698556083309478f, 47 | 6.5849625007211561f, 6.5999128421871278f, 6.6147098441152092f, 48 | 6.6293566200796095f, 6.6438561897747253f, 6.6582114827517955f, 49 | 6.6724253419714952f, 6.6865005271832185f, 6.7004397181410917f, 50 | 6.7142455176661224f, 6.7279204545631988f, 6.7414669864011465f, 51 | 6.7548875021634691f, 6.7681843247769260f, 6.7813597135246599f, 52 | 6.7944158663501062f, 6.8073549220576037f, 6.8201789624151887f, 53 | 6.8328900141647422f, 6.8454900509443757f, 6.8579809951275719f, 54 | 6.8703647195834048f, 6.8826430493618416f, 6.8948177633079437f, 55 | 6.9068905956085187f, 6.9188632372745955f, 6.9307373375628867f, 56 | 6.9425145053392399f, 6.9541963103868758f, 6.9657842846620879f, 57 | 6.9772799234999168f, 6.9886846867721664f, 7.0000000000000000f, 58 | 7.0112272554232540f, 7.0223678130284544f, 7.0334230015374501f, 59 | 7.0443941193584534f, 7.0552824355011898f, 7.0660891904577721f, 60 | 7.0768155970508317f, 7.0874628412503400f, 7.0980320829605272f, 61 | 7.1085244567781700f, 7.1189410727235076f, 7.1292830169449664f, 62 | 7.1395513523987937f, 7.1497471195046822f, 7.1598713367783891f, 63 | 7.1699250014423130f, 7.1799090900149345f, 7.1898245588800176f, 64 | 7.1996723448363644f, 7.2094533656289492f, 7.2191685204621621f, 65 | 7.2288186904958804f, 7.2384047393250794f, 7.2479275134435861f, 66 | 7.2573878426926521f, 7.2667865406949019f, 7.2761244052742384f, 67 | 7.2854022188622487f, 7.2946207488916270f, 7.3037807481771031f, 68 | 7.3128829552843557f, 7.3219280948873617f, 7.3309168781146177f, 69 | 7.3398500028846243f, 7.3487281542310781f, 7.3575520046180847f, 70 | 7.3663222142458151f, 7.3750394313469254f, 7.3837042924740528f, 71 | 7.3923174227787607f, 7.4008794362821844f, 7.4093909361377026f, 72 | 7.4178525148858991f, 7.4262647547020979f, 7.4346282276367255f, 73 | 7.4429434958487288f, 7.4512111118323299f, 7.4594316186372973f, 74 | 7.4676055500829976f, 7.4757334309663976f, 7.4838157772642564f, 75 | 7.4918530963296748f, 7.4998458870832057f, 7.5077946401986964f, 76 | 7.5156998382840436f, 7.5235619560570131f, 7.5313814605163119f, 77 | 7.5391588111080319f, 7.5468944598876373f, 7.5545888516776376f, 78 | 7.5622424242210728f, 7.5698556083309478f, 7.5774288280357487f, 79 | 7.5849625007211561f, 7.5924570372680806f, 7.5999128421871278f, 80 | 7.6073303137496113f, 7.6147098441152075f, 7.6220518194563764f, 81 | 7.6293566200796095f, 7.6366246205436488f, 7.6438561897747244f, 82 | 7.6510516911789290f, 7.6582114827517955f, 7.6653359171851765f, 83 | 7.6724253419714952f, 7.6794800995054464f, 7.6865005271832185f, 84 | 7.6934869574993252f, 7.7004397181410926f, 7.7073591320808825f, 85 | 7.7142455176661224f, 7.7210991887071856f, 7.7279204545631996f, 86 | 7.7347096202258392f, 7.7414669864011465f, 7.7481928495894596f, 87 | 7.7548875021634691f, 7.7615512324444795f, 7.7681843247769260f, 88 | 7.7747870596011737f, 7.7813597135246608f, 7.7879025593914317f, 89 | 7.7944158663501062f, 7.8008998999203047f, 7.8073549220576037f, 90 | 7.8137811912170374f, 7.8201789624151887f, 7.8265484872909159f, 91 | 7.8328900141647422f, 7.8392037880969445f, 7.8454900509443757f, 92 | 7.8517490414160571f, 7.8579809951275719f, 7.8641861446542798f, 93 | 7.8703647195834048f, 7.8765169465650002f, 7.8826430493618425f, 94 | 7.8887432488982601f, 7.8948177633079446f, 7.9008668079807496f, 95 | 7.9068905956085187f, 7.9128893362299619f, 7.9188632372745955f, 96 | 7.9248125036057813f, 7.9307373375628867f, 7.9366379390025719f, 97 | 7.9425145053392399f, 7.9483672315846778f, 7.9541963103868758f, 98 | 7.9600019320680806f, 7.9657842846620870f, 7.9715435539507720f, 99 | 7.9772799234999168f, 7.9829935746943104f, 7.9886846867721664f, 100 | 7.9943534368588578f 101 | }; 102 | 103 | #if defined(__cplusplus) || defined(c_plusplus) 104 | } /* extern "C" */ 105 | #endif 106 | -------------------------------------------------------------------------------- /c_src/enc/fast_log.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Utilities for fast computation of logarithms. */ 8 | 9 | #ifndef BROTLI_ENC_FAST_LOG_H_ 10 | #define BROTLI_ENC_FAST_LOG_H_ 11 | 12 | #include 13 | 14 | #include "../common/platform.h" 15 | #include 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) { 22 | #if defined(BROTLI_BSR32) 23 | return BROTLI_BSR32((uint32_t)n); 24 | #else 25 | uint32_t result = 0; 26 | while (n >>= 1) result++; 27 | return result; 28 | #endif 29 | } 30 | 31 | #define BROTLI_LOG2_TABLE_SIZE 256 32 | 33 | /* A lookup table for small values of log2(int) to be used in entropy 34 | computation. */ 35 | BROTLI_INTERNAL extern const double kBrotliLog2Table[BROTLI_LOG2_TABLE_SIZE]; 36 | 37 | /* Visual Studio 2012 and Android API levels < 18 do not have the log2() 38 | * function defined, so we use log() and a multiplication instead. */ 39 | #if !defined(BROTLI_HAVE_LOG2) 40 | #if ((defined(_MSC_VER) && _MSC_VER <= 1700) || \ 41 | (defined(__ANDROID_API__) && __ANDROID_API__ < 18)) 42 | #define BROTLI_HAVE_LOG2 0 43 | #else 44 | #define BROTLI_HAVE_LOG2 1 45 | #endif 46 | #endif 47 | 48 | #define LOG_2_INV 1.4426950408889634 49 | 50 | /* Faster logarithm for small integers, with the property of log2(0) == 0. */ 51 | static BROTLI_INLINE double FastLog2(size_t v) { 52 | if (v < BROTLI_LOG2_TABLE_SIZE) { 53 | return kBrotliLog2Table[v]; 54 | } 55 | #if !(BROTLI_HAVE_LOG2) 56 | return log((double)v) * LOG_2_INV; 57 | #else 58 | return log2((double)v); 59 | #endif 60 | } 61 | 62 | #if defined(__cplusplus) || defined(c_plusplus) 63 | } /* extern "C" */ 64 | #endif 65 | 66 | #endif /* BROTLI_ENC_FAST_LOG_H_ */ 67 | -------------------------------------------------------------------------------- /c_src/enc/find_match_length.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2010 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Function to find maximal matching prefixes of strings. */ 8 | 9 | #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_ 10 | #define BROTLI_ENC_FIND_MATCH_LENGTH_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /* Separate implementation for little-endian 64-bit targets, for speed. */ 20 | #if defined(BROTLI_TZCNT64) && BROTLI_64_BITS && BROTLI_LITTLE_ENDIAN 21 | static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1, 22 | const uint8_t* s2, 23 | size_t limit) { 24 | size_t matched = 0; 25 | size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */ 26 | while (BROTLI_PREDICT_TRUE(--limit2)) { 27 | if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) == 28 | BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) { 29 | s2 += 8; 30 | matched += 8; 31 | } else { 32 | uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^ 33 | BROTLI_UNALIGNED_LOAD64LE(s1 + matched); 34 | size_t matching_bits = (size_t)BROTLI_TZCNT64(x); 35 | matched += matching_bits >> 3; 36 | return matched; 37 | } 38 | } 39 | limit = (limit & 7) + 1; /* + 1 is for pre-decrement in while */ 40 | while (--limit) { 41 | if (BROTLI_PREDICT_TRUE(s1[matched] == *s2)) { 42 | ++s2; 43 | ++matched; 44 | } else { 45 | return matched; 46 | } 47 | } 48 | return matched; 49 | } 50 | #else 51 | static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1, 52 | const uint8_t* s2, 53 | size_t limit) { 54 | size_t matched = 0; 55 | const uint8_t* s2_limit = s2 + limit; 56 | const uint8_t* s2_ptr = s2; 57 | /* Find out how long the match is. We loop over the data 32 bits at a 58 | time until we find a 32-bit block that doesn't match; then we find 59 | the first non-matching bit and use that to calculate the total 60 | length of the match. */ 61 | while (s2_ptr <= s2_limit - 4 && 62 | BrotliUnalignedRead32(s2_ptr) == 63 | BrotliUnalignedRead32(s1 + matched)) { 64 | s2_ptr += 4; 65 | matched += 4; 66 | } 67 | while ((s2_ptr < s2_limit) && (s1[matched] == *s2_ptr)) { 68 | ++s2_ptr; 69 | ++matched; 70 | } 71 | return matched; 72 | } 73 | #endif 74 | 75 | #if defined(__cplusplus) || defined(c_plusplus) 76 | } /* extern "C" */ 77 | #endif 78 | 79 | #endif /* BROTLI_ENC_FIND_MATCH_LENGTH_H_ */ 80 | -------------------------------------------------------------------------------- /c_src/enc/hash_composite_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2018 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: FN, HASHER_A, HASHER_B */ 9 | 10 | /* Composite hasher: This hasher allows to combine two other hashers, HASHER_A 11 | and HASHER_B. */ 12 | 13 | #define HashComposite HASHER() 14 | 15 | #define FN_A(X) EXPAND_CAT(X, HASHER_A) 16 | #define FN_B(X) EXPAND_CAT(X, HASHER_B) 17 | 18 | static BROTLI_INLINE size_t FN(HashTypeLength)(void) { 19 | size_t a = FN_A(HashTypeLength)(); 20 | size_t b = FN_B(HashTypeLength)(); 21 | return a > b ? a : b; 22 | } 23 | 24 | static BROTLI_INLINE size_t FN(StoreLookahead)(void) { 25 | size_t a = FN_A(StoreLookahead)(); 26 | size_t b = FN_B(StoreLookahead)(); 27 | return a > b ? a : b; 28 | } 29 | 30 | typedef struct HashComposite { 31 | HASHER_A ha; 32 | HASHER_B hb; 33 | HasherCommon hb_common; 34 | 35 | /* Shortcuts. */ 36 | void* extra; 37 | HasherCommon* common; 38 | 39 | BROTLI_BOOL fresh; 40 | const BrotliEncoderParams* params; 41 | } HashComposite; 42 | 43 | static void FN(Initialize)(HasherCommon* common, 44 | HashComposite* BROTLI_RESTRICT self, const BrotliEncoderParams* params) { 45 | self->common = common; 46 | self->extra = common->extra; 47 | 48 | self->hb_common = *self->common; 49 | self->fresh = BROTLI_TRUE; 50 | self->params = params; 51 | /* TODO: Initialize of the hashers is defered to Prepare (and params 52 | remembered here) because we don't get the one_shot and input_size params 53 | here that are needed to know the memory size of them. Instead provide 54 | those params to all hashers FN(Initialize) */ 55 | } 56 | 57 | static void FN(Prepare)( 58 | HashComposite* BROTLI_RESTRICT self, BROTLI_BOOL one_shot, 59 | size_t input_size, const uint8_t* BROTLI_RESTRICT data) { 60 | if (self->fresh) { 61 | self->fresh = BROTLI_FALSE; 62 | self->hb_common.extra = (uint8_t*)self->extra + 63 | FN_A(HashMemAllocInBytes)(self->params, one_shot, input_size); 64 | 65 | FN_A(Initialize)(self->common, &self->ha, self->params); 66 | FN_B(Initialize)(&self->hb_common, &self->hb, self->params); 67 | } 68 | FN_A(Prepare)(&self->ha, one_shot, input_size, data); 69 | FN_B(Prepare)(&self->hb, one_shot, input_size, data); 70 | } 71 | 72 | static BROTLI_INLINE size_t FN(HashMemAllocInBytes)( 73 | const BrotliEncoderParams* params, BROTLI_BOOL one_shot, 74 | size_t input_size) { 75 | return FN_A(HashMemAllocInBytes)(params, one_shot, input_size) + 76 | FN_B(HashMemAllocInBytes)(params, one_shot, input_size); 77 | } 78 | 79 | static BROTLI_INLINE void FN(Store)(HashComposite* BROTLI_RESTRICT self, 80 | const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) { 81 | FN_A(Store)(&self->ha, data, mask, ix); 82 | FN_B(Store)(&self->hb, data, mask, ix); 83 | } 84 | 85 | static BROTLI_INLINE void FN(StoreRange)( 86 | HashComposite* BROTLI_RESTRICT self, const uint8_t* BROTLI_RESTRICT data, 87 | const size_t mask, const size_t ix_start, 88 | const size_t ix_end) { 89 | FN_A(StoreRange)(&self->ha, data, mask, ix_start, ix_end); 90 | FN_B(StoreRange)(&self->hb, data, mask, ix_start, ix_end); 91 | } 92 | 93 | static BROTLI_INLINE void FN(StitchToPreviousBlock)( 94 | HashComposite* BROTLI_RESTRICT self, 95 | size_t num_bytes, size_t position, const uint8_t* ringbuffer, 96 | size_t ring_buffer_mask) { 97 | FN_A(StitchToPreviousBlock)(&self->ha, num_bytes, position, 98 | ringbuffer, ring_buffer_mask); 99 | FN_B(StitchToPreviousBlock)(&self->hb, num_bytes, position, 100 | ringbuffer, ring_buffer_mask); 101 | } 102 | 103 | static BROTLI_INLINE void FN(PrepareDistanceCache)( 104 | HashComposite* BROTLI_RESTRICT self, int* BROTLI_RESTRICT distance_cache) { 105 | FN_A(PrepareDistanceCache)(&self->ha, distance_cache); 106 | FN_B(PrepareDistanceCache)(&self->hb, distance_cache); 107 | } 108 | 109 | static BROTLI_INLINE void FN(FindLongestMatch)( 110 | HashComposite* BROTLI_RESTRICT self, 111 | const BrotliEncoderDictionary* dictionary, 112 | const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, 113 | const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, 114 | const size_t max_length, const size_t max_backward, 115 | const size_t dictionary_distance, const size_t max_distance, 116 | HasherSearchResult* BROTLI_RESTRICT out) { 117 | FN_A(FindLongestMatch)(&self->ha, dictionary, data, ring_buffer_mask, 118 | distance_cache, cur_ix, max_length, max_backward, dictionary_distance, 119 | max_distance, out); 120 | FN_B(FindLongestMatch)(&self->hb, dictionary, data, ring_buffer_mask, 121 | distance_cache, cur_ix, max_length, max_backward, dictionary_distance, 122 | max_distance, out); 123 | } 124 | 125 | #undef HashComposite 126 | -------------------------------------------------------------------------------- /c_src/enc/hash_rolling_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2018 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: FN, JUMP, NUMBUCKETS, MASK, CHUNKLEN */ 9 | /* NUMBUCKETS / (MASK + 1) = probability of storing and using hash code. */ 10 | /* JUMP = skip bytes for speedup */ 11 | 12 | /* Rolling hash for long distance long string matches. Stores one position 13 | per bucket, bucket key is computed over a long region. */ 14 | 15 | #define HashRolling HASHER() 16 | 17 | static const uint32_t FN(kRollingHashMul32) = 69069; 18 | static const uint32_t FN(kInvalidPos) = 0xffffffff; 19 | 20 | /* This hasher uses a longer forward length, but returning a higher value here 21 | will hurt compression by the main hasher when combined with a composite 22 | hasher. The hasher tests for forward itself instead. */ 23 | static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; } 24 | static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; } 25 | 26 | /* Computes a code from a single byte. A lookup table of 256 values could be 27 | used, but simply adding 1 works about as good. */ 28 | static uint32_t FN(HashByte)(uint8_t byte) { 29 | return (uint32_t)byte + 1u; 30 | } 31 | 32 | static uint32_t FN(HashRollingFunctionInitial)(uint32_t state, uint8_t add, 33 | uint32_t factor) { 34 | return (uint32_t)(factor * state + FN(HashByte)(add)); 35 | } 36 | 37 | static uint32_t FN(HashRollingFunction)(uint32_t state, uint8_t add, 38 | uint8_t rem, uint32_t factor, 39 | uint32_t factor_remove) { 40 | return (uint32_t)(factor * state + 41 | FN(HashByte)(add) - factor_remove * FN(HashByte)(rem)); 42 | } 43 | 44 | typedef struct HashRolling { 45 | uint32_t state; 46 | uint32_t* table; 47 | size_t next_ix; 48 | 49 | uint32_t chunk_len; 50 | uint32_t factor; 51 | uint32_t factor_remove; 52 | } HashRolling; 53 | 54 | static void FN(Initialize)( 55 | HasherCommon* common, HashRolling* BROTLI_RESTRICT self, 56 | const BrotliEncoderParams* params) { 57 | size_t i; 58 | self->state = 0; 59 | self->next_ix = 0; 60 | 61 | self->factor = FN(kRollingHashMul32); 62 | 63 | /* Compute the factor of the oldest byte to remove: factor**steps modulo 64 | 0xffffffff (the multiplications rely on 32-bit overflow) */ 65 | self->factor_remove = 1; 66 | for (i = 0; i < CHUNKLEN; i += JUMP) { 67 | self->factor_remove *= self->factor; 68 | } 69 | 70 | self->table = (uint32_t*)common->extra; 71 | for (i = 0; i < NUMBUCKETS; i++) { 72 | self->table[i] = FN(kInvalidPos); 73 | } 74 | 75 | BROTLI_UNUSED(params); 76 | } 77 | 78 | static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot, 79 | size_t input_size, const uint8_t* BROTLI_RESTRICT data) { 80 | size_t i; 81 | /* Too small size, cannot use this hasher. */ 82 | if (input_size < CHUNKLEN) return; 83 | self->state = 0; 84 | for (i = 0; i < CHUNKLEN; i += JUMP) { 85 | self->state = FN(HashRollingFunctionInitial)( 86 | self->state, data[i], self->factor); 87 | } 88 | BROTLI_UNUSED(one_shot); 89 | } 90 | 91 | static BROTLI_INLINE size_t FN(HashMemAllocInBytes)( 92 | const BrotliEncoderParams* params, BROTLI_BOOL one_shot, 93 | size_t input_size) { 94 | return NUMBUCKETS * sizeof(uint32_t); 95 | BROTLI_UNUSED(params); 96 | BROTLI_UNUSED(one_shot); 97 | BROTLI_UNUSED(input_size); 98 | } 99 | 100 | static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self, 101 | const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) { 102 | BROTLI_UNUSED(self); 103 | BROTLI_UNUSED(data); 104 | BROTLI_UNUSED(mask); 105 | BROTLI_UNUSED(ix); 106 | } 107 | 108 | static BROTLI_INLINE void FN(StoreRange)(HashRolling* BROTLI_RESTRICT self, 109 | const uint8_t* BROTLI_RESTRICT data, const size_t mask, 110 | const size_t ix_start, const size_t ix_end) { 111 | BROTLI_UNUSED(self); 112 | BROTLI_UNUSED(data); 113 | BROTLI_UNUSED(mask); 114 | BROTLI_UNUSED(ix_start); 115 | BROTLI_UNUSED(ix_end); 116 | } 117 | 118 | static BROTLI_INLINE void FN(StitchToPreviousBlock)( 119 | HashRolling* BROTLI_RESTRICT self, 120 | size_t num_bytes, size_t position, const uint8_t* ringbuffer, 121 | size_t ring_buffer_mask) { 122 | /* In this case we must re-initialize the hasher from scratch from the 123 | current position. */ 124 | size_t position_masked; 125 | size_t available = num_bytes; 126 | if ((position & (JUMP - 1)) != 0) { 127 | size_t diff = JUMP - (position & (JUMP - 1)); 128 | available = (diff > available) ? 0 : (available - diff); 129 | position += diff; 130 | } 131 | position_masked = position & ring_buffer_mask; 132 | /* wrapping around ringbuffer not handled. */ 133 | if (available > ring_buffer_mask - position_masked) { 134 | available = ring_buffer_mask - position_masked; 135 | } 136 | 137 | FN(Prepare)(self, BROTLI_FALSE, available, 138 | ringbuffer + (position & ring_buffer_mask)); 139 | self->next_ix = position; 140 | BROTLI_UNUSED(num_bytes); 141 | } 142 | 143 | static BROTLI_INLINE void FN(PrepareDistanceCache)( 144 | HashRolling* BROTLI_RESTRICT self, 145 | int* BROTLI_RESTRICT distance_cache) { 146 | BROTLI_UNUSED(self); 147 | BROTLI_UNUSED(distance_cache); 148 | } 149 | 150 | static BROTLI_INLINE void FN(FindLongestMatch)( 151 | HashRolling* BROTLI_RESTRICT self, 152 | const BrotliEncoderDictionary* dictionary, 153 | const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, 154 | const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, 155 | const size_t max_length, const size_t max_backward, 156 | const size_t dictionary_distance, const size_t max_distance, 157 | HasherSearchResult* BROTLI_RESTRICT out) { 158 | const size_t cur_ix_masked = cur_ix & ring_buffer_mask; 159 | size_t pos; 160 | 161 | if ((cur_ix & (JUMP - 1)) != 0) return; 162 | 163 | /* Not enough lookahead */ 164 | if (max_length < CHUNKLEN) return; 165 | 166 | for (pos = self->next_ix; pos <= cur_ix; pos += JUMP) { 167 | uint32_t code = self->state & MASK; 168 | 169 | uint8_t rem = data[pos & ring_buffer_mask]; 170 | uint8_t add = data[(pos + CHUNKLEN) & ring_buffer_mask]; 171 | size_t found_ix = FN(kInvalidPos); 172 | 173 | self->state = FN(HashRollingFunction)( 174 | self->state, add, rem, self->factor, self->factor_remove); 175 | 176 | if (code < NUMBUCKETS) { 177 | found_ix = self->table[code]; 178 | self->table[code] = (uint32_t)pos; 179 | if (pos == cur_ix && found_ix != FN(kInvalidPos)) { 180 | /* The cast to 32-bit makes backward distances up to 4GB work even 181 | if cur_ix is above 4GB, despite using 32-bit values in the table. */ 182 | size_t backward = (uint32_t)(cur_ix - found_ix); 183 | if (backward <= max_backward) { 184 | const size_t found_ix_masked = found_ix & ring_buffer_mask; 185 | const size_t len = FindMatchLengthWithLimit(&data[found_ix_masked], 186 | &data[cur_ix_masked], 187 | max_length); 188 | if (len >= 4 && len > out->len) { 189 | score_t score = BackwardReferenceScore(len, backward); 190 | if (score > out->score) { 191 | out->len = len; 192 | out->distance = backward; 193 | out->score = score; 194 | out->len_code_delta = 0; 195 | } 196 | } 197 | } 198 | } 199 | } 200 | } 201 | 202 | self->next_ix = cur_ix + JUMP; 203 | 204 | /* NOTE: this hasher does not search in the dictionary. It is used as 205 | backup-hasher, the main hasher already searches in it. */ 206 | BROTLI_UNUSED(dictionary); 207 | BROTLI_UNUSED(distance_cache); 208 | BROTLI_UNUSED(dictionary_distance); 209 | BROTLI_UNUSED(max_distance); 210 | } 211 | 212 | #undef HashRolling 213 | -------------------------------------------------------------------------------- /c_src/enc/histogram.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Build per-context histograms of literals, commands and distance codes. */ 8 | 9 | #include "./histogram.h" 10 | 11 | #include "../common/context.h" 12 | #include "./block_splitter.h" 13 | #include "./command.h" 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | typedef struct BlockSplitIterator { 20 | const BlockSplit* split_; /* Not owned. */ 21 | size_t idx_; 22 | size_t type_; 23 | size_t length_; 24 | } BlockSplitIterator; 25 | 26 | static void InitBlockSplitIterator(BlockSplitIterator* self, 27 | const BlockSplit* split) { 28 | self->split_ = split; 29 | self->idx_ = 0; 30 | self->type_ = 0; 31 | self->length_ = split->lengths ? split->lengths[0] : 0; 32 | } 33 | 34 | static void BlockSplitIteratorNext(BlockSplitIterator* self) { 35 | if (self->length_ == 0) { 36 | ++self->idx_; 37 | self->type_ = self->split_->types[self->idx_]; 38 | self->length_ = self->split_->lengths[self->idx_]; 39 | } 40 | --self->length_; 41 | } 42 | 43 | void BrotliBuildHistogramsWithContext( 44 | const Command* cmds, const size_t num_commands, 45 | const BlockSplit* literal_split, const BlockSplit* insert_and_copy_split, 46 | const BlockSplit* dist_split, const uint8_t* ringbuffer, size_t start_pos, 47 | size_t mask, uint8_t prev_byte, uint8_t prev_byte2, 48 | const ContextType* context_modes, HistogramLiteral* literal_histograms, 49 | HistogramCommand* insert_and_copy_histograms, 50 | HistogramDistance* copy_dist_histograms) { 51 | size_t pos = start_pos; 52 | BlockSplitIterator literal_it; 53 | BlockSplitIterator insert_and_copy_it; 54 | BlockSplitIterator dist_it; 55 | size_t i; 56 | 57 | InitBlockSplitIterator(&literal_it, literal_split); 58 | InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split); 59 | InitBlockSplitIterator(&dist_it, dist_split); 60 | for (i = 0; i < num_commands; ++i) { 61 | const Command* cmd = &cmds[i]; 62 | size_t j; 63 | BlockSplitIteratorNext(&insert_and_copy_it); 64 | HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], 65 | cmd->cmd_prefix_); 66 | /* TODO: unwrap iterator blocks. */ 67 | for (j = cmd->insert_len_; j != 0; --j) { 68 | size_t context; 69 | BlockSplitIteratorNext(&literal_it); 70 | context = literal_it.type_; 71 | if (context_modes) { 72 | ContextLut lut = BROTLI_CONTEXT_LUT(context_modes[context]); 73 | context = (context << BROTLI_LITERAL_CONTEXT_BITS) + 74 | BROTLI_CONTEXT(prev_byte, prev_byte2, lut); 75 | } 76 | HistogramAddLiteral(&literal_histograms[context], 77 | ringbuffer[pos & mask]); 78 | prev_byte2 = prev_byte; 79 | prev_byte = ringbuffer[pos & mask]; 80 | ++pos; 81 | } 82 | pos += CommandCopyLen(cmd); 83 | if (CommandCopyLen(cmd)) { 84 | prev_byte2 = ringbuffer[(pos - 2) & mask]; 85 | prev_byte = ringbuffer[(pos - 1) & mask]; 86 | if (cmd->cmd_prefix_ >= 128) { 87 | size_t context; 88 | BlockSplitIteratorNext(&dist_it); 89 | context = (dist_it.type_ << BROTLI_DISTANCE_CONTEXT_BITS) + 90 | CommandDistanceContext(cmd); 91 | HistogramAddDistance(©_dist_histograms[context], 92 | cmd->dist_prefix_ & 0x3FF); 93 | } 94 | } 95 | } 96 | } 97 | 98 | #if defined(__cplusplus) || defined(c_plusplus) 99 | } /* extern "C" */ 100 | #endif 101 | -------------------------------------------------------------------------------- /c_src/enc/histogram.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Models the histograms of literals, commands and distance codes. */ 8 | 9 | #ifndef BROTLI_ENC_HISTOGRAM_H_ 10 | #define BROTLI_ENC_HISTOGRAM_H_ 11 | 12 | #include /* memset */ 13 | 14 | #include "../common/constants.h" 15 | #include "../common/context.h" 16 | #include "../common/platform.h" 17 | #include 18 | #include "./block_splitter.h" 19 | #include "./command.h" 20 | 21 | #if defined(__cplusplus) || defined(c_plusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* The distance symbols effectively used by "Large Window Brotli" (32-bit). */ 26 | #define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544 27 | 28 | #define FN(X) X ## Literal 29 | #define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS 30 | #define DataType uint8_t 31 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 32 | #undef DataType 33 | #undef DATA_SIZE 34 | #undef FN 35 | 36 | #define FN(X) X ## Command 37 | #define DataType uint16_t 38 | #define DATA_SIZE BROTLI_NUM_COMMAND_SYMBOLS 39 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 40 | #undef DATA_SIZE 41 | #undef FN 42 | 43 | #define FN(X) X ## Distance 44 | #define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 45 | #include "./histogram_inc.h" /* NOLINT(build/include) */ 46 | #undef DataType 47 | #undef DATA_SIZE 48 | #undef FN 49 | 50 | BROTLI_INTERNAL void BrotliBuildHistogramsWithContext( 51 | const Command* cmds, const size_t num_commands, 52 | const BlockSplit* literal_split, const BlockSplit* insert_and_copy_split, 53 | const BlockSplit* dist_split, const uint8_t* ringbuffer, size_t pos, 54 | size_t mask, uint8_t prev_byte, uint8_t prev_byte2, 55 | const ContextType* context_modes, HistogramLiteral* literal_histograms, 56 | HistogramCommand* insert_and_copy_histograms, 57 | HistogramDistance* copy_dist_histograms); 58 | 59 | #if defined(__cplusplus) || defined(c_plusplus) 60 | } /* extern "C" */ 61 | #endif 62 | 63 | #endif /* BROTLI_ENC_HISTOGRAM_H_ */ 64 | -------------------------------------------------------------------------------- /c_src/enc/histogram_inc.h: -------------------------------------------------------------------------------- 1 | /* NOLINT(build/header_guard) */ 2 | /* Copyright 2013 Google Inc. All Rights Reserved. 3 | 4 | Distributed under MIT license. 5 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 6 | */ 7 | 8 | /* template parameters: Histogram, DATA_SIZE, DataType */ 9 | 10 | /* A simple container for histograms of data in blocks. */ 11 | 12 | typedef struct FN(Histogram) { 13 | uint32_t data_[DATA_SIZE]; 14 | size_t total_count_; 15 | double bit_cost_; 16 | } FN(Histogram); 17 | 18 | static BROTLI_INLINE void FN(HistogramClear)(FN(Histogram)* self) { 19 | memset(self->data_, 0, sizeof(self->data_)); 20 | self->total_count_ = 0; 21 | self->bit_cost_ = HUGE_VAL; 22 | } 23 | 24 | static BROTLI_INLINE void FN(ClearHistograms)( 25 | FN(Histogram)* array, size_t length) { 26 | size_t i; 27 | for (i = 0; i < length; ++i) FN(HistogramClear)(array + i); 28 | } 29 | 30 | static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) { 31 | ++self->data_[val]; 32 | ++self->total_count_; 33 | } 34 | 35 | static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self, 36 | const DataType* p, size_t n) { 37 | self->total_count_ += n; 38 | n += 1; 39 | while (--n) ++self->data_[*p++]; 40 | } 41 | 42 | static BROTLI_INLINE void FN(HistogramAddHistogram)(FN(Histogram)* self, 43 | const FN(Histogram)* v) { 44 | size_t i; 45 | self->total_count_ += v->total_count_; 46 | for (i = 0; i < DATA_SIZE; ++i) { 47 | self->data_[i] += v->data_[i]; 48 | } 49 | } 50 | 51 | static BROTLI_INLINE size_t FN(HistogramDataSize)(void) { return DATA_SIZE; } 52 | -------------------------------------------------------------------------------- /c_src/enc/literal_cost.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Literal cost model to allow backward reference replacement to be efficient. 8 | */ 9 | 10 | #include "./literal_cost.h" 11 | 12 | #include "../common/platform.h" 13 | #include 14 | #include "./fast_log.h" 15 | #include "./utf8_util.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | static size_t UTF8Position(size_t last, size_t c, size_t clamp) { 22 | if (c < 128) { 23 | return 0; /* Next one is the 'Byte 1' again. */ 24 | } else if (c >= 192) { /* Next one is the 'Byte 2' of utf-8 encoding. */ 25 | return BROTLI_MIN(size_t, 1, clamp); 26 | } else { 27 | /* Let's decide over the last byte if this ends the sequence. */ 28 | if (last < 0xE0) { 29 | return 0; /* Completed two or three byte coding. */ 30 | } else { /* Next one is the 'Byte 3' of utf-8 encoding. */ 31 | return BROTLI_MIN(size_t, 2, clamp); 32 | } 33 | } 34 | } 35 | 36 | static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, 37 | const uint8_t* data) { 38 | size_t counts[3] = { 0 }; 39 | size_t max_utf8 = 1; /* should be 2, but 1 compresses better. */ 40 | size_t last_c = 0; 41 | size_t i; 42 | for (i = 0; i < len; ++i) { 43 | size_t c = data[(pos + i) & mask]; 44 | ++counts[UTF8Position(last_c, c, 2)]; 45 | last_c = c; 46 | } 47 | if (counts[2] < 500) { 48 | max_utf8 = 1; 49 | } 50 | if (counts[1] + counts[2] < 25) { 51 | max_utf8 = 0; 52 | } 53 | return max_utf8; 54 | } 55 | 56 | static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, 57 | const uint8_t* data, float* cost) { 58 | /* max_utf8 is 0 (normal ASCII single byte modeling), 59 | 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */ 60 | const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); 61 | size_t histogram[3][256] = { { 0 } }; 62 | size_t window_half = 495; 63 | size_t in_window = BROTLI_MIN(size_t, window_half, len); 64 | size_t in_window_utf8[3] = { 0 }; 65 | 66 | size_t i; 67 | { /* Bootstrap histograms. */ 68 | size_t last_c = 0; 69 | size_t utf8_pos = 0; 70 | for (i = 0; i < in_window; ++i) { 71 | size_t c = data[(pos + i) & mask]; 72 | ++histogram[utf8_pos][c]; 73 | ++in_window_utf8[utf8_pos]; 74 | utf8_pos = UTF8Position(last_c, c, max_utf8); 75 | last_c = c; 76 | } 77 | } 78 | 79 | /* Compute bit costs with sliding window. */ 80 | for (i = 0; i < len; ++i) { 81 | if (i >= window_half) { 82 | /* Remove a byte in the past. */ 83 | size_t c = 84 | i < window_half + 1 ? 0 : data[(pos + i - window_half - 1) & mask]; 85 | size_t last_c = 86 | i < window_half + 2 ? 0 : data[(pos + i - window_half - 2) & mask]; 87 | size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); 88 | --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; 89 | --in_window_utf8[utf8_pos2]; 90 | } 91 | if (i + window_half < len) { 92 | /* Add a byte in the future. */ 93 | size_t c = data[(pos + i + window_half - 1) & mask]; 94 | size_t last_c = data[(pos + i + window_half - 2) & mask]; 95 | size_t utf8_pos2 = UTF8Position(last_c, c, max_utf8); 96 | ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; 97 | ++in_window_utf8[utf8_pos2]; 98 | } 99 | { 100 | size_t c = i < 1 ? 0 : data[(pos + i - 1) & mask]; 101 | size_t last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; 102 | size_t utf8_pos = UTF8Position(last_c, c, max_utf8); 103 | size_t masked_pos = (pos + i) & mask; 104 | size_t histo = histogram[utf8_pos][data[masked_pos]]; 105 | double lit_cost; 106 | if (histo == 0) { 107 | histo = 1; 108 | } 109 | lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); 110 | lit_cost += 0.02905; 111 | if (lit_cost < 1.0) { 112 | lit_cost *= 0.5; 113 | lit_cost += 0.5; 114 | } 115 | /* Make the first bytes more expensive -- seems to help, not sure why. 116 | Perhaps because the entropy source is changing its properties 117 | rapidly in the beginning of the file, perhaps because the beginning 118 | of the data is a statistical "anomaly". */ 119 | if (i < 2000) { 120 | lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35); 121 | } 122 | cost[i] = (float)lit_cost; 123 | } 124 | } 125 | } 126 | 127 | void BrotliEstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, 128 | const uint8_t* data, float* cost) { 129 | if (BrotliIsMostlyUTF8(data, pos, mask, len, kMinUTF8Ratio)) { 130 | EstimateBitCostsForLiteralsUTF8(pos, len, mask, data, cost); 131 | return; 132 | } else { 133 | size_t histogram[256] = { 0 }; 134 | size_t window_half = 2000; 135 | size_t in_window = BROTLI_MIN(size_t, window_half, len); 136 | 137 | /* Bootstrap histogram. */ 138 | size_t i; 139 | for (i = 0; i < in_window; ++i) { 140 | ++histogram[data[(pos + i) & mask]]; 141 | } 142 | 143 | /* Compute bit costs with sliding window. */ 144 | for (i = 0; i < len; ++i) { 145 | size_t histo; 146 | if (i >= window_half) { 147 | /* Remove a byte in the past. */ 148 | --histogram[data[(pos + i - window_half) & mask]]; 149 | --in_window; 150 | } 151 | if (i + window_half < len) { 152 | /* Add a byte in the future. */ 153 | ++histogram[data[(pos + i + window_half) & mask]]; 154 | ++in_window; 155 | } 156 | histo = histogram[data[(pos + i) & mask]]; 157 | if (histo == 0) { 158 | histo = 1; 159 | } 160 | { 161 | double lit_cost = FastLog2(in_window) - FastLog2(histo); 162 | lit_cost += 0.029; 163 | if (lit_cost < 1.0) { 164 | lit_cost *= 0.5; 165 | lit_cost += 0.5; 166 | } 167 | cost[i] = (float)lit_cost; 168 | } 169 | } 170 | } 171 | } 172 | 173 | #if defined(__cplusplus) || defined(c_plusplus) 174 | } /* extern "C" */ 175 | #endif 176 | -------------------------------------------------------------------------------- /c_src/enc/literal_cost.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Literal cost model to allow backward reference replacement to be efficient. 8 | */ 9 | 10 | #ifndef BROTLI_ENC_LITERAL_COST_H_ 11 | #define BROTLI_ENC_LITERAL_COST_H_ 12 | 13 | #include "../common/platform.h" 14 | #include 15 | 16 | #if defined(__cplusplus) || defined(c_plusplus) 17 | extern "C" { 18 | #endif 19 | 20 | /* Estimates how many bits the literals in the interval [pos, pos + len) in the 21 | ring-buffer (data, mask) will take entropy coded and writes these estimates 22 | to the cost[0..len) array. */ 23 | BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals( 24 | size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost); 25 | 26 | #if defined(__cplusplus) || defined(c_plusplus) 27 | } /* extern "C" */ 28 | #endif 29 | 30 | #endif /* BROTLI_ENC_LITERAL_COST_H_ */ 31 | -------------------------------------------------------------------------------- /c_src/enc/memory.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Algorithms for distributing the literals and commands of a metablock between 8 | block types and contexts. */ 9 | 10 | #include "./memory.h" 11 | 12 | #include /* exit, free, malloc */ 13 | #include /* memcpy */ 14 | 15 | #include "../common/platform.h" 16 | #include 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | #define MAX_PERM_ALLOCATED 128 23 | #define MAX_NEW_ALLOCATED 64 24 | #define MAX_NEW_FREED 64 25 | 26 | #define PERM_ALLOCATED_OFFSET 0 27 | #define NEW_ALLOCATED_OFFSET MAX_PERM_ALLOCATED 28 | #define NEW_FREED_OFFSET (MAX_PERM_ALLOCATED + MAX_NEW_ALLOCATED) 29 | 30 | void BrotliInitMemoryManager( 31 | MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func, 32 | void* opaque) { 33 | if (!alloc_func) { 34 | m->alloc_func = BrotliDefaultAllocFunc; 35 | m->free_func = BrotliDefaultFreeFunc; 36 | m->opaque = 0; 37 | } else { 38 | m->alloc_func = alloc_func; 39 | m->free_func = free_func; 40 | m->opaque = opaque; 41 | } 42 | #if !defined(BROTLI_ENCODER_EXIT_ON_OOM) 43 | m->is_oom = BROTLI_FALSE; 44 | m->perm_allocated = 0; 45 | m->new_allocated = 0; 46 | m->new_freed = 0; 47 | #endif /* BROTLI_ENCODER_EXIT_ON_OOM */ 48 | } 49 | 50 | #if defined(BROTLI_ENCODER_EXIT_ON_OOM) 51 | 52 | void* BrotliAllocate(MemoryManager* m, size_t n) { 53 | void* result = m->alloc_func(m->opaque, n); 54 | if (!result) exit(EXIT_FAILURE); 55 | return result; 56 | } 57 | 58 | void BrotliFree(MemoryManager* m, void* p) { 59 | m->free_func(m->opaque, p); 60 | } 61 | 62 | void BrotliWipeOutMemoryManager(MemoryManager* m) { 63 | BROTLI_UNUSED(m); 64 | } 65 | 66 | #else /* BROTLI_ENCODER_EXIT_ON_OOM */ 67 | 68 | static void SortPointers(void** items, const size_t n) { 69 | /* Shell sort. */ 70 | static const size_t gaps[] = {23, 10, 4, 1}; 71 | int g = 0; 72 | for (; g < 4; ++g) { 73 | size_t gap = gaps[g]; 74 | size_t i; 75 | for (i = gap; i < n; ++i) { 76 | size_t j = i; 77 | void* tmp = items[i]; 78 | for (; j >= gap && tmp < items[j - gap]; j -= gap) { 79 | items[j] = items[j - gap]; 80 | } 81 | items[j] = tmp; 82 | } 83 | } 84 | } 85 | 86 | static size_t Annihilate(void** a, size_t a_len, void** b, size_t b_len) { 87 | size_t a_read_index = 0; 88 | size_t b_read_index = 0; 89 | size_t a_write_index = 0; 90 | size_t b_write_index = 0; 91 | size_t annihilated = 0; 92 | while (a_read_index < a_len && b_read_index < b_len) { 93 | if (a[a_read_index] == b[b_read_index]) { 94 | a_read_index++; 95 | b_read_index++; 96 | annihilated++; 97 | } else if (a[a_read_index] < b[b_read_index]) { 98 | a[a_write_index++] = a[a_read_index++]; 99 | } else { 100 | b[b_write_index++] = b[b_read_index++]; 101 | } 102 | } 103 | while (a_read_index < a_len) a[a_write_index++] = a[a_read_index++]; 104 | while (b_read_index < b_len) b[b_write_index++] = b[b_read_index++]; 105 | return annihilated; 106 | } 107 | 108 | static void CollectGarbagePointers(MemoryManager* m) { 109 | size_t annihilated; 110 | SortPointers(m->pointers + NEW_ALLOCATED_OFFSET, m->new_allocated); 111 | SortPointers(m->pointers + NEW_FREED_OFFSET, m->new_freed); 112 | annihilated = Annihilate( 113 | m->pointers + NEW_ALLOCATED_OFFSET, m->new_allocated, 114 | m->pointers + NEW_FREED_OFFSET, m->new_freed); 115 | m->new_allocated -= annihilated; 116 | m->new_freed -= annihilated; 117 | 118 | if (m->new_freed != 0) { 119 | annihilated = Annihilate( 120 | m->pointers + PERM_ALLOCATED_OFFSET, m->perm_allocated, 121 | m->pointers + NEW_FREED_OFFSET, m->new_freed); 122 | m->perm_allocated -= annihilated; 123 | m->new_freed -= annihilated; 124 | BROTLI_DCHECK(m->new_freed == 0); 125 | } 126 | 127 | if (m->new_allocated != 0) { 128 | BROTLI_DCHECK(m->perm_allocated + m->new_allocated <= MAX_PERM_ALLOCATED); 129 | memcpy(m->pointers + PERM_ALLOCATED_OFFSET + m->perm_allocated, 130 | m->pointers + NEW_ALLOCATED_OFFSET, 131 | sizeof(void*) * m->new_allocated); 132 | m->perm_allocated += m->new_allocated; 133 | m->new_allocated = 0; 134 | SortPointers(m->pointers + PERM_ALLOCATED_OFFSET, m->perm_allocated); 135 | } 136 | } 137 | 138 | void* BrotliAllocate(MemoryManager* m, size_t n) { 139 | void* result = m->alloc_func(m->opaque, n); 140 | if (!result) { 141 | m->is_oom = BROTLI_TRUE; 142 | return NULL; 143 | } 144 | if (m->new_allocated == MAX_NEW_ALLOCATED) CollectGarbagePointers(m); 145 | m->pointers[NEW_ALLOCATED_OFFSET + (m->new_allocated++)] = result; 146 | return result; 147 | } 148 | 149 | void BrotliFree(MemoryManager* m, void* p) { 150 | if (!p) return; 151 | m->free_func(m->opaque, p); 152 | if (m->new_freed == MAX_NEW_FREED) CollectGarbagePointers(m); 153 | m->pointers[NEW_FREED_OFFSET + (m->new_freed++)] = p; 154 | } 155 | 156 | void BrotliWipeOutMemoryManager(MemoryManager* m) { 157 | size_t i; 158 | CollectGarbagePointers(m); 159 | /* Now all unfreed pointers are in perm-allocated list. */ 160 | for (i = 0; i < m->perm_allocated; ++i) { 161 | m->free_func(m->opaque, m->pointers[PERM_ALLOCATED_OFFSET + i]); 162 | } 163 | m->perm_allocated = 0; 164 | } 165 | 166 | #endif /* BROTLI_ENCODER_EXIT_ON_OOM */ 167 | 168 | #if defined(__cplusplus) || defined(c_plusplus) 169 | } /* extern "C" */ 170 | #endif 171 | -------------------------------------------------------------------------------- /c_src/enc/memory.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Macros for memory management. */ 8 | 9 | #ifndef BROTLI_ENC_MEMORY_H_ 10 | #define BROTLI_ENC_MEMORY_H_ 11 | 12 | #include /* memcpy */ 13 | 14 | #include "../common/platform.h" 15 | #include 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | #if !defined(BROTLI_ENCODER_CLEANUP_ON_OOM) && \ 22 | !defined(BROTLI_ENCODER_EXIT_ON_OOM) 23 | #define BROTLI_ENCODER_EXIT_ON_OOM 24 | #endif 25 | 26 | typedef struct MemoryManager { 27 | brotli_alloc_func alloc_func; 28 | brotli_free_func free_func; 29 | void* opaque; 30 | #if !defined(BROTLI_ENCODER_EXIT_ON_OOM) 31 | BROTLI_BOOL is_oom; 32 | size_t perm_allocated; 33 | size_t new_allocated; 34 | size_t new_freed; 35 | void* pointers[256]; 36 | #endif /* BROTLI_ENCODER_EXIT_ON_OOM */ 37 | } MemoryManager; 38 | 39 | BROTLI_INTERNAL void BrotliInitMemoryManager( 40 | MemoryManager* m, brotli_alloc_func alloc_func, brotli_free_func free_func, 41 | void* opaque); 42 | 43 | BROTLI_INTERNAL void* BrotliAllocate(MemoryManager* m, size_t n); 44 | #define BROTLI_ALLOC(M, T, N) \ 45 | ((N) > 0 ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL) 46 | 47 | BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p); 48 | #define BROTLI_FREE(M, P) { \ 49 | BrotliFree((M), (P)); \ 50 | P = NULL; \ 51 | } 52 | 53 | #if defined(BROTLI_ENCODER_EXIT_ON_OOM) 54 | #define BROTLI_IS_OOM(M) (!!0) 55 | #else /* BROTLI_ENCODER_EXIT_ON_OOM */ 56 | #define BROTLI_IS_OOM(M) (!!(M)->is_oom) 57 | #endif /* BROTLI_ENCODER_EXIT_ON_OOM */ 58 | 59 | /* 60 | BROTLI_IS_NULL is a fake check, BROTLI_IS_OOM does the heavy lifting. 61 | The only purpose of it is to explain static analyzers the state of things. 62 | NB: use ONLY together with BROTLI_IS_OOM 63 | AND ONLY for allocations in the current scope. 64 | */ 65 | #if defined(__clang_analyzer__) && !defined(BROTLI_ENCODER_EXIT_ON_OOM) 66 | #define BROTLI_IS_NULL(A) ((A) == nullptr) 67 | #else /* defined(__clang_analyzer__) */ 68 | #define BROTLI_IS_NULL(A) (!!0) 69 | #endif /* defined(__clang_analyzer__) */ 70 | 71 | BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m); 72 | 73 | /* 74 | Dynamically grows array capacity to at least the requested size 75 | M: MemoryManager 76 | T: data type 77 | A: array 78 | C: capacity 79 | R: requested size 80 | */ 81 | #define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \ 82 | if (C < (R)) { \ 83 | size_t _new_size = (C == 0) ? (R) : C; \ 84 | T* new_array; \ 85 | while (_new_size < (R)) _new_size *= 2; \ 86 | new_array = BROTLI_ALLOC((M), T, _new_size); \ 87 | if (!BROTLI_IS_OOM(M) && !BROTLI_IS_NULL(new_array) && C != 0) \ 88 | memcpy(new_array, A, C * sizeof(T)); \ 89 | BROTLI_FREE((M), A); \ 90 | A = new_array; \ 91 | C = _new_size; \ 92 | } \ 93 | } 94 | 95 | /* 96 | Appends value and dynamically grows array capacity when needed 97 | M: MemoryManager 98 | T: data type 99 | A: array 100 | C: array capacity 101 | S: array size 102 | V: value to append 103 | */ 104 | #define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \ 105 | (S)++; \ 106 | BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \ 107 | A[(S) - 1] = (V); \ 108 | } 109 | 110 | #if defined(__cplusplus) || defined(c_plusplus) 111 | } /* extern "C" */ 112 | #endif 113 | 114 | #endif /* BROTLI_ENC_MEMORY_H_ */ 115 | -------------------------------------------------------------------------------- /c_src/enc/metablock.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Algorithms for distributing the literals and commands of a metablock between 8 | block types and contexts. */ 9 | 10 | #ifndef BROTLI_ENC_METABLOCK_H_ 11 | #define BROTLI_ENC_METABLOCK_H_ 12 | 13 | #include "../common/context.h" 14 | #include "../common/platform.h" 15 | #include 16 | #include "./block_splitter.h" 17 | #include "./command.h" 18 | #include "./histogram.h" 19 | #include "./memory.h" 20 | #include "./quality.h" 21 | 22 | #if defined(__cplusplus) || defined(c_plusplus) 23 | extern "C" { 24 | #endif 25 | 26 | typedef struct MetaBlockSplit { 27 | BlockSplit literal_split; 28 | BlockSplit command_split; 29 | BlockSplit distance_split; 30 | uint32_t* literal_context_map; 31 | size_t literal_context_map_size; 32 | uint32_t* distance_context_map; 33 | size_t distance_context_map_size; 34 | HistogramLiteral* literal_histograms; 35 | size_t literal_histograms_size; 36 | HistogramCommand* command_histograms; 37 | size_t command_histograms_size; 38 | HistogramDistance* distance_histograms; 39 | size_t distance_histograms_size; 40 | } MetaBlockSplit; 41 | 42 | static BROTLI_INLINE void InitMetaBlockSplit(MetaBlockSplit* mb) { 43 | BrotliInitBlockSplit(&mb->literal_split); 44 | BrotliInitBlockSplit(&mb->command_split); 45 | BrotliInitBlockSplit(&mb->distance_split); 46 | mb->literal_context_map = 0; 47 | mb->literal_context_map_size = 0; 48 | mb->distance_context_map = 0; 49 | mb->distance_context_map_size = 0; 50 | mb->literal_histograms = 0; 51 | mb->literal_histograms_size = 0; 52 | mb->command_histograms = 0; 53 | mb->command_histograms_size = 0; 54 | mb->distance_histograms = 0; 55 | mb->distance_histograms_size = 0; 56 | } 57 | 58 | static BROTLI_INLINE void DestroyMetaBlockSplit( 59 | MemoryManager* m, MetaBlockSplit* mb) { 60 | BrotliDestroyBlockSplit(m, &mb->literal_split); 61 | BrotliDestroyBlockSplit(m, &mb->command_split); 62 | BrotliDestroyBlockSplit(m, &mb->distance_split); 63 | BROTLI_FREE(m, mb->literal_context_map); 64 | BROTLI_FREE(m, mb->distance_context_map); 65 | BROTLI_FREE(m, mb->literal_histograms); 66 | BROTLI_FREE(m, mb->command_histograms); 67 | BROTLI_FREE(m, mb->distance_histograms); 68 | } 69 | 70 | /* Uses the slow shortest-path block splitter and does context clustering. 71 | The distance parameters are dynamically selected based on the commands 72 | which get recomputed under the new distance parameters. The new distance 73 | parameters are stored into *params. */ 74 | BROTLI_INTERNAL void BrotliBuildMetaBlock(MemoryManager* m, 75 | const uint8_t* ringbuffer, 76 | const size_t pos, 77 | const size_t mask, 78 | BrotliEncoderParams* params, 79 | uint8_t prev_byte, 80 | uint8_t prev_byte2, 81 | Command* cmds, 82 | size_t num_commands, 83 | ContextType literal_context_mode, 84 | MetaBlockSplit* mb); 85 | 86 | /* Uses a fast greedy block splitter that tries to merge current block with the 87 | last or the second last block and uses a static context clustering which 88 | is the same for all block types. */ 89 | BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy( 90 | MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask, 91 | uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut, 92 | size_t num_contexts, const uint32_t* static_context_map, 93 | const Command* commands, size_t n_commands, MetaBlockSplit* mb); 94 | 95 | BROTLI_INTERNAL void BrotliOptimizeHistograms(uint32_t num_distance_codes, 96 | MetaBlockSplit* mb); 97 | 98 | BROTLI_INTERNAL void BrotliInitDistanceParams(BrotliEncoderParams* params, 99 | uint32_t npostfix, uint32_t ndirect); 100 | 101 | #if defined(__cplusplus) || defined(c_plusplus) 102 | } /* extern "C" */ 103 | #endif 104 | 105 | #endif /* BROTLI_ENC_METABLOCK_H_ */ 106 | -------------------------------------------------------------------------------- /c_src/enc/params.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Parameters for the Brotli encoder with chosen quality levels. */ 8 | 9 | #ifndef BROTLI_ENC_PARAMS_H_ 10 | #define BROTLI_ENC_PARAMS_H_ 11 | 12 | #include 13 | #include "./encoder_dict.h" 14 | 15 | typedef struct BrotliHasherParams { 16 | int type; 17 | int bucket_bits; 18 | int block_bits; 19 | int hash_len; 20 | int num_last_distances_to_check; 21 | } BrotliHasherParams; 22 | 23 | typedef struct BrotliDistanceParams { 24 | uint32_t distance_postfix_bits; 25 | uint32_t num_direct_distance_codes; 26 | uint32_t alphabet_size_max; 27 | uint32_t alphabet_size_limit; 28 | size_t max_distance; 29 | } BrotliDistanceParams; 30 | 31 | /* Encoding parameters */ 32 | typedef struct BrotliEncoderParams { 33 | BrotliEncoderMode mode; 34 | int quality; 35 | int lgwin; 36 | int lgblock; 37 | size_t stream_offset; 38 | size_t size_hint; 39 | BROTLI_BOOL disable_literal_context_modeling; 40 | BROTLI_BOOL large_window; 41 | BrotliHasherParams hasher; 42 | BrotliDistanceParams dist; 43 | BrotliEncoderDictionary dictionary; 44 | } BrotliEncoderParams; 45 | 46 | #endif /* BROTLI_ENC_PARAMS_H_ */ 47 | -------------------------------------------------------------------------------- /c_src/enc/prefix.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Functions for encoding of integers into prefix codes the amount of extra 8 | bits, and the actual values of the extra bits. */ 9 | 10 | #ifndef BROTLI_ENC_PREFIX_H_ 11 | #define BROTLI_ENC_PREFIX_H_ 12 | 13 | #include "../common/constants.h" 14 | #include "../common/platform.h" 15 | #include 16 | #include "./fast_log.h" 17 | 18 | #if defined(__cplusplus) || defined(c_plusplus) 19 | extern "C" { 20 | #endif 21 | 22 | /* Here distance_code is an intermediate code, i.e. one of the special codes or 23 | the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ 24 | static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code, 25 | size_t num_direct_codes, 26 | size_t postfix_bits, 27 | uint16_t* code, 28 | uint32_t* extra_bits) { 29 | if (distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes) { 30 | *code = (uint16_t)distance_code; 31 | *extra_bits = 0; 32 | return; 33 | } else { 34 | size_t dist = ((size_t)1 << (postfix_bits + 2u)) + 35 | (distance_code - BROTLI_NUM_DISTANCE_SHORT_CODES - num_direct_codes); 36 | size_t bucket = Log2FloorNonZero(dist) - 1; 37 | size_t postfix_mask = (1u << postfix_bits) - 1; 38 | size_t postfix = dist & postfix_mask; 39 | size_t prefix = (dist >> bucket) & 1; 40 | size_t offset = (2 + prefix) << bucket; 41 | size_t nbits = bucket - postfix_bits; 42 | *code = (uint16_t)((nbits << 10) | 43 | (BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes + 44 | ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix)); 45 | *extra_bits = (uint32_t)((dist - offset) >> postfix_bits); 46 | } 47 | } 48 | 49 | #if defined(__cplusplus) || defined(c_plusplus) 50 | } /* extern "C" */ 51 | #endif 52 | 53 | #endif /* BROTLI_ENC_PREFIX_H_ */ 54 | -------------------------------------------------------------------------------- /c_src/enc/quality.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2016 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Constants and formulas that affect speed-ratio trade-offs and thus define 8 | quality levels. */ 9 | 10 | #ifndef BROTLI_ENC_QUALITY_H_ 11 | #define BROTLI_ENC_QUALITY_H_ 12 | 13 | #include "../common/platform.h" 14 | #include 15 | #include "./params.h" 16 | 17 | #define FAST_ONE_PASS_COMPRESSION_QUALITY 0 18 | #define FAST_TWO_PASS_COMPRESSION_QUALITY 1 19 | #define ZOPFLIFICATION_QUALITY 10 20 | #define HQ_ZOPFLIFICATION_QUALITY 11 21 | 22 | #define MAX_QUALITY_FOR_STATIC_ENTROPY_CODES 2 23 | #define MIN_QUALITY_FOR_BLOCK_SPLIT 4 24 | #define MIN_QUALITY_FOR_NONZERO_DISTANCE_PARAMS 4 25 | #define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4 26 | #define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5 27 | #define MIN_QUALITY_FOR_CONTEXT_MODELING 5 28 | #define MIN_QUALITY_FOR_HQ_CONTEXT_MODELING 7 29 | #define MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING 10 30 | 31 | /* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting, 32 | so we buffer at most this much literals and commands. */ 33 | #define MAX_NUM_DELAYED_SYMBOLS 0x2FFF 34 | 35 | /* Returns hash-table size for quality levels 0 and 1. */ 36 | static BROTLI_INLINE size_t MaxHashTableSize(int quality) { 37 | return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17; 38 | } 39 | 40 | /* The maximum length for which the zopflification uses distinct distances. */ 41 | #define MAX_ZOPFLI_LEN_QUALITY_10 150 42 | #define MAX_ZOPFLI_LEN_QUALITY_11 325 43 | 44 | /* Do not thoroughly search when a long copy is found. */ 45 | #define BROTLI_LONG_COPY_QUICK_STEP 16384 46 | 47 | static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) { 48 | return params->quality <= 10 ? 49 | MAX_ZOPFLI_LEN_QUALITY_10 : 50 | MAX_ZOPFLI_LEN_QUALITY_11; 51 | } 52 | 53 | /* Number of best candidates to evaluate to expand Zopfli chain. */ 54 | static BROTLI_INLINE size_t MaxZopfliCandidates( 55 | const BrotliEncoderParams* params) { 56 | return params->quality <= 10 ? 1 : 5; 57 | } 58 | 59 | static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) { 60 | params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY, 61 | BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality)); 62 | if (params->quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES) { 63 | params->large_window = BROTLI_FALSE; 64 | } 65 | if (params->lgwin < BROTLI_MIN_WINDOW_BITS) { 66 | params->lgwin = BROTLI_MIN_WINDOW_BITS; 67 | } else { 68 | int max_lgwin = params->large_window ? BROTLI_LARGE_MAX_WINDOW_BITS : 69 | BROTLI_MAX_WINDOW_BITS; 70 | if (params->lgwin > max_lgwin) params->lgwin = max_lgwin; 71 | } 72 | } 73 | 74 | /* Returns optimized lg_block value. */ 75 | static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) { 76 | int lgblock = params->lgblock; 77 | if (params->quality == FAST_ONE_PASS_COMPRESSION_QUALITY || 78 | params->quality == FAST_TWO_PASS_COMPRESSION_QUALITY) { 79 | lgblock = params->lgwin; 80 | } else if (params->quality < MIN_QUALITY_FOR_BLOCK_SPLIT) { 81 | lgblock = 14; 82 | } else if (lgblock == 0) { 83 | lgblock = 16; 84 | if (params->quality >= 9 && params->lgwin > lgblock) { 85 | lgblock = BROTLI_MIN(int, 18, params->lgwin); 86 | } 87 | } else { 88 | lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS, 89 | BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock)); 90 | } 91 | return lgblock; 92 | } 93 | 94 | /* Returns log2 of the size of main ring buffer area. 95 | Allocate at least lgwin + 1 bits for the ring buffer so that the newly 96 | added block fits there completely and we still get lgwin bits and at least 97 | read_block_size_bits + 1 bits because the copy tail length needs to be 98 | smaller than ring-buffer size. */ 99 | static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) { 100 | return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock); 101 | } 102 | 103 | static BROTLI_INLINE size_t MaxMetablockSize( 104 | const BrotliEncoderParams* params) { 105 | int bits = 106 | BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS); 107 | return (size_t)1 << bits; 108 | } 109 | 110 | /* When searching for backward references and have not seen matches for a long 111 | time, we can skip some match lookups. Unsuccessful match lookups are very 112 | expensive and this kind of a heuristic speeds up compression quite a lot. 113 | At first 8 byte strides are taken and every second byte is put to hasher. 114 | After 4x more literals stride by 16 bytes, every put 4-th byte to hasher. 115 | Applied only to qualities 2 to 9. */ 116 | static BROTLI_INLINE size_t LiteralSpreeLengthForSparseSearch( 117 | const BrotliEncoderParams* params) { 118 | return params->quality < 9 ? 64 : 512; 119 | } 120 | 121 | static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params, 122 | BrotliHasherParams* hparams) { 123 | if (params->quality > 9) { 124 | hparams->type = 10; 125 | } else if (params->quality == 4 && params->size_hint >= (1 << 20)) { 126 | hparams->type = 54; 127 | } else if (params->quality < 5) { 128 | hparams->type = params->quality; 129 | } else if (params->lgwin <= 16) { 130 | hparams->type = params->quality < 7 ? 40 : params->quality < 9 ? 41 : 42; 131 | } else if (params->size_hint >= (1 << 20) && params->lgwin >= 19) { 132 | hparams->type = 6; 133 | hparams->block_bits = params->quality - 1; 134 | hparams->bucket_bits = 15; 135 | hparams->hash_len = 5; 136 | hparams->num_last_distances_to_check = 137 | params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16; 138 | } else { 139 | hparams->type = 5; 140 | hparams->block_bits = params->quality - 1; 141 | hparams->bucket_bits = params->quality < 7 ? 14 : 15; 142 | hparams->num_last_distances_to_check = 143 | params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16; 144 | } 145 | 146 | if (params->lgwin > 24) { 147 | /* Different hashers for large window brotli: not for qualities <= 2, 148 | these are too fast for large window. Not for qualities >= 10: their 149 | hasher already works well with large window. So the changes are: 150 | H3 --> H35: for quality 3. 151 | H54 --> H55: for quality 4 with size hint > 1MB 152 | H6 --> H65: for qualities 5, 6, 7, 8, 9. */ 153 | if (hparams->type == 3) { 154 | hparams->type = 35; 155 | } 156 | if (hparams->type == 54) { 157 | hparams->type = 55; 158 | } 159 | if (hparams->type == 6) { 160 | hparams->type = 65; 161 | } 162 | } 163 | } 164 | 165 | #endif /* BROTLI_ENC_QUALITY_H_ */ 166 | -------------------------------------------------------------------------------- /c_src/enc/ringbuffer.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Sliding window over the input data. */ 8 | 9 | #ifndef BROTLI_ENC_RINGBUFFER_H_ 10 | #define BROTLI_ENC_RINGBUFFER_H_ 11 | 12 | #include /* memcpy */ 13 | 14 | #include "../common/platform.h" 15 | #include 16 | #include "./memory.h" 17 | #include "./quality.h" 18 | 19 | #if defined(__cplusplus) || defined(c_plusplus) 20 | extern "C" { 21 | #endif 22 | 23 | /* A RingBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of 24 | data in a circular manner: writing a byte writes it to: 25 | `position() % (1 << window_bits)'. 26 | For convenience, the RingBuffer array contains another copy of the 27 | first `1 << tail_bits' bytes: 28 | buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits), 29 | and another copy of the last two bytes: 30 | buffer_[-1] == buffer_[(1 << window_bits) - 1] and 31 | buffer_[-2] == buffer_[(1 << window_bits) - 2]. */ 32 | typedef struct RingBuffer { 33 | /* Size of the ring-buffer is (1 << window_bits) + tail_size_. */ 34 | const uint32_t size_; 35 | const uint32_t mask_; 36 | const uint32_t tail_size_; 37 | const uint32_t total_size_; 38 | 39 | uint32_t cur_size_; 40 | /* Position to write in the ring buffer. */ 41 | uint32_t pos_; 42 | /* The actual ring buffer containing the copy of the last two bytes, the data, 43 | and the copy of the beginning as a tail. */ 44 | uint8_t* data_; 45 | /* The start of the ring-buffer. */ 46 | uint8_t* buffer_; 47 | } RingBuffer; 48 | 49 | static BROTLI_INLINE void RingBufferInit(RingBuffer* rb) { 50 | rb->cur_size_ = 0; 51 | rb->pos_ = 0; 52 | rb->data_ = 0; 53 | rb->buffer_ = 0; 54 | } 55 | 56 | static BROTLI_INLINE void RingBufferSetup( 57 | const BrotliEncoderParams* params, RingBuffer* rb) { 58 | int window_bits = ComputeRbBits(params); 59 | int tail_bits = params->lgblock; 60 | *(uint32_t*)&rb->size_ = 1u << window_bits; 61 | *(uint32_t*)&rb->mask_ = (1u << window_bits) - 1; 62 | *(uint32_t*)&rb->tail_size_ = 1u << tail_bits; 63 | *(uint32_t*)&rb->total_size_ = rb->size_ + rb->tail_size_; 64 | } 65 | 66 | static BROTLI_INLINE void RingBufferFree(MemoryManager* m, RingBuffer* rb) { 67 | BROTLI_FREE(m, rb->data_); 68 | } 69 | 70 | /* Allocates or re-allocates data_ to the given length + plus some slack 71 | region before and after. Fills the slack regions with zeros. */ 72 | static BROTLI_INLINE void RingBufferInitBuffer( 73 | MemoryManager* m, const uint32_t buflen, RingBuffer* rb) { 74 | static const size_t kSlackForEightByteHashingEverywhere = 7; 75 | uint8_t* new_data = BROTLI_ALLOC( 76 | m, uint8_t, 2 + buflen + kSlackForEightByteHashingEverywhere); 77 | size_t i; 78 | if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_data)) return; 79 | if (rb->data_) { 80 | memcpy(new_data, rb->data_, 81 | 2 + rb->cur_size_ + kSlackForEightByteHashingEverywhere); 82 | BROTLI_FREE(m, rb->data_); 83 | } 84 | rb->data_ = new_data; 85 | rb->cur_size_ = buflen; 86 | rb->buffer_ = rb->data_ + 2; 87 | rb->buffer_[-2] = rb->buffer_[-1] = 0; 88 | for (i = 0; i < kSlackForEightByteHashingEverywhere; ++i) { 89 | rb->buffer_[rb->cur_size_ + i] = 0; 90 | } 91 | } 92 | 93 | static BROTLI_INLINE void RingBufferWriteTail( 94 | const uint8_t* bytes, size_t n, RingBuffer* rb) { 95 | const size_t masked_pos = rb->pos_ & rb->mask_; 96 | if (BROTLI_PREDICT_FALSE(masked_pos < rb->tail_size_)) { 97 | /* Just fill the tail buffer with the beginning data. */ 98 | const size_t p = rb->size_ + masked_pos; 99 | memcpy(&rb->buffer_[p], bytes, 100 | BROTLI_MIN(size_t, n, rb->tail_size_ - masked_pos)); 101 | } 102 | } 103 | 104 | /* Push bytes into the ring buffer. */ 105 | static BROTLI_INLINE void RingBufferWrite( 106 | MemoryManager* m, const uint8_t* bytes, size_t n, RingBuffer* rb) { 107 | if (rb->pos_ == 0 && n < rb->tail_size_) { 108 | /* Special case for the first write: to process the first block, we don't 109 | need to allocate the whole ring-buffer and we don't need the tail 110 | either. However, we do this memory usage optimization only if the 111 | first write is less than the tail size, which is also the input block 112 | size, otherwise it is likely that other blocks will follow and we 113 | will need to reallocate to the full size anyway. */ 114 | rb->pos_ = (uint32_t)n; 115 | RingBufferInitBuffer(m, rb->pos_, rb); 116 | if (BROTLI_IS_OOM(m)) return; 117 | memcpy(rb->buffer_, bytes, n); 118 | return; 119 | } 120 | if (rb->cur_size_ < rb->total_size_) { 121 | /* Lazily allocate the full buffer. */ 122 | RingBufferInitBuffer(m, rb->total_size_, rb); 123 | if (BROTLI_IS_OOM(m)) return; 124 | /* Initialize the last two bytes to zero, so that we don't have to worry 125 | later when we copy the last two bytes to the first two positions. */ 126 | rb->buffer_[rb->size_ - 2] = 0; 127 | rb->buffer_[rb->size_ - 1] = 0; 128 | /* Initialize tail; might be touched by "best_len++" optimization when 129 | ring buffer is "full". */ 130 | rb->buffer_[rb->size_] = 241; 131 | } 132 | { 133 | const size_t masked_pos = rb->pos_ & rb->mask_; 134 | /* The length of the writes is limited so that we do not need to worry 135 | about a write */ 136 | RingBufferWriteTail(bytes, n, rb); 137 | if (BROTLI_PREDICT_TRUE(masked_pos + n <= rb->size_)) { 138 | /* A single write fits. */ 139 | memcpy(&rb->buffer_[masked_pos], bytes, n); 140 | } else { 141 | /* Split into two writes. 142 | Copy into the end of the buffer, including the tail buffer. */ 143 | memcpy(&rb->buffer_[masked_pos], bytes, 144 | BROTLI_MIN(size_t, n, rb->total_size_ - masked_pos)); 145 | /* Copy into the beginning of the buffer */ 146 | memcpy(&rb->buffer_[0], bytes + (rb->size_ - masked_pos), 147 | n - (rb->size_ - masked_pos)); 148 | } 149 | } 150 | { 151 | BROTLI_BOOL not_first_lap = (rb->pos_ & (1u << 31)) != 0; 152 | uint32_t rb_pos_mask = (1u << 31) - 1; 153 | rb->buffer_[-2] = rb->buffer_[rb->size_ - 2]; 154 | rb->buffer_[-1] = rb->buffer_[rb->size_ - 1]; 155 | rb->pos_ = (rb->pos_ & rb_pos_mask) + (uint32_t)(n & rb_pos_mask); 156 | if (not_first_lap) { 157 | /* Wrap, but preserve not-a-first-lap feature. */ 158 | rb->pos_ |= 1u << 31; 159 | } 160 | } 161 | } 162 | 163 | #if defined(__cplusplus) || defined(c_plusplus) 164 | } /* extern "C" */ 165 | #endif 166 | 167 | #endif /* BROTLI_ENC_RINGBUFFER_H_ */ 168 | -------------------------------------------------------------------------------- /c_src/enc/static_dict.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Class to model the static dictionary. */ 8 | 9 | #ifndef BROTLI_ENC_STATIC_DICT_H_ 10 | #define BROTLI_ENC_STATIC_DICT_H_ 11 | 12 | #include "../common/dictionary.h" 13 | #include "../common/platform.h" 14 | #include 15 | #include "./encoder_dict.h" 16 | 17 | #if defined(__cplusplus) || defined(c_plusplus) 18 | extern "C" { 19 | #endif 20 | 21 | #define BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN 37 22 | static const uint32_t kInvalidMatch = 0xFFFFFFF; 23 | 24 | /* Matches data against static dictionary words, and for each length l, 25 | for which a match is found, updates matches[l] to be the minimum possible 26 | (distance << 5) + len_code. 27 | Returns 1 if matches have been found, otherwise 0. 28 | Prerequisites: 29 | matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long 30 | all elements are initialized to kInvalidMatch */ 31 | BROTLI_INTERNAL BROTLI_BOOL BrotliFindAllStaticDictionaryMatches( 32 | const BrotliEncoderDictionary* dictionary, 33 | const uint8_t* data, size_t min_length, size_t max_length, 34 | uint32_t* matches); 35 | 36 | #if defined(__cplusplus) || defined(c_plusplus) 37 | } /* extern "C" */ 38 | #endif 39 | 40 | #endif /* BROTLI_ENC_STATIC_DICT_H_ */ 41 | -------------------------------------------------------------------------------- /c_src/enc/utf8_util.c: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Heuristics for deciding about the UTF8-ness of strings. */ 8 | 9 | #include "./utf8_util.h" 10 | 11 | #include 12 | 13 | #if defined(__cplusplus) || defined(c_plusplus) 14 | extern "C" { 15 | #endif 16 | 17 | static size_t BrotliParseAsUTF8( 18 | int* symbol, const uint8_t* input, size_t size) { 19 | /* ASCII */ 20 | if ((input[0] & 0x80) == 0) { 21 | *symbol = input[0]; 22 | if (*symbol > 0) { 23 | return 1; 24 | } 25 | } 26 | /* 2-byte UTF8 */ 27 | if (size > 1u && 28 | (input[0] & 0xE0) == 0xC0 && 29 | (input[1] & 0xC0) == 0x80) { 30 | *symbol = (((input[0] & 0x1F) << 6) | 31 | (input[1] & 0x3F)); 32 | if (*symbol > 0x7F) { 33 | return 2; 34 | } 35 | } 36 | /* 3-byte UFT8 */ 37 | if (size > 2u && 38 | (input[0] & 0xF0) == 0xE0 && 39 | (input[1] & 0xC0) == 0x80 && 40 | (input[2] & 0xC0) == 0x80) { 41 | *symbol = (((input[0] & 0x0F) << 12) | 42 | ((input[1] & 0x3F) << 6) | 43 | (input[2] & 0x3F)); 44 | if (*symbol > 0x7FF) { 45 | return 3; 46 | } 47 | } 48 | /* 4-byte UFT8 */ 49 | if (size > 3u && 50 | (input[0] & 0xF8) == 0xF0 && 51 | (input[1] & 0xC0) == 0x80 && 52 | (input[2] & 0xC0) == 0x80 && 53 | (input[3] & 0xC0) == 0x80) { 54 | *symbol = (((input[0] & 0x07) << 18) | 55 | ((input[1] & 0x3F) << 12) | 56 | ((input[2] & 0x3F) << 6) | 57 | (input[3] & 0x3F)); 58 | if (*symbol > 0xFFFF && *symbol <= 0x10FFFF) { 59 | return 4; 60 | } 61 | } 62 | /* Not UTF8, emit a special symbol above the UTF8-code space */ 63 | *symbol = 0x110000 | input[0]; 64 | return 1; 65 | } 66 | 67 | /* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/ 68 | BROTLI_BOOL BrotliIsMostlyUTF8( 69 | const uint8_t* data, const size_t pos, const size_t mask, 70 | const size_t length, const double min_fraction) { 71 | size_t size_utf8 = 0; 72 | size_t i = 0; 73 | while (i < length) { 74 | int symbol; 75 | size_t bytes_read = 76 | BrotliParseAsUTF8(&symbol, &data[(pos + i) & mask], length - i); 77 | i += bytes_read; 78 | if (symbol < 0x110000) size_utf8 += bytes_read; 79 | } 80 | return TO_BROTLI_BOOL((double)size_utf8 > min_fraction * (double)length); 81 | } 82 | 83 | #if defined(__cplusplus) || defined(c_plusplus) 84 | } /* extern "C" */ 85 | #endif 86 | -------------------------------------------------------------------------------- /c_src/enc/utf8_util.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Heuristics for deciding about the UTF8-ness of strings. */ 8 | 9 | #ifndef BROTLI_ENC_UTF8_UTIL_H_ 10 | #define BROTLI_ENC_UTF8_UTIL_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | static const double kMinUTF8Ratio = 0.75; 20 | 21 | /* Returns 1 if at least min_fraction of the bytes between pos and 22 | pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise 23 | returns 0. */ 24 | BROTLI_INTERNAL BROTLI_BOOL BrotliIsMostlyUTF8( 25 | const uint8_t* data, const size_t pos, const size_t mask, 26 | const size_t length, const double min_fraction); 27 | 28 | #if defined(__cplusplus) || defined(c_plusplus) 29 | } /* extern "C" */ 30 | #endif 31 | 32 | #endif /* BROTLI_ENC_UTF8_UTIL_H_ */ 33 | -------------------------------------------------------------------------------- /c_src/enc/write_bits.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2010 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /* Write bits into a byte array. */ 8 | 9 | #ifndef BROTLI_ENC_WRITE_BITS_H_ 10 | #define BROTLI_ENC_WRITE_BITS_H_ 11 | 12 | #include "../common/platform.h" 13 | #include 14 | 15 | #if defined(__cplusplus) || defined(c_plusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /* This function writes bits into bytes in increasing addresses, and within 20 | a byte least-significant-bit first. 21 | 22 | The function can write up to 56 bits in one go with WriteBits 23 | Example: let's assume that 3 bits (Rs below) have been written already: 24 | 25 | BYTE-0 BYTE+1 BYTE+2 26 | 27 | 0000 0RRR 0000 0000 0000 0000 28 | 29 | Now, we could write 5 or less bits in MSB by just shifting by 3 30 | and OR'ing to BYTE-0. 31 | 32 | For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, 33 | and locate the rest in BYTE+1, BYTE+2, etc. */ 34 | static BROTLI_INLINE void BrotliWriteBits(size_t n_bits, 35 | uint64_t bits, 36 | size_t* BROTLI_RESTRICT pos, 37 | uint8_t* BROTLI_RESTRICT array) { 38 | BROTLI_LOG(("WriteBits %2d 0x%08x%08x %10d\n", (int)n_bits, 39 | (uint32_t)(bits >> 32), (uint32_t)(bits & 0xFFFFFFFF), 40 | (int)*pos)); 41 | BROTLI_DCHECK((bits >> n_bits) == 0); 42 | BROTLI_DCHECK(n_bits <= 56); 43 | #if defined(BROTLI_LITTLE_ENDIAN) 44 | /* This branch of the code can write up to 56 bits at a time, 45 | 7 bits are lost by being perhaps already in *p and at least 46 | 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 47 | bits are in *p and we write 57 bits, then the next write will 48 | access a byte that was never initialized). */ 49 | { 50 | uint8_t* p = &array[*pos >> 3]; 51 | uint64_t v = (uint64_t)(*p); /* Zero-extend 8 to 64 bits. */ 52 | v |= bits << (*pos & 7); 53 | BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */ 54 | *pos += n_bits; 55 | } 56 | #else 57 | /* implicit & 0xFF is assumed for uint8_t arithmetics */ 58 | { 59 | uint8_t* array_pos = &array[*pos >> 3]; 60 | const size_t bits_reserved_in_first_byte = (*pos & 7); 61 | size_t bits_left_to_write; 62 | bits <<= bits_reserved_in_first_byte; 63 | *array_pos++ |= (uint8_t)bits; 64 | for (bits_left_to_write = n_bits + bits_reserved_in_first_byte; 65 | bits_left_to_write >= 9; 66 | bits_left_to_write -= 8) { 67 | bits >>= 8; 68 | *array_pos++ = (uint8_t)bits; 69 | } 70 | *array_pos = 0; 71 | *pos += n_bits; 72 | } 73 | #endif 74 | } 75 | 76 | static BROTLI_INLINE void BrotliWriteBitsPrepareStorage( 77 | size_t pos, uint8_t* array) { 78 | BROTLI_LOG(("WriteBitsPrepareStorage %10d\n", (int)pos)); 79 | BROTLI_DCHECK((pos & 7) == 0); 80 | array[pos >> 3] = 0; 81 | } 82 | 83 | #if defined(__cplusplus) || defined(c_plusplus) 84 | } /* extern "C" */ 85 | #endif 86 | 87 | #endif /* BROTLI_ENC_WRITE_BITS_H_ */ 88 | -------------------------------------------------------------------------------- /c_src/include/brotli/types.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2013 Google Inc. All Rights Reserved. 2 | 3 | Distributed under MIT license. 4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT 5 | */ 6 | 7 | /** 8 | * @file 9 | * Common types used in decoder and encoder API. 10 | */ 11 | 12 | #ifndef BROTLI_COMMON_TYPES_H_ 13 | #define BROTLI_COMMON_TYPES_H_ 14 | 15 | #include /* for size_t */ 16 | 17 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 18 | typedef __int8 int8_t; 19 | typedef unsigned __int8 uint8_t; 20 | typedef __int16 int16_t; 21 | typedef unsigned __int16 uint16_t; 22 | typedef __int32 int32_t; 23 | typedef unsigned __int32 uint32_t; 24 | typedef unsigned __int64 uint64_t; 25 | typedef __int64 int64_t; 26 | #else 27 | #include 28 | #endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */ 29 | 30 | /** 31 | * A portable @c bool replacement. 32 | * 33 | * ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it 34 | * denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE. 35 | * 36 | * ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or 37 | * ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros. 38 | * 39 | * ::BROTLI_BOOL values returned by Brotli should not be tested for equality 40 | * with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be 41 | * evaluated, for example: @code{.cpp} 42 | * if (SomeBrotliFunction(encoder, BROTLI_TRUE) && 43 | * !OtherBrotliFunction(decoder, BROTLI_FALSE)) { 44 | * bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4)); 45 | * DoSomething(x); 46 | * } 47 | * @endcode 48 | */ 49 | #define BROTLI_BOOL int 50 | /** Portable @c true replacement. */ 51 | #define BROTLI_TRUE 1 52 | /** Portable @c false replacement. */ 53 | #define BROTLI_FALSE 0 54 | /** @c bool to ::BROTLI_BOOL conversion macros. */ 55 | #define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE) 56 | 57 | #define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low) 58 | 59 | #define BROTLI_UINT32_MAX (~((uint32_t)0)) 60 | #define BROTLI_SIZE_MAX (~((size_t)0)) 61 | 62 | /** 63 | * Allocating function pointer type. 64 | * 65 | * @param opaque custom memory manager handle provided by client 66 | * @param size requested memory region size; can not be @c 0 67 | * @returns @c 0 in the case of failure 68 | * @returns a valid pointer to a memory region of at least @p size bytes 69 | * long otherwise 70 | */ 71 | typedef void* (*brotli_alloc_func)(void* opaque, size_t size); 72 | 73 | /** 74 | * Deallocating function pointer type. 75 | * 76 | * This function @b SHOULD do nothing if @p address is @c 0. 77 | * 78 | * @param opaque custom memory manager handle provided by client 79 | * @param address memory region pointer returned by ::brotli_alloc_func, or @c 0 80 | */ 81 | typedef void (*brotli_free_func)(void* opaque, void* address); 82 | 83 | #endif /* BROTLI_COMMON_TYPES_H_ */ 84 | -------------------------------------------------------------------------------- /c_src/tools/brotli.md: -------------------------------------------------------------------------------- 1 | brotli(1) -- brotli, unbrotli - compress or decompress files 2 | ================================================================ 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | `brotli` [*OPTION|FILE*]... 8 | 9 | `unbrotli` is equivalent to `brotli --decompress` 10 | 11 | DESCRIPTION 12 | ----------- 13 | `brotli` is a generic-purpose lossless compression algorithm that compresses 14 | data using a combination of a modern variant of the **LZ77** algorithm, Huffman 15 | coding and 2-nd order context modeling, with a compression ratio comparable to 16 | the best currently available general-purpose compression methods. It is similar 17 | in speed with deflate but offers more dense compression. 18 | 19 | `brotli` command line syntax similar to `gzip (1)` and `zstd (1)`. 20 | Unlike `gzip (1)`, source files are preserved by default. It is possible to 21 | remove them after processing by using the `--rm` _option_. 22 | 23 | Arguments that look like "`--name`" or "`--name=value`" are _options_. Every 24 | _option_ has a short form "`-x`" or "`-x value`". Multiple short form _options_ 25 | could be coalesced: 26 | 27 | * "`--decompress --stdout --suffix=.b`" works the same as 28 | * "`-d -s -S .b`" and 29 | * "`-dsS .b`" 30 | 31 | `brotli` has 3 operation modes: 32 | 33 | * default mode is compression; 34 | * `--decompress` option activates decompression mode; 35 | * `--test` option switches to integrity test mode; this option is equivalent to 36 | "`--decompress --stdout`" except that the decompressed data is discarded 37 | instead of being written to standard output. 38 | 39 | Every non-option argument is a _file_ entry. If no _files_ are given or _file_ 40 | is "`-`", `brotli` reads from standard input. All arguments after "`--`" are 41 | _file_ entries. 42 | 43 | Unless `--stdout` or `--output` is specified, _files_ are written to a new file 44 | whose name is derived from the source _file_ name: 45 | 46 | * when compressing, a suffix is appended to the source filename to 47 | get the target filename 48 | * when decompressing, a suffix is removed from the source filename to 49 | get the target filename 50 | 51 | Default suffix is `.br`, but it could be specified with `--suffix` option. 52 | 53 | Conflicting or duplicate _options_ are not allowed. 54 | 55 | OPTIONS 56 | ------- 57 | 58 | * `-#`: 59 | compression level (0-9); bigger values cause denser, but slower compression 60 | * `-c`, `--stdout`: 61 | write on standard output 62 | * `-d`, `--decompress`: 63 | decompress mode 64 | * `-f`, `--force`: 65 | force output file overwrite 66 | * `-h`, `--help`: 67 | display this help and exit 68 | * `-j`, `--rm`: 69 | remove source file(s); `gzip (1)`-like behaviour 70 | * `-k`, `--keep`: 71 | keep source file(s); `zstd (1)`-like behaviour 72 | * `-n`, `--no-copy-stat`: 73 | do not copy source file(s) attributes 74 | * `-o FILE`, `--output=FILE` 75 | output file; valid only if there is a single input entry 76 | * `-q NUM`, `--quality=NUM`: 77 | compression level (0-11); bigger values cause denser, but slower compression 78 | * `-t`, `--test`: 79 | test file integrity mode 80 | * `-v`, `--verbose`: 81 | increase output verbosity 82 | * `-w NUM`, `--lgwin=NUM`: 83 | set LZ77 window size (0, 10-24) (default: 22); window size is 84 | `(2**NUM - 16)`; 0 lets compressor decide over the optimal value; bigger 85 | windows size improve density; decoder might require up to window size 86 | memory to operate 87 | * `-S SUF`, `--suffix=SUF`: 88 | output file suffix (default: `.br`) 89 | * `-V`, `--version`: 90 | display version and exit 91 | * `-Z`, `--best`: 92 | use best compression level (default); same as "`-q 11`" 93 | 94 | SEE ALSO 95 | -------- 96 | 97 | `brotli` file format is defined in 98 | [RFC 7932](https://www.ietf.org/rfc/rfc7932.txt). 99 | 100 | `brotli` is open-sourced under the 101 | [MIT License](https://opensource.org/licenses/MIT). 102 | 103 | Mailing list: https://groups.google.com/forum/#!forum/brotli 104 | 105 | BUGS 106 | ---- 107 | Report bugs at: https://github.com/google/brotli/issues 108 | -------------------------------------------------------------------------------- /priv/.gitignore: -------------------------------------------------------------------------------- 1 | brotli 2 | *.so 3 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [ 2 | debug_info, 3 | warnings_as_errors, 4 | warn_export_all, 5 | warn_obsolete_guards, 6 | warn_unused_import 7 | ]}. 8 | 9 | {project_plugins, [ 10 | covertool, 11 | erlfmt, 12 | rebar3_proper, 13 | rebar3_ex_doc 14 | ]}. 15 | 16 | {cover_enabled, true}. 17 | {cover_export_enabled, true}. 18 | {cover_excl_mods, [brotli_nif]}. 19 | {covertool, [ 20 | {coverdata_files, ["ct.coverdata", "eunit.coverdata", "proper.coverdata"]} 21 | ]}. 22 | 23 | {ex_doc, [ 24 | {source_url, <<"https://github.com/yjh0502/erl-brotli">>}, 25 | {extras, [<<"README.md">>, <<"LICENSE">>]}, 26 | {main, <<"readme">>} 27 | ]}. 28 | 29 | {hex, [ 30 | {doc, #{provider => ex_doc}} 31 | ]}. 32 | 33 | {xref_checks, [ 34 | undefined_function_calls, 35 | undefined_functions, 36 | deprecated_function_calls, 37 | deprecated_functions 38 | ]}. 39 | {xref_ignores, []}. 40 | 41 | {profiles, [ 42 | {test, [ 43 | {erl_opts, [debug_info, nowarn_export_all, warnings_as_errors]}, 44 | {deps, [proper]} 45 | ]} 46 | ]}. 47 | 48 | {pre_hooks, [ 49 | {"(linux|darwin|solaris)", compile, "make -C c_src -j"}, 50 | {"(freebsd)", compile, "gmake -C c_src"}, 51 | {"(linux|darwin|solaris)", proper, "make -C c_src -j tool"}, 52 | {"(freebsd)", proper, "gmake -C c_src tool"} 53 | ]}. 54 | {post_hooks, [ 55 | {"(linux|darwin|solaris)", clean, "make -C c_src clean"}, 56 | {"(freebsd)", clean, "gmake -C c_src clean"} 57 | ]}. 58 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/brotli.app.src: -------------------------------------------------------------------------------- 1 | {application, brotli, [ 2 | {description, "brotli compression library wrapper for Erlang"}, 3 | {vsn, git}, 4 | {registered, []}, 5 | {applications, [kernel, stdlib]}, 6 | {env, []}, 7 | {modules, []}, 8 | {maintainers, ["Jihyun Yu"]}, 9 | {licenses, ["BSD-3-Clause"]}, 10 | {exclude_files, ["priv/brotli.so", "priv/brotli"]}, 11 | {links, [{"Github", "https://github.com/yjh0502/erl-brotli"}]} 12 | ]}. 13 | -------------------------------------------------------------------------------- /src/brotli.erl: -------------------------------------------------------------------------------- 1 | %%% Copyright (c) 2016 Jihyun Yu 2 | %%% Copyright (c) 2021 Łukasz Niemier 3 | %%% All rights reserved. 4 | %%% 5 | %%% Redistribution and use in source and binary forms, with or without 6 | %%% modification, are permitted provided that the following conditions 7 | %%% are met: 8 | %%% 9 | %%% 1. Redistributions of source code must retain the above copyright 10 | %%% notice, this list of conditions and the following disclaimer. 11 | %%% 2. Redistributions in binary form must reproduce the above copyright 12 | %%% notice, this list of conditions and the following disclaimer in 13 | %%% the documentation and/or other materials provided with the 14 | %%% distribution. 15 | %%% 16 | %%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | %%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | %%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 | %%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 20 | %%% COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | %%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | %%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | %%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | %%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 | %%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 26 | %%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | %%% POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -module(brotli). 30 | 31 | -export([version/0, max_compressed_size/1]). 32 | -export([encode/1, encode/2]). 33 | -export([decode/1, decode/2]). 34 | 35 | %% @doc Return version of the underlying Brotli C implementation. 36 | %% 37 | %% This is not version of the Erlang library. 38 | %% @end 39 | -spec version() -> {Major :: integer(), Minor :: integer(), Patch :: integer()}. 40 | version() -> 41 | brotli_nif:version(). 42 | 43 | %% @doc Maximal possible size of the compressed data of size `Size'. 44 | %% @end 45 | -spec max_compressed_size(non_neg_integer()) -> non_neg_integer(). 46 | max_compressed_size(Size) -> 47 | brotli_nif:max_compressed_size(Size). 48 | 49 | %% @doc Compress data in one step. 50 | %% @end 51 | -spec encode(Data :: iodata()) -> {ok, iodata()} | error. 52 | encode(Data) -> 53 | encode(Data, #{}). 54 | 55 | %% @doc Compress data in one step with custom options. 56 | %% @end 57 | -spec encode(Data :: iodata(), Opts :: brotli_encoder:options()) -> {ok, iodata()} | error. 58 | encode(Data, Opts0) when is_map(Opts0) -> 59 | Opts = maps:put(size_hint, iolist_size(Data), Opts0), 60 | Encoder = brotli_encoder:new(Opts), 61 | case encode_chunks(Encoder, iolist_to_binary(Data), []) of 62 | {ok, Compressed} -> 63 | case brotli_encoder:is_finished(Encoder) of 64 | true -> {ok, iolist_to_binary(Compressed)}; 65 | false -> {error, nf} 66 | end; 67 | Other -> 68 | Other 69 | end. 70 | 71 | encode_chunks(Encoder, <>, Acc) -> 72 | case brotli_encoder:append(Encoder, Data) of 73 | {ok, Compressed} -> 74 | encode_chunks(Encoder, Rest, [Compressed | Acc]); 75 | Other -> 76 | Other 77 | end; 78 | encode_chunks(Encoder, Data, Acc) -> 79 | case brotli_encoder:finish(Encoder, Data) of 80 | {ok, Compressed} -> 81 | {ok, lists:reverse([Compressed | Acc])}; 82 | Other -> 83 | Other 84 | end. 85 | 86 | decode(Data) -> 87 | decode(Data, #{}). 88 | decode(Data, _Opts) -> 89 | Decoder = brotli_decoder:new(), 90 | case brotli_decoder:stream(Decoder, Data) of 91 | {ok, _} = Result -> 92 | case brotli_decoder:is_finished(Decoder) of 93 | true -> Result; 94 | false -> error 95 | end; 96 | _ -> 97 | error 98 | end. 99 | -------------------------------------------------------------------------------- /src/brotli_decoder.erl: -------------------------------------------------------------------------------- 1 | %%% Copyright (c) 2021 Łukasz Niemier 2 | %%% All rights reserved. 3 | %%% 4 | %%% Redistribution and use in source and binary forms, with or without 5 | %%% modification, are permitted provided that the following conditions 6 | %%% are met: 7 | %%% 8 | %%% 1. Redistributions of source code must retain the above copyright 9 | %%% notice, this list of conditions and the following disclaimer. 10 | %%% 2. Redistributions in binary form must reproduce the above copyright 11 | %%% notice, this list of conditions and the following disclaimer in 12 | %%% the documentation and/or other materials provided with the 13 | %%% distribution. 14 | %%% 15 | %%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | %%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | %%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 | %%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 | %%% COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | %%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 | %%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | %%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | %%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 | %%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 25 | %%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | %%% POSSIBILITY OF SUCH DAMAGE. 27 | 28 | -module(brotli_decoder). 29 | 30 | -export([new/0, stream/2]). 31 | -export([is_finished/1, is_used/1]). 32 | 33 | -export_type([t/0]). 34 | 35 | -opaque t() :: reference(). 36 | 37 | -spec new() -> t(). 38 | new() -> 39 | brotli_nif:decoder_create(). 40 | 41 | -spec stream(Decoder :: t(), Data :: iodata()) -> {ok | more, iodata()} | error. 42 | stream(Decoder, Data) -> 43 | case brotli_nif:decoder_decompress_stream(Decoder, Data) of 44 | ok -> 45 | {ok, brotli_nif:decoder_take_output(Decoder)}; 46 | more -> 47 | {more, brotli_nif:decoder_take_output(Decoder)}; 48 | Other -> 49 | Other 50 | end. 51 | 52 | is_finished(Decoder) -> 53 | brotli_nif:decoder_is_finished(Decoder). 54 | 55 | is_used(Decoder) -> 56 | brotli_nif:decoder_is_used(Decoder). 57 | -------------------------------------------------------------------------------- /src/brotli_encoder.erl: -------------------------------------------------------------------------------- 1 | %%% Copyright (c) 2021 Łukasz Niemier 2 | %%% All rights reserved. 3 | %%% 4 | %%% Redistribution and use in source and binary forms, with or without 5 | %%% modification, are permitted provided that the following conditions 6 | %%% are met: 7 | %%% 8 | %%% 1. Redistributions of source code must retain the above copyright 9 | %%% notice, this list of conditions and the following disclaimer. 10 | %%% 2. Redistributions in binary form must reproduce the above copyright 11 | %%% notice, this list of conditions and the following disclaimer in 12 | %%% the documentation and/or other materials provided with the 13 | %%% distribution. 14 | %%% 15 | %%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | %%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | %%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 | %%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 | %%% COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | %%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 | %%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | %%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | %%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 | %%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 25 | %%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | %%% POSSIBILITY OF SUCH DAMAGE. 27 | 28 | -module(brotli_encoder). 29 | 30 | -export([new/0, new/1, set_opts/2, append/2, finish/2, finish/1]). 31 | -export([is_finished/1]). 32 | 33 | -export_type([t/0, mode/0, options/0, ndirect/0]). 34 | 35 | -opaque t() :: reference(). 36 | 37 | -type mode() :: generic | text | font. 38 | -type ndirect() :: non_neg_integer(). 39 | -type options() :: #{ 40 | mode => mode(), 41 | quality => 0..11, 42 | window => 10..30, 43 | block_size => 16..24, 44 | literal_context_modeling => boolean(), 45 | size_hint => non_neg_integer(), 46 | large_window => boolean(), 47 | npostfix => 0..3, 48 | ndirect => ndirect(), 49 | stream_offset => non_neg_integer() 50 | }. 51 | 52 | -spec new() -> t(). 53 | new() -> 54 | brotli_nif:encoder_create(). 55 | 56 | -spec new(Opts :: options()) -> t(). 57 | new(Opts) when is_map(Opts) -> 58 | Encoder = new(), 59 | set_opts(Encoder, Opts), 60 | Encoder. 61 | 62 | -spec set_opts(Encoder :: t(), Opts :: options()) -> ok. 63 | set_opts(Encoder, Opts) when is_map(Opts) -> 64 | maps:fold( 65 | fun(Key, Value, _Acc) -> 66 | true = brotli_nif:encoder_set_parameter(Encoder, Key, Value) 67 | end, 68 | [], 69 | Opts 70 | ), 71 | ok. 72 | 73 | -spec append(Encoder :: t(), Data :: iodata()) -> {ok, iodata()} | error. 74 | append(Encoder, Data) -> 75 | case brotli_nif:encoder_compress_stream(Encoder, process, Data) of 76 | true -> 77 | {ok, extract_output(Encoder)}; 78 | false -> 79 | error 80 | end. 81 | 82 | -spec finish(Encoder :: t()) -> {ok, iodata()} | error. 83 | finish(Encoder) -> 84 | finish(Encoder, []). 85 | 86 | -spec finish(Encoder :: t(), Data :: iodata()) -> {ok, iodata()} | error. 87 | finish(Encoder, Data) -> 88 | case brotli_nif:encoder_compress_stream(Encoder, finish, Data) of 89 | true -> 90 | {ok, extract_output(Encoder)}; 91 | false -> 92 | error 93 | end. 94 | 95 | extract_output(Encoder) -> 96 | case brotli_nif:encoder_has_more_output(Encoder) of 97 | false -> []; 98 | true -> [brotli_nif:encoder_take_output(Encoder) | extract_output(Encoder)] 99 | end. 100 | 101 | -spec is_finished(Encoder :: t()) -> boolean(). 102 | is_finished(Encoder) -> 103 | brotli_nif:encoder_is_finished(Encoder). 104 | -------------------------------------------------------------------------------- /src/brotli_nif.erl: -------------------------------------------------------------------------------- 1 | %%% Copyright (c) 2016 Jihyun Yu 2 | %%% Copyright (c) 2021 Łukasz Niemier 3 | %%% All rights reserved. 4 | %%% 5 | %%% Redistribution and use in source and binary forms, with or without 6 | %%% modification, are permitted provided that the following conditions 7 | %%% are met: 8 | %%% 9 | %%% 1. Redistributions of source code must retain the above copyright 10 | %%% notice, this list of conditions and the following disclaimer. 11 | %%% 2. Redistributions in binary form must reproduce the above copyright 12 | %%% notice, this list of conditions and the following disclaimer in 13 | %%% the documentation and/or other materials provided with the 14 | %%% distribution. 15 | %%% 16 | %%% THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | %%% "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | %%% LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 | %%% FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 20 | %%% COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | %%% INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | %%% BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | %%% LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | %%% CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 | %%% LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 26 | %%% ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | %%% POSSIBILITY OF SUCH DAMAGE. 28 | 29 | %% @private 30 | 31 | -module(brotli_nif). 32 | 33 | -on_load(init/0). 34 | 35 | -define(APPNAME, brotli). 36 | -define(LIBNAME, brotli). 37 | 38 | -export([ 39 | encoder_create/0, 40 | encoder_set_parameter/3, 41 | encoder_compress_stream/3, 42 | encoder_has_more_output/1, 43 | encoder_is_finished/1, 44 | encoder_take_output/1 45 | ]). 46 | -export([ 47 | decoder_create/0, 48 | decoder_decompress_stream/2, 49 | decoder_has_more_output/1, 50 | decoder_is_finished/1, 51 | decoder_is_used/1, 52 | decoder_take_output/1, 53 | decoder_error_description/1 54 | ]). 55 | -export([max_compressed_size/1, version/0]). 56 | 57 | %%% 58 | init() -> 59 | SoName = 60 | case code:priv_dir(?APPNAME) of 61 | {error, bad_name} -> 62 | case filelib:is_dir(filename:join(["..", priv])) of 63 | true -> 64 | filename:join(["..", priv, ?LIBNAME]); 65 | _ -> 66 | filename:join([priv, ?LIBNAME]) 67 | end; 68 | Dir -> 69 | filename:join(Dir, ?LIBNAME) 70 | end, 71 | erlang:load_nif(SoName, 0). 72 | 73 | %%% Exported from brotli_nif.c. 74 | 75 | encoder_create() -> 76 | erlang:nif_error(not_loaded). 77 | 78 | encoder_set_parameter(_Encoder, _Param, _Value) -> 79 | erlang:nif_error(not_loaded). 80 | 81 | encoder_compress_stream(_Encoder, _Op, _Value) -> 82 | erlang:nif_error(not_loaded). 83 | 84 | encoder_has_more_output(_Encoder) -> 85 | erlang:nif_error(not_loaded). 86 | 87 | encoder_is_finished(_Encoder) -> 88 | erlang:nif_error(not_loaded). 89 | 90 | encoder_take_output(_Encoder) -> 91 | erlang:nif_error(not_loaded). 92 | 93 | decoder_create() -> 94 | erlang:nif_error(not_loaded). 95 | 96 | decoder_decompress_stream(_Encoder, _Value) -> 97 | erlang:nif_error(not_loaded). 98 | 99 | decoder_has_more_output(_Encoder) -> 100 | erlang:nif_error(not_loaded). 101 | 102 | decoder_is_finished(_Encoder) -> 103 | erlang:nif_error(not_loaded). 104 | 105 | decoder_is_used(_Encoder) -> 106 | erlang:nif_error(not_loaded). 107 | 108 | decoder_take_output(_Encoder) -> 109 | erlang:nif_error(not_loaded). 110 | 111 | decoder_error_description(_Encoder) -> 112 | erlang:nif_error(not_loaded). 113 | 114 | max_compressed_size(_Size) -> 115 | erlang:nif_error(not_loaded). 116 | 117 | version() -> 118 | erlang:nif_error(not_loaded). 119 | -------------------------------------------------------------------------------- /test/brotli_decoder_test.erl: -------------------------------------------------------------------------------- 1 | -module(brotli_decoder_test). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | used_on_any_input_test() -> 6 | Decoder = brotli_decoder:new(), 7 | brotli_decoder:stream(Decoder, <<11>>), 8 | ?assert(brotli_decoder:is_used(Decoder)). 9 | 10 | not_used_on_empty_input_test() -> 11 | Decoder = brotli_decoder:new(), 12 | brotli_decoder:stream(Decoder, <<>>), 13 | ?assertNot(brotli_decoder:is_used(Decoder)). 14 | 15 | finished_on_full_input_test() -> 16 | {ok, Encoded} = brotli:encode("test"), 17 | Decoder = brotli_decoder:new(), 18 | brotli_decoder:stream(Decoder, Encoded), 19 | ?assert(brotli_decoder:is_finished(Decoder)). 20 | 21 | not_finished_on_partial_input_test() -> 22 | {ok, <>} = brotli:encode("test"), 23 | Decoder = brotli_decoder:new(), 24 | brotli_decoder:stream(Decoder, <>), 25 | ?assertNot(brotli_decoder:is_finished(Decoder)). 26 | -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(brotli_encoder_SUITE). 2 | 3 | -compile(export_all). 4 | 5 | -include_lib("stdlib/include/assert.hrl"). 6 | -include_lib("common_test/include/ct.hrl"). 7 | 8 | all() -> [fixture_equal, large_file, huge_file]. 9 | 10 | fixture_equal(Config) -> 11 | DataDir = ?config(data_dir, Config), 12 | InDir = filename:join(DataDir, "in"), 13 | OutDir = filename:join(DataDir, "out"), 14 | {ok, InFiles} = file:list_dir(InDir), 15 | 16 | Test = fun(FileName) -> 17 | InFile = filename:join(InDir, FileName), 18 | OutFile = filename:join(OutDir, FileName), 19 | ct:log("~s = encode(~s)~n", [OutFile, InFile]), 20 | {ok, In} = file:read_file(InFile), 21 | {ok, Out} = file:read_file(OutFile), 22 | {ok, Encoded} = brotli:encode(In), 23 | ?assertEqual(Out, iolist_to_binary(Encoded)) 24 | end, 25 | 26 | [Test(Name) || Name <- InFiles]. 27 | 28 | large_file(Config) -> 29 | DataDir = ?config(data_dir, Config), 30 | InFile = filename:join(DataDir, "large"), 31 | Sha256Sum = <<"24AF7F424EF5433FA9C1BD741B041E8D718162E42BE429858BA042630AB7D0F2">>, 32 | Encoder = brotli_encoder:new(), 33 | Hasher = crypto:hash_init(sha256), 34 | 35 | {ok, File} = file:open(InFile, [read, raw, binary]), 36 | 37 | Hash = compress_and_hash(File, Encoder, Hasher), 38 | 39 | ?assertEqual(Sha256Sum, to_hex(Hash)). 40 | 41 | huge_file(Config) -> 42 | DataDir = ?config(data_dir, Config), 43 | InFile = filename:join(DataDir, "huge"), 44 | Sha256Sum = <<"E7FDC0419AC432ED2CE0ABB2E9EED97E05E54CD2930B1187FB02483AA3DC7D7B">>, 45 | Encoder = brotli_encoder:new(#{ 46 | quality => 11, 47 | window => 24 48 | }), 49 | Hasher = crypto:hash_init(sha256), 50 | 51 | {ok, File} = file:open(InFile, [read, raw, binary]), 52 | 53 | Hash = compress_and_hash(File, Encoder, Hasher), 54 | 55 | ?assertEqual(Sha256Sum, to_hex(Hash)). 56 | 57 | compress_and_hash(File, Encoder, Hasher) -> 58 | case file:read(File, 1024) of 59 | {ok, Data} -> 60 | {ok, Compressed} = brotli_encoder:append(Encoder, Data), 61 | NewHasher = crypto:hash_update(Hasher, Compressed), 62 | compress_and_hash(File, Encoder, NewHasher); 63 | eof -> 64 | {ok, Compressed} = brotli_encoder:finish(Encoder), 65 | NewHasher = crypto:hash_update(Hasher, Compressed), 66 | crypto:hash_final(NewHasher) 67 | end. 68 | 69 | to_hex(Bin) -> 70 | <<<> || <> <= Bin, Y <- integer_to_list(X, 16)>>. 71 | -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/huge: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/huge -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/huge.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/huge.br -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/in/random_1024: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/in/random_1024 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/in/random_16: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/in/random_16 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/in/random_512: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/in/random_512 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/in/simple: -------------------------------------------------------------------------------- 1 | example file 2 | -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/large.br: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/large.br -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/out/random_1024: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/out/random_1024 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/out/random_16: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/out/random_16 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/out/random_512: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/out/random_512 -------------------------------------------------------------------------------- /test/brotli_encoder_SUITE_data/out/simple: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjh0502/erl-brotli/48e707f9a5bbd79042bcb46e61c8b5d1dd01f0d6/test/brotli_encoder_SUITE_data/out/simple -------------------------------------------------------------------------------- /test/brotli_test.erl: -------------------------------------------------------------------------------- 1 | -module(brotli_test). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | encode_simple_test_() -> 6 | Input = <<"hello">>, 7 | Compressed = <<11, 2, 128, 104, 101, 108, 108, 111, 3>>, 8 | [ 9 | ?_assertEqual({ok, Compressed}, brotli:encode(Input)), 10 | ?_assertEqual({ok, Compressed}, brotli:encode(Input, #{})), 11 | ?_assertEqual({ok, Compressed}, brotli:encode(Input, #{mode => generic})), 12 | ?_assertEqual({ok, Compressed}, brotli:encode(Input, #{mode => text})), 13 | ?_assertEqual({ok, Compressed}, brotli:encode(Input, #{mode => font})) 14 | ] ++ 15 | lists:map( 16 | fun(Level) -> 17 | ?_assertEqual({ok, Compressed}, brotli:encode(<<"hello">>, #{quality => Level})) 18 | end, 19 | lists:seq(0, 11) 20 | ). 21 | 22 | encode_window_test_() -> 23 | Input = <<"hello">>, 24 | Fun = fun(Window) -> 25 | {ok, Encoded} = brotli:encode(Input, #{window => Window}), 26 | {ok, Decoded} = brotli:decode(Encoded), 27 | ?_assertEqual(Input, Decoded) 28 | end, 29 | lists:map(Fun, lists:seq(10, 24)). 30 | 31 | encode_string_test() -> 32 | ?assertEqual(brotli:encode(<<"hello">>), brotli:encode("hello")). 33 | 34 | encode_iodata_test() -> 35 | ?assertEqual(brotli:encode(<<"hello">>), brotli:encode(["he", $l, <<"lo">>])). 36 | 37 | decode_simple_test() -> 38 | Input = <<"hello">>, 39 | Compressed = <<11, 2, 128, 104, 101, 108, 108, 111, 3>>, 40 | ?assertEqual({ok, Input}, brotli:decode(Compressed)). 41 | 42 | version_is_3ary_tuple_test() -> 43 | {Major, Minor, Patch} = brotli:version(), 44 | ?assert(is_integer(Major)), 45 | ?assert(is_integer(Minor)), 46 | ?assert(is_integer(Patch)). 47 | -------------------------------------------------------------------------------- /test/prop_brotli.erl: -------------------------------------------------------------------------------- 1 | %%% This file is part of the brotli distribution (https://github.com/hauleth/erl-brotli). 2 | %%% Copyright (c) 2021 Łukasz Niemier . 3 | %%% 4 | %%% This program is free software: you can redistribute it and/or modify 5 | %%% it under the terms of the GNU General Public License as published by 6 | %%% the Free Software Foundation, version 3. 7 | %%% 8 | %%% This program is distributed in the hope that it will be useful, but 9 | %%% WITHOUT ANY WARRANTY; without even the implied warranty of 10 | %%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | %%% General Public License for more details. 12 | %%% 13 | %%% You should have received a copy of the GNU General Public License 14 | %%% along with this program. If not, see . 15 | 16 | -module(prop_brotli). 17 | 18 | -include_lib("proper/include/proper.hrl"). 19 | -export([tool_encode/1]). 20 | 21 | prop_encoded_can_be_decoded() -> 22 | ?FORALL(Data, binary(), begin 23 | {ok, Encoded} = brotli:encode(Data), 24 | {ok, Decoded} = brotli:decode(Encoded), 25 | Data =:= Decoded 26 | end). 27 | 28 | prop_encoded_data_can_be_decoded_partially() -> 29 | ?FORALL(Data, binary(), begin 30 | {ok, Encoded} = brotli:encode(Data), 31 | Decoder = brotli_decoder:new(), 32 | Decoded = [element(2, brotli_decoder:stream(Decoder, <>)) || <> <= Encoded], 33 | Data =:= iolist_to_binary(Decoded) 34 | end). 35 | 36 | prop_encoding_all_at_once_and_byte_by_byte_is_equivalent() -> 37 | ?FORALL(Data, binary(), begin 38 | Encoder = brotli_encoder:new(), 39 | Encoded = [element(2, brotli_encoder:append(Encoder, <>)) || <> <= Data], 40 | {ok, Last} = brotli_encoder:finish(Encoder), 41 | brotli:encode(Data) =:= {ok, iolist_to_binary([Encoded, Last])} 42 | end). 43 | 44 | prop_encoded_is_not_bigger_than_max_compressed_size() -> 45 | ?FORALL(Data, binary(), begin 46 | {ok, Encoded} = brotli:encode(Data), 47 | byte_size(Encoded) =< brotli:max_compressed_size(byte_size(Data)) 48 | end). 49 | 50 | prop_encoded_is_the_same_as_from_tool() -> 51 | ?FORALL(Data, binary(), begin 52 | {ok, EncodedErl} = brotli:encode( 53 | Data, 54 | #{window => 22} 55 | ), 56 | {ok, EncodedTool} = tool_encode(Data), 57 | EncodedErl =:= EncodedTool 58 | end). 59 | 60 | prop_can_decode_data_encoded_by_tool() -> 61 | ?FORALL(Data, binary(), begin 62 | {ok, Encoded} = tool_encode(Data), 63 | {ok, Decoded} = brotli:decode(Encoded), 64 | Decoded =:= Data 65 | end). 66 | 67 | tool_encode(Data) -> 68 | Hash = to_hex(erlang:md5(Data)), 69 | TmpPath = filename:join(temp_dir(), "erl-brotli"), 70 | file:make_dir(TmpPath), 71 | Path = filename:join(TmpPath, Hash), 72 | ok = file:write_file(Path, Data, [binary, raw, write]), 73 | PrivDir = code:priv_dir(brotli), 74 | Tool = filename:join(PrivDir, "brotli"), 75 | Port = open_port({spawn_executable, Tool}, [ 76 | {args, ["-cfw", "22", "--", Path]}, 77 | exit_status, 78 | use_stdio, 79 | binary 80 | ]), 81 | {Result, 0} = receive_all(Port, []), 82 | {ok, Result}. 83 | 84 | temp_dir() -> 85 | Envs = ["TEMPDIR", "TMPDIR", "TEMP", "TMP"], 86 | find_value(Envs). 87 | 88 | find_value([]) -> 89 | "/tmp"; 90 | find_value([Env | Rest]) -> 91 | case os:getenv(Env) of 92 | false -> find_value(Rest); 93 | Path -> Path 94 | end. 95 | 96 | to_hex(Bin) -> 97 | <<<> || <> <= Bin, Y <- integer_to_list(X, 16)>>. 98 | 99 | receive_all(Port, Acc) -> 100 | receive 101 | {Port, {data, Data}} -> receive_all(Port, [Data | Acc]); 102 | {Port, {exit_status, Status}} -> {iolist_to_binary(lists:reverse(Acc)), Status} 103 | end. 104 | --------------------------------------------------------------------------------