├── .editorconfig ├── .gitattributes ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING ├── FUNDING.yml ├── ISSUE_TEMPLATE ├── PULL_REQUEST_TEMPLATE └── workflows │ └── test.yml ├── .gitignore ├── .vscode ├── extensions.json └── settings.json ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bin └── ttf2woff2.js ├── binding.gyp ├── csrc ├── addon.cc ├── enc │ ├── backward_references.cc │ ├── backward_references.h │ ├── bit_cost.h │ ├── block_splitter.cc │ ├── block_splitter.h │ ├── brotli_bit_stream.cc │ ├── brotli_bit_stream.h │ ├── cluster.h │ ├── command.h │ ├── context.h │ ├── dictionary.h │ ├── dictionary_hash.h │ ├── encode.cc │ ├── encode.h │ ├── encode_parallel.cc │ ├── encode_parallel.h │ ├── entropy_encode.cc │ ├── entropy_encode.h │ ├── fast_log.h │ ├── find_match_length.h │ ├── hash.h │ ├── histogram.cc │ ├── histogram.h │ ├── literal_cost.cc │ ├── literal_cost.h │ ├── metablock.cc │ ├── metablock.h │ ├── port.h │ ├── prefix.h │ ├── ringbuffer.h │ ├── static_dict.h │ ├── streams.cc │ ├── streams.h │ ├── streams.h.gch │ ├── transform.h │ └── write_bits.h ├── fallback.cc └── woff2 │ ├── buffer.h │ ├── font.cc │ ├── font.h │ ├── glyph.cc │ ├── glyph.h │ ├── normalize.cc │ ├── normalize.h │ ├── port.h │ ├── round.h │ ├── store_bytes.h │ ├── table_tags.cc │ ├── table_tags.h │ ├── transform.cc │ ├── transform.h │ ├── variable_length.cc │ ├── variable_length.h │ ├── woff2_common.cc │ ├── woff2_common.h │ ├── woff2_dec.cc │ ├── woff2_dec.h │ ├── woff2_enc.cc │ └── woff2_enc.h ├── eslint.config.js ├── fixtures ├── iconsfont.ttf └── iconsfont.woff2 ├── jssrc ├── index.js ├── post.js ├── ttf2woff2.cjs └── ttf2woff2.wasm ├── package-lock.json ├── package.json ├── src ├── cli.test.ts ├── index.ts └── tests.test.ts └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by a `metapak` 2 | # module. Do not change it elsewhere, changes would 3 | # be overridden. 4 | 5 | # EditorConfig is awesome: http://EditorConfig.org 6 | 7 | # top-most EditorConfig file 8 | root = true 9 | 10 | # Unix-style newlines with a newline ending every file 11 | [*] 12 | end_of_line = lf 13 | insert_final_newline = true 14 | 15 | # Matches multiple files with brace expansion notation 16 | # Set default charset 17 | # 2 space indentation 18 | [*.{js,css}] 19 | charset = utf-8 20 | indent_style = space 21 | trim_trailing_whitespace = true 22 | indent_size = 2 23 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Enforce Unix newlines 2 | * text=auto eol=lf 3 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Be kind, except if I behave like an asshole, if so, tell me by linking to this 4 | file. 5 | 6 | I try hard to document and automate things so that you cannot create noises 7 | without really willing to do so. 8 | 9 | This is why I'll just delete issues/comments making be sad. 10 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | Contributing to this project requires you to be 2 | a gentleman. 3 | 4 | By contributing you must agree with publishing your 5 | changes into the same license that apply to the current code. 6 | 7 | You will find the license in the LICENSE file at 8 | the root of this repository. 9 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [nfroidure] 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE: -------------------------------------------------------------------------------- 1 | ## Issue 2 | 16 | 17 | I'm a gentledev I: 18 | - [ ] fully read the README recently 19 | - [ ] searched for existing issues 20 | - [ ] checked I'm up to date with the latest version of the project 21 | 22 | ### Expected behavior 23 | 24 | ### Actual behavior 25 | 26 | ### Steps to reproduce the behavior 27 | 28 | ### Debugging informations 29 | - `node -v` result: 30 | ``` 31 | 32 | ``` 33 | 34 | - `npm -v` result: 35 | ``` 36 | 37 | ``` 38 | If the result is lower than 20.11.1, there is 39 | poor chances I even have a look to it. Please, 40 | use the last [NodeJS LTS version](https://nodejs.org/en/). 41 | 42 | ## Feature request 43 | 54 | 55 | ### Feature description 56 | 57 | ### Use cases 58 | 59 | - [ ] I will/did implement the feature 60 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | 10 | 11 | Fixes # 12 | 13 | ### Proposed changes 14 | - 15 | - 16 | 17 | 18 | 19 | ### Code quality 20 | - [ ] I made some tests for my changes 21 | - [ ] I added my name in the 22 | [contributors](https://docs.npmjs.com/files/package.json#people-fields-author-contributors) 23 | field of the `package.json` file. Beware to use the same format than for the author field 24 | for the entries so that you'll get a mention in the `README.md` with a link to your website. 25 | 26 | ### License 27 | To get your contribution merged, you must check the following. 28 | 29 | - [ ] I read the project license in the LICENSE file 30 | - [ ] I agree with publishing under this project license 31 | 32 | 46 | ### Join 47 | - [ ] I wish to join the core team 48 | - [ ] I agree that with great powers comes responsibilities 49 | - [ ] I'm a nice person 50 | 51 | My NPM username: 52 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Run tests 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | node: [20, 22, 23] 8 | os: [macos-latest, ubuntu-latest] 9 | runs-on: ${{ matrix.os }} 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: actions/setup-node@v4 13 | with: 14 | node-version: ${{ matrix.node }} 15 | cache: 'npm' 16 | - run: npm i -g npm@latest 17 | - run: npm ci 18 | - run: npm test 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by a `metapak` 2 | # module. Do not change it elsewhere, changes would 3 | # be overridden. 4 | 5 | # Created by https://www.gitignore.io/api/osx,node,linux 6 | 7 | ### Linux ### 8 | *~ 9 | 10 | # temporary files which can be created if a process still has a handle open of a deleted file 11 | .fuse_hidden* 12 | 13 | # KDE directory preferences 14 | .directory 15 | 16 | # Linux trash folder which might appear on any partition or disk 17 | .Trash-* 18 | 19 | # .nfs files are created when an open file is removed but is still being accessed 20 | .nfs* 21 | 22 | ### Node ### 23 | # Logs 24 | logs 25 | *.log 26 | npm-debug.log* 27 | yarn-debug.log* 28 | yarn-error.log* 29 | 30 | # Runtime data 31 | pids 32 | *.pid 33 | *.seed 34 | *.pid.lock 35 | 36 | # Directory for instrumented libs generated by jscoverage/JSCover 37 | lib-cov 38 | 39 | # Coverage directory used by tools like istanbul 40 | coverage 41 | 42 | # nyc test coverage 43 | .nyc_output 44 | 45 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 46 | .grunt 47 | 48 | # Bower dependency directory (https://bower.io/) 49 | bower_components 50 | 51 | # node-waf configuration 52 | .lock-wscript 53 | 54 | # Compiled binary addons (https://nodejs.org/api/addons.html) 55 | build/Release 56 | 57 | # Dependency directories 58 | node_modules/ 59 | jspm_packages/ 60 | 61 | # TypeScript v1 declaration files 62 | typings/ 63 | 64 | # Optional npm cache directory 65 | .npm 66 | 67 | # Optional eslint cache 68 | .eslintcache 69 | 70 | # Optional REPL history 71 | .node_repl_history 72 | 73 | # Output of 'npm pack' 74 | *.tgz 75 | 76 | # Yarn Integrity file 77 | .yarn-integrity 78 | 79 | # dotenv environment variables file 80 | .env 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | 85 | # next.js build output 86 | .next 87 | 88 | # nuxt.js build output 89 | .nuxt 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless 96 | 97 | ### OSX ### 98 | # General 99 | .DS_Store 100 | .AppleDouble 101 | .LSOverride 102 | 103 | # Icon must end with two \r 104 | Icon 105 | 106 | # Thumbnails 107 | ._* 108 | 109 | # Files that might appear in the root of a volume 110 | .DocumentRevisions-V100 111 | .fseventsd 112 | .Spotlight-V100 113 | .TemporaryItems 114 | .Trashes 115 | .VolumeIcon.icns 116 | .com.apple.timemachine.donotpresent 117 | 118 | # Directories potentially created on remote AFP share 119 | .AppleDB 120 | .AppleDesktop 121 | Network Trash Folder 122 | Temporary Items 123 | .apdisk 124 | 125 | 126 | # End of https://www.gitignore.io/api/osx,node,linux 127 | 128 | # Coveralls key 129 | .coveralls.yml 130 | 131 | # Project custom ignored file 132 | dist 133 | build 134 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "esbenp.prettier-vscode", 5 | "gruntfuggly.todo-tree" 6 | ] 7 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules/typescript/lib" 3 | } 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # [7.0.0](https://github.com/nfroidure/ttf2woff2/compare/v6.0.1...v7.0.0) (2025-05-22) 2 | 3 | 4 | ### Bug Fixes 5 | 6 | * **ci:** add Node 22 and 23 to test matrix ([d335554](https://github.com/nfroidure/ttf2woff2/commit/d335554ad4679c4d508cb69141dba67de2bbcce5)) 7 | * **core:** fix TS errors ([7535aa1](https://github.com/nfroidure/ttf2woff2/commit/7535aa1288f9cb24be3e7026926b2b5973cbde20)) 8 | 9 | 10 | 11 | ## [6.0.1](https://github.com/nfroidure/ttf2woff2/compare/v6.0.0...v6.0.1) (2024-07-20) 12 | 13 | 14 | ### Bug Fixes 15 | 16 | * **ci:** fix CI runs ([4650b51](https://github.com/nfroidure/ttf2woff2/commit/4650b511a725a7f18128a0b3a691146cb40387d0)) 17 | * **core:** fix native bindings import ([420930f](https://github.com/nfroidure/ttf2woff2/commit/420930f8f2776904753d197a1f93263b1b68d66f)) 18 | 19 | 20 | 21 | # [6.0.0](https://github.com/nfroidure/ttf2woff2/compare/v5.0.0...v6.0.0) (2024-07-20) 22 | 23 | 24 | ### chore 25 | 26 | * **dependencies:** update dependencies ([4319d41](https://github.com/nfroidure/ttf2woff2/commit/4319d41f552e563a1163f0a4a3664d68895871da)) 27 | 28 | 29 | ### Features 30 | 31 | * **core:** better choice between builds ([ff54055](https://github.com/nfroidure/ttf2woff2/commit/ff540555e6a55fcd4e7c69a8702167a1f9abfe1e)), closes [#22](https://github.com/nfroidure/ttf2woff2/issues/22) [#46](https://github.com/nfroidure/ttf2woff2/issues/46) 32 | 33 | 34 | ### BREAKING CHANGES 35 | 36 | * **dependencies:** Requires Node20+, use ESM, provide TypeScript types 37 | 38 | 39 | 40 | ## [5.0.0](https://github.com/nfroidure/ttf2woff2/compare/v4.0.5...v5.0.0) (2022-12-06) 41 | 42 | ### Chore 43 | 44 | - **bundle:** Update Emscripten builds 45 | - **ci:** Add Node 19 to test matrix 46 | - **dependencies:** Update dependencies (jest) and remove unsued dependencies 47 | (mocha, mocha-lcov-reporter) 48 | 49 | ### BREAKING CHANGES 50 | 51 | - **dependencies:** Drop Node 12 support 52 | 53 | ## [4.0.5](https://github.com/nfroidure/ttf2woff2/compare/v4.0.4...v4.0.5) (2022-04-23) 54 | 55 | ### Chore 56 | 57 | - **bundle:** Update Emscripten builds 58 | - **ci:** Add Node 18 to test matrix 59 | - **dependencies:** Update dependencies (node-gyp, eslint, prettier) and remove 60 | unsued dependencies (babel-eslint, @babel/register) 61 | - **readme:** Add PowerShell use case 62 | ([#72](https://github.com/nfroidure/ttf2woff2/pull/72)) 63 | 64 | ## [4.0.4](https://github.com/nfroidure/ttf2woff2/compare/v4.0.3...v4.0.4) (2021-05-29) 65 | 66 | ### Chore 67 | 68 | - **dependencies:** Update jest, mocha, node-gyp 69 | ([#71](https://github.com/nfroidure/ttf2woff2/pull/71)) 70 | 71 | ## [4.0.3](https://github.com/nfroidure/ttf2woff2/compare/v4.0.2...v4.0.3) (2021-05-29) 72 | 73 | ### Bug Fixes 74 | 75 | - **build:** support Node 16 76 | ([b8a8986](https://github.com/nfroidure/ttf2woff2/commit/b8a898636b5d66e55bc1344caa15a03f49c46b59)) 77 | 78 | ## [4.0.2](https://github.com/nfroidure/ttf2woff2/compare/v4.0.1...v4.0.2) (2021-03-15) 79 | 80 | ### Bug Fixes 81 | 82 | - **docs:** fix the readme 83 | ([acb62c5](https://github.com/nfroidure/ttf2woff2/commit/acb62c579974b510a4d824ee5a6fed74923f3935)) 84 | 85 | ## [4.0.1](https://github.com/nfroidure/ttf2woff2/compare/v4.0.0...v4.0.1) (2020-12-22) 86 | 87 | ### Bug Fixes 88 | 89 | - **bundle:** add necessary files to the bundle 90 | ([fc9bb76](https://github.com/nfroidure/ttf2woff2/commit/fc9bb76fa8a51b55a81aec5a7a8efb72722860cc)) 91 | 92 | ## [4.0.0](https://github.com/nfroidure/ttf2woff2/compare/v3.0.0...v4.0.0) (2020-12-22) 93 | 94 | ### Chore 95 | 96 | - **dependencies:** update dependencies 97 | ([0657e8d](https://github.com/nfroidure/ttf2woff2/commit/0657e8df60fa5b984406cf0d33b86c64c759a2c8)) 98 | 99 | ### BREAKING CHANGES 100 | 101 | - **dependencies:** Requiring Node12 to work 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2017 Nicolas Froidure 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the “Software”), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [//]: # ( ) 2 | [//]: # (This file is automatically generated by a `metapak`) 3 | [//]: # (module. Do not change it except between the) 4 | [//]: # (`content:start/end` flags, your changes would) 5 | [//]: # (be overridden.) 6 | [//]: # ( ) 7 | # ttf2woff2 8 | > Convert TTF files to WOFF2 ones. 9 | 10 | [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/nfroidure/ttf2woff2/blob/main/LICENSE) 11 | 12 | 13 | [//]: # (::contents:start) 14 | 15 | This is a NodeJS wrapper for the Google [WOFF2](https://github.com/google/woff2) 16 | project. If the C++ wrapper compilation fail, it 17 | [fallbacks to an Emscripten build](http://insertafter.com/en/blog/native-node-module.html). 18 | 19 | ## Usage 20 | 21 | ### CLI 22 | 23 | Install `ttf2woff2` globally, then: 24 | 25 | ```sh 26 | cat font.ttf | ttf2woff2 > font.woff2 27 | ``` 28 | 29 | On Windows without `cat`, use (in PowerShell): 30 | 31 | ```pwsh 32 | Start-Process -NoNewWindow -Wait ttf2woff2.cmd -RedirectStandardInput font.ttf -RedirectStandardOutput font.woff2 33 | # OR 34 | start-process -nnw -wait ttf2woff2.cmd -rsi font.ttf -rso font.woff2 35 | ``` 36 | 37 | ### API 38 | 39 | ```js 40 | import { readFile, writeFile } from 'node:fs/promises'; 41 | import ttf2woff2 from 'ttf2woff2'; 42 | 43 | const input = await readFile('font.ttf'); 44 | 45 | await writeFile('font.woff2', ttf2woff2(input)); 46 | ``` 47 | 48 | ## Development 49 | 50 | To build the binary, clone the repository and run the following: 51 | 52 | ``` 53 | ## Setup (works for Ubuntu/Linux, may be different on other OS) 54 | apt-get install make g++ 55 | 56 | ## Actual build 57 | npm i 58 | npm run configure 59 | npm run make 60 | ``` 61 | 62 | To build the Emscripten fallback, install [Emscripten](https://emscripten.org/) and run: 63 | 64 | ``` 65 | npm run emcc 66 | ``` 67 | 68 | Finally the build can be tested: 69 | ```sh 70 | npm run build 71 | npm t 72 | ``` 73 | 74 | ## Contributing 75 | 76 | Feel free to push your code if you agree with publishing under the MIT license. 77 | 78 | [//]: # (::contents:end) 79 | 80 | # Authors 81 | - [Nicolas Froidure](https://insertafter.com/en/index.html) 82 | - [Anders Kaseorg](mailto:andersk@mit.edu) 83 | 84 | # License 85 | [MIT](https://github.com/nfroidure/ttf2woff2/blob/main/LICENSE) 86 | -------------------------------------------------------------------------------- /bin/ttf2woff2.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | import { 4 | BufferStream 5 | } from 'bufferstreams'; 6 | import ttf2woff2 from '../dist/index.js'; 7 | 8 | process.stdin 9 | .pipe( 10 | new BufferStream(function (err, buf, cb) { 11 | if (err) { 12 | throw err; 13 | } 14 | cb(null, ttf2woff2(buf)); 15 | }), 16 | ) 17 | .pipe(process.stdout); 18 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "addon", 5 | "sources": [ 6 | "csrc/addon.cc", 7 | 8 | "csrc/woff2/glyph.cc", 9 | "csrc/woff2/font.cc", 10 | "csrc/woff2/normalize.cc", 11 | "csrc/woff2/table_tags.cc", 12 | "csrc/woff2/transform.cc", 13 | "csrc/woff2/variable_length.cc", 14 | "csrc/woff2/woff2_common.cc", 15 | "csrc/woff2/woff2_enc.cc", 16 | 17 | "csrc/enc/backward_references.cc", 18 | "csrc/enc/block_splitter.cc", 19 | "csrc/enc/brotli_bit_stream.cc", 20 | "csrc/enc/encode.cc", 21 | "csrc/enc/encode_parallel.cc", 22 | "csrc/enc/entropy_encode.cc", 23 | "csrc/enc/histogram.cc", 24 | "csrc/enc/literal_cost.cc", 25 | "csrc/enc/metablock.cc", 26 | "csrc/enc/streams.cc" 27 | ], 28 | "include_dirs" : [ 29 | " 2 | #include 3 | #include 4 | #include 5 | #include "./woff2/woff2_enc.h" 6 | 7 | using namespace v8; 8 | 9 | NAN_METHOD(convert) { 10 | Isolate *isolate = info.GetIsolate(); 11 | Local context = isolate->GetCurrentContext(); 12 | Local inputBuffer = info[0]->ToObject(context).ToLocalChecked(); 13 | 14 | if (!node::Buffer::HasInstance(inputBuffer)) { 15 | Nan::ThrowError(Nan::TypeError("First arg should be a Buffer")); 16 | return; 17 | } 18 | 19 | size_t input_length = node::Buffer::Length(inputBuffer); 20 | char* input_data = node::Buffer::Data(inputBuffer); 21 | 22 | // Determine the maximum needed length 23 | size_t max_output_length = woff2::MaxWOFF2CompressedSize( 24 | reinterpret_cast(input_data), input_length); 25 | size_t actual_output_length = max_output_length; 26 | 27 | char* output_data = (char*) calloc(max_output_length, 1); 28 | 29 | // Create the Woff2 font 30 | if (!woff2::ConvertTTFToWOFF2( 31 | reinterpret_cast(input_data), input_length, 32 | reinterpret_cast(output_data), &actual_output_length 33 | )) { 34 | Nan::ThrowError(Nan::Error("Could not convert the given font.")); 35 | return; 36 | } 37 | 38 | // Free the unused memory 39 | output_data = (char*) realloc(output_data, actual_output_length); 40 | 41 | Nan::MaybeLocal outputBuffer = Nan::NewBuffer( 42 | output_data, 43 | actual_output_length 44 | ); 45 | 46 | info.GetReturnValue().Set(outputBuffer.ToLocalChecked()); 47 | } 48 | 49 | 50 | NAN_MODULE_INIT(Init) { 51 | Nan::Set(target, Nan::New("convert").ToLocalChecked(), 52 | Nan::GetFunction(Nan::New(convert)).ToLocalChecked()); 53 | } 54 | 55 | NODE_MODULE(addon, Init) 56 | -------------------------------------------------------------------------------- /csrc/enc/backward_references.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Function to find backward reference copies. 16 | 17 | #ifndef BROTLI_ENC_BACKWARD_REFERENCES_H_ 18 | #define BROTLI_ENC_BACKWARD_REFERENCES_H_ 19 | 20 | #include 21 | #include 22 | 23 | #include "./hash.h" 24 | #include "./command.h" 25 | 26 | namespace brotli { 27 | 28 | void CreateBackwardReferences(size_t num_bytes, 29 | size_t position, 30 | const uint8_t* ringbuffer, 31 | size_t ringbuffer_mask, 32 | const float* literal_cost, 33 | size_t literal_cost_mask, 34 | const size_t max_backward_limit, 35 | const double base_min_score, 36 | const int quality, 37 | Hashers* hashers, 38 | int hash_type, 39 | int* dist_cache, 40 | int* last_insert_len, 41 | Command* commands, 42 | int* num_commands); 43 | 44 | } // namespace brotli 45 | 46 | #endif // BROTLI_ENC_BACKWARD_REFERENCES_H_ 47 | -------------------------------------------------------------------------------- /csrc/enc/bit_cost.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions to estimate the bit cost of Huffman trees. 16 | 17 | #ifndef BROTLI_ENC_BIT_COST_H_ 18 | #define BROTLI_ENC_BIT_COST_H_ 19 | 20 | #include 21 | 22 | #include "./entropy_encode.h" 23 | #include "./fast_log.h" 24 | 25 | namespace brotli { 26 | 27 | static inline double BitsEntropy(const int *population, int size) { 28 | int sum = 0; 29 | double retval = 0; 30 | const int *population_end = population + size; 31 | int p; 32 | if (size & 1) { 33 | goto odd_number_of_elements_left; 34 | } 35 | while (population < population_end) { 36 | p = *population++; 37 | sum += p; 38 | retval -= p * FastLog2(p); 39 | odd_number_of_elements_left: 40 | p = *population++; 41 | sum += p; 42 | retval -= p * FastLog2(p); 43 | } 44 | if (sum) retval += sum * FastLog2(sum); 45 | if (retval < sum) { 46 | // At least one bit per literal is needed. 47 | retval = sum; 48 | } 49 | return retval; 50 | } 51 | 52 | static const int kHuffmanExtraBits[kCodeLengthCodes] = { 53 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 54 | }; 55 | 56 | static inline int HuffmanTreeBitCost(const int* counts, const uint8_t* depth) { 57 | int nbits = 0; 58 | for (int i = 0; i < kCodeLengthCodes; ++i) { 59 | nbits += counts[i] * (depth[i] + kHuffmanExtraBits[i]); 60 | } 61 | return nbits; 62 | } 63 | 64 | static inline int HuffmanTreeBitCost( 65 | const Histogram& histogram, 66 | const EntropyCode& entropy) { 67 | return HuffmanTreeBitCost(&histogram.data_[0], &entropy.depth_[0]); 68 | } 69 | 70 | static inline int HuffmanBitCost(const uint8_t* depth, int length) { 71 | int max_depth = 1; 72 | int histogram[kCodeLengthCodes] = { 0 }; 73 | int tail_start = 0; 74 | int prev_value = 8; 75 | // compute histogram of compacted huffman tree 76 | for (int i = 0; i < length;) { 77 | const int value = depth[i]; 78 | if (value > max_depth) { 79 | max_depth = value; 80 | } 81 | int reps = 1; 82 | for (int k = i + 1; k < length && depth[k] == value; ++k) { 83 | ++reps; 84 | } 85 | i += reps; 86 | if (i == length && value == 0) 87 | break; 88 | if (value == 0) { 89 | if (reps < 3) { 90 | histogram[0] += reps; 91 | } else { 92 | reps -= 2; 93 | while (reps > 0) { 94 | ++histogram[17]; 95 | reps >>= 3; 96 | } 97 | } 98 | } else { 99 | tail_start = i; 100 | if (value != prev_value) { 101 | ++histogram[value]; 102 | --reps; 103 | } 104 | prev_value = value; 105 | if (reps < 3) { 106 | histogram[value] += reps; 107 | } else { 108 | reps -= 2; 109 | while (reps > 0) { 110 | ++histogram[16]; 111 | reps >>= 2; 112 | } 113 | } 114 | } 115 | } 116 | 117 | // create huffman tree of huffman tree 118 | uint8_t cost[kCodeLengthCodes] = { 0 }; 119 | CreateHuffmanTree(histogram, kCodeLengthCodes, 7, cost); 120 | // account for rle extra bits 121 | cost[16] += 2; 122 | cost[17] += 3; 123 | 124 | int tree_size = 0; 125 | int bits = 18 + 2 * max_depth; // huffman tree of huffman tree cost 126 | for (int i = 0; i < kCodeLengthCodes; ++i) { 127 | bits += histogram[i] * cost[i]; // huffman tree bit cost 128 | tree_size += histogram[i]; 129 | } 130 | return bits; 131 | } 132 | 133 | template 134 | double PopulationCost(const Histogram& histogram) { 135 | if (histogram.total_count_ == 0) { 136 | return 12; 137 | } 138 | int count = 0; 139 | for (int i = 0; i < kSize && count < 5; ++i) { 140 | if (histogram.data_[i] > 0) { 141 | ++count; 142 | } 143 | } 144 | if (count == 1) { 145 | return 12; 146 | } 147 | if (count == 2) { 148 | return 20 + histogram.total_count_; 149 | } 150 | uint8_t depth[kSize] = { 0 }; 151 | CreateHuffmanTree(&histogram.data_[0], kSize, 15, depth); 152 | int bits = 0; 153 | for (int i = 0; i < kSize; ++i) { 154 | bits += histogram.data_[i] * depth[i]; 155 | } 156 | if (count == 3) { 157 | bits += 28; 158 | } else if (count == 4) { 159 | bits += 37; 160 | } else { 161 | bits += HuffmanBitCost(depth, kSize); 162 | } 163 | return bits; 164 | } 165 | 166 | } // namespace brotli 167 | 168 | #endif // BROTLI_ENC_BIT_COST_H_ 169 | -------------------------------------------------------------------------------- /csrc/enc/block_splitter.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Block split point selection utilities. 16 | 17 | #ifndef BROTLI_ENC_BLOCK_SPLITTER_H_ 18 | #define BROTLI_ENC_BLOCK_SPLITTER_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "./command.h" 27 | #include "./metablock.h" 28 | 29 | namespace brotli { 30 | 31 | struct BlockSplitIterator { 32 | explicit BlockSplitIterator(const BlockSplit& split) 33 | : split_(split), idx_(0), type_(0), length_(0) { 34 | if (!split.lengths.empty()) { 35 | length_ = split.lengths[0]; 36 | } 37 | } 38 | 39 | void Next() { 40 | if (length_ == 0) { 41 | ++idx_; 42 | type_ = split_.types[idx_]; 43 | length_ = split_.lengths[idx_]; 44 | } 45 | --length_; 46 | } 47 | 48 | const BlockSplit& split_; 49 | int idx_; 50 | int type_; 51 | int length_; 52 | }; 53 | 54 | void CopyLiteralsToByteArray(const Command* cmds, 55 | const size_t num_commands, 56 | const uint8_t* data, 57 | std::vector* literals); 58 | 59 | void SplitBlock(const Command* cmds, 60 | const size_t num_commands, 61 | const uint8_t* data, 62 | BlockSplit* literal_split, 63 | BlockSplit* insert_and_copy_split, 64 | BlockSplit* dist_split); 65 | 66 | void SplitBlockByTotalLength(const Command* all_commands, 67 | const size_t num_commands, 68 | int input_size, 69 | int target_length, 70 | std::vector >* blocks); 71 | 72 | } // namespace brotli 73 | 74 | #endif // BROTLI_ENC_BLOCK_SPLITTER_H_ 75 | -------------------------------------------------------------------------------- /csrc/enc/brotli_bit_stream.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions to convert brotli-related data structures into the 16 | // brotli bit stream. The functions here operate under 17 | // assumption that there is enough space in the storage, i.e., there are 18 | // no out-of-range checks anywhere. 19 | // 20 | // These functions do bit addressing into a byte array. The byte array 21 | // is called "storage" and the index to the bit is called storage_ix 22 | // in function arguments. 23 | 24 | #ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_ 25 | #define BROTLI_ENC_BROTLI_BIT_STREAM_H_ 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | #include "./metablock.h" 32 | 33 | namespace brotli { 34 | 35 | // All Store functions here will use a storage_ix, which is always the bit 36 | // position for the current storage. 37 | 38 | // Stores a number between 0 and 255. 39 | void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage); 40 | 41 | // Stores the compressed meta-block header. 42 | bool StoreCompressedMetaBlockHeader(bool final_block, 43 | size_t length, 44 | int* storage_ix, 45 | uint8_t* storage); 46 | 47 | // Stores the uncompressed meta-block header. 48 | bool StoreUncompressedMetaBlockHeader(size_t length, 49 | int* storage_ix, 50 | uint8_t* storage); 51 | 52 | // Stores a context map where the histogram type is always the block type. 53 | void StoreTrivialContextMap(int num_types, 54 | int context_bits, 55 | int* storage_ix, 56 | uint8_t* storage); 57 | 58 | void StoreHuffmanTreeOfHuffmanTreeToBitMask( 59 | const int num_codes, 60 | const uint8_t *code_length_bitdepth, 61 | int *storage_ix, 62 | uint8_t *storage); 63 | 64 | // Builds a Huffman tree from histogram[0:length] into depth[0:length] and 65 | // bits[0:length] and stores the encoded tree to the bit stream. 66 | void BuildAndStoreHuffmanTree(const int *histogram, 67 | const int length, 68 | uint8_t* depth, 69 | uint16_t* bits, 70 | int* storage_ix, 71 | uint8_t* storage); 72 | 73 | // Encodes the given context map to the bit stream. The number of different 74 | // histogram ids is given by num_clusters. 75 | void EncodeContextMap(const std::vector& context_map, 76 | int num_clusters, 77 | int* storage_ix, uint8_t* storage); 78 | 79 | // Data structure that stores everything that is needed to encode each block 80 | // switch command. 81 | struct BlockSplitCode { 82 | std::vector type_code; 83 | std::vector length_prefix; 84 | std::vector length_nextra; 85 | std::vector length_extra; 86 | std::vector type_depths; 87 | std::vector type_bits; 88 | std::vector length_depths; 89 | std::vector length_bits; 90 | }; 91 | 92 | // Builds a BlockSplitCode data structure from the block split given by the 93 | // vector of block types and block lengths and stores it to the bit stream. 94 | void BuildAndStoreBlockSplitCode(const std::vector& types, 95 | const std::vector& lengths, 96 | const int num_types, 97 | BlockSplitCode* code, 98 | int* storage_ix, 99 | uint8_t* storage); 100 | 101 | // Stores the block switch command with index block_ix to the bit stream. 102 | void StoreBlockSwitch(const BlockSplitCode& code, 103 | const int block_ix, 104 | int* storage_ix, 105 | uint8_t* storage); 106 | 107 | bool StoreMetaBlock(const uint8_t* input, 108 | size_t start_pos, 109 | size_t length, 110 | size_t mask, 111 | uint8_t prev_byte, 112 | uint8_t prev_byte2, 113 | bool final_block, 114 | int num_direct_distance_codes, 115 | int distance_postfix_bits, 116 | int literal_context_mode, 117 | const brotli::Command *commands, 118 | size_t n_commands, 119 | const MetaBlockSplit& mb, 120 | int *storage_ix, 121 | uint8_t *storage); 122 | 123 | // This is for storing uncompressed blocks (simple raw storage of 124 | // bytes-as-bytes). 125 | bool StoreUncompressedMetaBlock(bool final_block, 126 | const uint8_t* input, 127 | size_t position, size_t mask, 128 | size_t len, 129 | int* storage_ix, 130 | uint8_t* storage); 131 | 132 | // Stores an empty metadata meta-block and syncs to a byte boundary. 133 | void StoreSyncMetaBlock(int* storage_ix, uint8_t* storage); 134 | 135 | } // namespace brotli 136 | 137 | #endif // BROTLI_ENC_BROTLI_BIT_STREAM_H_ 138 | -------------------------------------------------------------------------------- /csrc/enc/cluster.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions for clustering similar histograms together. 16 | 17 | #ifndef BROTLI_ENC_CLUSTER_H_ 18 | #define BROTLI_ENC_CLUSTER_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "./bit_cost.h" 31 | #include "./entropy_encode.h" 32 | #include "./fast_log.h" 33 | #include "./histogram.h" 34 | 35 | namespace brotli { 36 | 37 | struct HistogramPair { 38 | int idx1; 39 | int idx2; 40 | bool valid; 41 | double cost_combo; 42 | double cost_diff; 43 | }; 44 | 45 | struct HistogramPairComparator { 46 | bool operator()(const HistogramPair& p1, const HistogramPair& p2) const { 47 | if (p1.cost_diff != p2.cost_diff) { 48 | return p1.cost_diff > p2.cost_diff; 49 | } 50 | return abs(p1.idx1 - p1.idx2) > abs(p2.idx1 - p2.idx2); 51 | } 52 | }; 53 | 54 | // Returns entropy reduction of the context map when we combine two clusters. 55 | inline double ClusterCostDiff(int size_a, int size_b) { 56 | int size_c = size_a + size_b; 57 | return size_a * FastLog2(size_a) + size_b * FastLog2(size_b) - 58 | size_c * FastLog2(size_c); 59 | } 60 | 61 | // Computes the bit cost reduction by combining out[idx1] and out[idx2] and if 62 | // it is below a threshold, stores the pair (idx1, idx2) in the *pairs heap. 63 | template 64 | void CompareAndPushToHeap(const HistogramType* out, 65 | const int* cluster_size, 66 | int idx1, int idx2, 67 | std::vector* pairs) { 68 | if (idx1 == idx2) { 69 | return; 70 | } 71 | if (idx2 < idx1) { 72 | int t = idx2; 73 | idx2 = idx1; 74 | idx1 = t; 75 | } 76 | bool store_pair = false; 77 | HistogramPair p; 78 | p.idx1 = idx1; 79 | p.idx2 = idx2; 80 | p.valid = true; 81 | p.cost_diff = 0.5 * ClusterCostDiff(cluster_size[idx1], cluster_size[idx2]); 82 | p.cost_diff -= out[idx1].bit_cost_; 83 | p.cost_diff -= out[idx2].bit_cost_; 84 | 85 | if (out[idx1].total_count_ == 0) { 86 | p.cost_combo = out[idx2].bit_cost_; 87 | store_pair = true; 88 | } else if (out[idx2].total_count_ == 0) { 89 | p.cost_combo = out[idx1].bit_cost_; 90 | store_pair = true; 91 | } else { 92 | double threshold = pairs->empty() ? 1e99 : 93 | std::max(0.0, (*pairs)[0].cost_diff); 94 | HistogramType combo = out[idx1]; 95 | combo.AddHistogram(out[idx2]); 96 | double cost_combo = PopulationCost(combo); 97 | if (cost_combo < threshold - p.cost_diff) { 98 | p.cost_combo = cost_combo; 99 | store_pair = true; 100 | } 101 | } 102 | if (store_pair) { 103 | p.cost_diff += p.cost_combo; 104 | pairs->push_back(p); 105 | std::push_heap(pairs->begin(), pairs->end(), HistogramPairComparator()); 106 | } 107 | } 108 | 109 | template 110 | void HistogramCombine(HistogramType* out, 111 | int* cluster_size, 112 | int* symbols, 113 | int symbols_size, 114 | int max_clusters) { 115 | double cost_diff_threshold = 0.0; 116 | int min_cluster_size = 1; 117 | std::set all_symbols; 118 | std::vector clusters; 119 | for (int i = 0; i < symbols_size; ++i) { 120 | if (all_symbols.find(symbols[i]) == all_symbols.end()) { 121 | all_symbols.insert(symbols[i]); 122 | clusters.push_back(symbols[i]); 123 | } 124 | } 125 | 126 | // We maintain a heap of histogram pairs, ordered by the bit cost reduction. 127 | std::vector pairs; 128 | for (int idx1 = 0; idx1 < clusters.size(); ++idx1) { 129 | for (int idx2 = idx1 + 1; idx2 < clusters.size(); ++idx2) { 130 | CompareAndPushToHeap(out, cluster_size, clusters[idx1], clusters[idx2], 131 | &pairs); 132 | } 133 | } 134 | 135 | while (clusters.size() > min_cluster_size) { 136 | if (pairs[0].cost_diff >= cost_diff_threshold) { 137 | cost_diff_threshold = 1e99; 138 | min_cluster_size = max_clusters; 139 | continue; 140 | } 141 | // Take the best pair from the top of heap. 142 | int best_idx1 = pairs[0].idx1; 143 | int best_idx2 = pairs[0].idx2; 144 | out[best_idx1].AddHistogram(out[best_idx2]); 145 | out[best_idx1].bit_cost_ = pairs[0].cost_combo; 146 | cluster_size[best_idx1] += cluster_size[best_idx2]; 147 | for (int i = 0; i < symbols_size; ++i) { 148 | if (symbols[i] == best_idx2) { 149 | symbols[i] = best_idx1; 150 | } 151 | } 152 | for (int i = 0; i + 1 < clusters.size(); ++i) { 153 | if (clusters[i] >= best_idx2) { 154 | clusters[i] = clusters[i + 1]; 155 | } 156 | } 157 | clusters.pop_back(); 158 | // Invalidate pairs intersecting the just combined best pair. 159 | for (int i = 0; i < pairs.size(); ++i) { 160 | HistogramPair& p = pairs[i]; 161 | if (p.idx1 == best_idx1 || p.idx2 == best_idx1 || 162 | p.idx1 == best_idx2 || p.idx2 == best_idx2) { 163 | p.valid = false; 164 | } 165 | } 166 | // Pop invalid pairs from the top of the heap. 167 | while (!pairs.empty() && !pairs[0].valid) { 168 | std::pop_heap(pairs.begin(), pairs.end(), HistogramPairComparator()); 169 | pairs.pop_back(); 170 | } 171 | // Push new pairs formed with the combined histogram to the heap. 172 | for (int i = 0; i < clusters.size(); ++i) { 173 | CompareAndPushToHeap(out, cluster_size, best_idx1, clusters[i], &pairs); 174 | } 175 | } 176 | } 177 | 178 | // ----------------------------------------------------------------------------- 179 | // Histogram refinement 180 | 181 | // What is the bit cost of moving histogram from cur_symbol to candidate. 182 | template 183 | double HistogramBitCostDistance(const HistogramType& histogram, 184 | const HistogramType& candidate) { 185 | if (histogram.total_count_ == 0) { 186 | return 0.0; 187 | } 188 | HistogramType tmp = histogram; 189 | tmp.AddHistogram(candidate); 190 | return PopulationCost(tmp) - candidate.bit_cost_; 191 | } 192 | 193 | // Find the best 'out' histogram for each of the 'in' histograms. 194 | // Note: we assume that out[]->bit_cost_ is already up-to-date. 195 | template 196 | void HistogramRemap(const HistogramType* in, int in_size, 197 | HistogramType* out, int* symbols) { 198 | std::set all_symbols; 199 | for (int i = 0; i < in_size; ++i) { 200 | all_symbols.insert(symbols[i]); 201 | } 202 | for (int i = 0; i < in_size; ++i) { 203 | int best_out = i == 0 ? symbols[0] : symbols[i - 1]; 204 | double best_bits = HistogramBitCostDistance(in[i], out[best_out]); 205 | for (std::set::const_iterator k = all_symbols.begin(); 206 | k != all_symbols.end(); ++k) { 207 | const double cur_bits = HistogramBitCostDistance(in[i], out[*k]); 208 | if (cur_bits < best_bits) { 209 | best_bits = cur_bits; 210 | best_out = *k; 211 | } 212 | } 213 | symbols[i] = best_out; 214 | } 215 | 216 | // Recompute each out based on raw and symbols. 217 | for (std::set::const_iterator k = all_symbols.begin(); 218 | k != all_symbols.end(); ++k) { 219 | out[*k].Clear(); 220 | } 221 | for (int i = 0; i < in_size; ++i) { 222 | out[symbols[i]].AddHistogram(in[i]); 223 | } 224 | } 225 | 226 | // Reorder histograms in *out so that the new symbols in *symbols come in 227 | // increasing order. 228 | template 229 | void HistogramReindex(std::vector* out, 230 | std::vector* symbols) { 231 | std::vector tmp(*out); 232 | std::map new_index; 233 | int next_index = 0; 234 | for (int i = 0; i < symbols->size(); ++i) { 235 | if (new_index.find((*symbols)[i]) == new_index.end()) { 236 | new_index[(*symbols)[i]] = next_index; 237 | (*out)[next_index] = tmp[(*symbols)[i]]; 238 | ++next_index; 239 | } 240 | } 241 | out->resize(next_index); 242 | for (int i = 0; i < symbols->size(); ++i) { 243 | (*symbols)[i] = new_index[(*symbols)[i]]; 244 | } 245 | } 246 | 247 | template 248 | void ClusterHistogramsTrivial(const std::vector& in, 249 | int num_contexts, int num_blocks, 250 | int max_histograms, 251 | std::vector* out, 252 | std::vector* histogram_symbols) { 253 | out->resize(num_blocks); 254 | for (int i = 0; i < num_blocks; ++i) { 255 | (*out)[i].Clear(); 256 | for (int j = 0; j < num_contexts; ++j) { 257 | (*out)[i].AddHistogram(in[i * num_contexts + j]); 258 | histogram_symbols->push_back(i); 259 | } 260 | } 261 | } 262 | 263 | // Clusters similar histograms in 'in' together, the selected histograms are 264 | // placed in 'out', and for each index in 'in', *histogram_symbols will 265 | // indicate which of the 'out' histograms is the best approximation. 266 | template 267 | void ClusterHistograms(const std::vector& in, 268 | int num_contexts, int num_blocks, 269 | int max_histograms, 270 | std::vector* out, 271 | std::vector* histogram_symbols) { 272 | const int in_size = num_contexts * num_blocks; 273 | std::vector cluster_size(in_size, 1); 274 | out->resize(in_size); 275 | histogram_symbols->resize(in_size); 276 | for (int i = 0; i < in_size; ++i) { 277 | (*out)[i] = in[i]; 278 | (*out)[i].bit_cost_ = PopulationCost(in[i]); 279 | (*histogram_symbols)[i] = i; 280 | } 281 | 282 | // Collapse similar histograms within a block type. 283 | if (num_contexts > 1) { 284 | for (int i = 0; i < num_blocks; ++i) { 285 | HistogramCombine(&(*out)[0], &cluster_size[0], 286 | &(*histogram_symbols)[i * num_contexts], num_contexts, 287 | max_histograms); 288 | } 289 | } 290 | 291 | // Collapse similar histograms. 292 | HistogramCombine(&(*out)[0], &cluster_size[0], 293 | &(*histogram_symbols)[0], in_size, 294 | max_histograms); 295 | 296 | // Find the optimal map from original histograms to the final ones. 297 | HistogramRemap(&in[0], in_size, &(*out)[0], &(*histogram_symbols)[0]); 298 | 299 | // Convert the context map to a canonical form. 300 | HistogramReindex(out, histogram_symbols); 301 | } 302 | 303 | } // namespace brotli 304 | 305 | #endif // BROTLI_ENC_CLUSTER_H_ 306 | -------------------------------------------------------------------------------- /csrc/enc/command.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // This class models a sequence of literals and a backward reference copy. 16 | 17 | #ifndef BROTLI_ENC_COMMAND_H_ 18 | #define BROTLI_ENC_COMMAND_H_ 19 | 20 | #include 21 | #include "./fast_log.h" 22 | 23 | namespace brotli { 24 | 25 | static inline void GetDistCode(int distance_code, 26 | uint16_t* code, uint32_t* extra) { 27 | distance_code -= 1; 28 | if (distance_code < 16) { 29 | *code = distance_code; 30 | *extra = 0; 31 | } else { 32 | distance_code -= 12; 33 | int numextra = Log2FloorNonZero(distance_code) - 1; 34 | int prefix = distance_code >> numextra; 35 | *code = 12 + 2 * numextra + prefix; 36 | *extra = (numextra << 24) | (distance_code - (prefix << numextra)); 37 | } 38 | } 39 | 40 | static int insbase[] = { 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 41 | 98, 130, 194, 322, 578, 1090, 2114, 6210, 22594 }; 42 | static int insextra[] = { 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 43 | 5, 6, 7, 8, 9, 10, 12, 14, 24 }; 44 | static int copybase[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 45 | 54, 70, 102, 134, 198, 326, 582, 1094, 2118 }; 46 | static int copyextra[] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 47 | 4, 5, 5, 6, 7, 8, 9, 10, 24 }; 48 | 49 | static inline int GetInsertLengthCode(int insertlen) { 50 | if (insertlen < 6) { 51 | return insertlen; 52 | } else if (insertlen < 130) { 53 | insertlen -= 2; 54 | int nbits = Log2FloorNonZero(insertlen) - 1; 55 | return (nbits << 1) + (insertlen >> nbits) + 2; 56 | } else if (insertlen < 2114) { 57 | return Log2FloorNonZero(insertlen - 66) + 10; 58 | } else if (insertlen < 6210) { 59 | return 21; 60 | } else if (insertlen < 22594) { 61 | return 22; 62 | } else { 63 | return 23; 64 | } 65 | } 66 | 67 | static inline int GetCopyLengthCode(int copylen) { 68 | if (copylen < 10) { 69 | return copylen - 2; 70 | } else if (copylen < 134) { 71 | copylen -= 6; 72 | int nbits = Log2FloorNonZero(copylen) - 1; 73 | return (nbits << 1) + (copylen >> nbits) + 4; 74 | } else if (copylen < 2118) { 75 | return Log2FloorNonZero(copylen - 70) + 12; 76 | } else { 77 | return 23; 78 | } 79 | } 80 | 81 | static inline int CombineLengthCodes( 82 | int inscode, int copycode, int distancecode) { 83 | int bits64 = (copycode & 0x7u) | ((inscode & 0x7u) << 3); 84 | if (distancecode == 0 && inscode < 8 && copycode < 16) { 85 | return (copycode < 8) ? bits64 : (bits64 | 64); 86 | } else { 87 | // "To convert an insert-and-copy length code to an insert length code and 88 | // a copy length code, the following table can be used" 89 | static const int cells[9] = { 2, 3, 6, 4, 5, 8, 7, 9, 10 }; 90 | return (cells[(copycode >> 3) + 3 * (inscode >> 3)] << 6) | bits64; 91 | } 92 | } 93 | 94 | static inline void GetLengthCode(int insertlen, int copylen, int distancecode, 95 | uint16_t* code, uint64_t* extra) { 96 | int inscode = GetInsertLengthCode(insertlen); 97 | int copycode = GetCopyLengthCode(copylen); 98 | uint64_t insnumextra = insextra[inscode]; 99 | uint64_t numextra = insnumextra + copyextra[copycode]; 100 | uint64_t insextraval = insertlen - insbase[inscode]; 101 | uint64_t copyextraval = copylen - copybase[copycode]; 102 | *code = CombineLengthCodes(inscode, copycode, distancecode); 103 | *extra = (numextra << 48) | (copyextraval << insnumextra) | insextraval; 104 | } 105 | 106 | struct Command { 107 | Command() {} 108 | 109 | Command(int insertlen, int copylen, int copylen_code, int distance_code) 110 | : insert_len_(insertlen), copy_len_(copylen) { 111 | GetDistCode(distance_code, &dist_prefix_, &dist_extra_); 112 | GetLengthCode(insertlen, copylen_code, dist_prefix_, 113 | &cmd_prefix_, &cmd_extra_); 114 | } 115 | 116 | Command(int insertlen) 117 | : insert_len_(insertlen), copy_len_(0), dist_prefix_(16), dist_extra_(0) { 118 | GetLengthCode(insertlen, 4, dist_prefix_, &cmd_prefix_, &cmd_extra_); 119 | } 120 | 121 | int DistanceCode() const { 122 | if (dist_prefix_ < 16) { 123 | return dist_prefix_ + 1; 124 | } 125 | int nbits = dist_extra_ >> 24; 126 | int extra = dist_extra_ & 0xffffff; 127 | int prefix = dist_prefix_ - 12 - 2 * nbits; 128 | return (prefix << nbits) + extra + 13; 129 | } 130 | 131 | int DistanceContext() const { 132 | int r = cmd_prefix_ >> 6; 133 | int c = cmd_prefix_ & 7; 134 | if ((r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2)) { 135 | return c; 136 | } 137 | return 3; 138 | } 139 | 140 | int insert_len_; 141 | int copy_len_; 142 | uint16_t cmd_prefix_; 143 | uint16_t dist_prefix_; 144 | uint64_t cmd_extra_; 145 | uint32_t dist_extra_; 146 | }; 147 | 148 | } // namespace brotli 149 | 150 | #endif // BROTLI_ENC_COMMAND_H_ 151 | -------------------------------------------------------------------------------- /csrc/enc/context.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions to map previous bytes into a context id. 16 | 17 | #ifndef BROTLI_ENC_CONTEXT_H_ 18 | #define BROTLI_ENC_CONTEXT_H_ 19 | 20 | #include 21 | 22 | namespace brotli { 23 | 24 | // Second-order context lookup table for UTF8 byte streams. 25 | // 26 | // If p1 and p2 are the previous two bytes, we calcualte the context as 27 | // 28 | // context = kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256]. 29 | // 30 | // If the previous two bytes are ASCII characters (i.e. < 128), this will be 31 | // equivalent to 32 | // 33 | // context = 4 * context1(p1) + context2(p2), 34 | // 35 | // where context1 is based on the previous byte in the following way: 36 | // 37 | // 0 : non-ASCII control 38 | // 1 : \t, \n, \r 39 | // 2 : space 40 | // 3 : other punctuation 41 | // 4 : " ' 42 | // 5 : % 43 | // 6 : ( < [ { 44 | // 7 : ) > ] } 45 | // 8 : , ; : 46 | // 9 : . 47 | // 10 : = 48 | // 11 : number 49 | // 12 : upper-case vowel 50 | // 13 : upper-case consonant 51 | // 14 : lower-case vowel 52 | // 15 : lower-case consonant 53 | // 54 | // and context2 is based on the second last byte: 55 | // 56 | // 0 : control, space 57 | // 1 : punctuation 58 | // 2 : upper-case letter, number 59 | // 3 : lower-case letter 60 | // 61 | // If the last byte is ASCII, and the second last byte is not (in a valid UTF8 62 | // stream it will be a continuation byte, value between 128 and 191), the 63 | // context is the same as if the second last byte was an ASCII control or space. 64 | // 65 | // If the last byte is a UTF8 lead byte (value >= 192), then the next byte will 66 | // be a continuation byte and the context id is 2 or 3 depending on the LSB of 67 | // the last byte and to a lesser extent on the second last byte if it is ASCII. 68 | // 69 | // If the last byte is a UTF8 continuation byte, the second last byte can be: 70 | // - continuation byte: the next byte is probably ASCII or lead byte (assuming 71 | // 4-byte UTF8 characters are rare) and the context id is 0 or 1. 72 | // - lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1 73 | // - lead byte (208 - 255): next byte is continuation byte, context is 2 or 3 74 | // 75 | // The possible value combinations of the previous two bytes, the range of 76 | // context ids and the type of the next byte is summarized in the table below: 77 | // 78 | // |--------\-----------------------------------------------------------------| 79 | // | \ Last byte | 80 | // | Second \---------------------------------------------------------------| 81 | // | last byte \ ASCII | cont. byte | lead byte | 82 | // | \ (0-127) | (128-191) | (192-) | 83 | // |=============|===================|=====================|==================| 84 | // | ASCII | next: ASCII/lead | not valid | next: cont. | 85 | // | (0-127) | context: 4 - 63 | | context: 2 - 3 | 86 | // |-------------|-------------------|---------------------|------------------| 87 | // | cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. | 88 | // | (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 | 89 | // |-------------|-------------------|---------------------|------------------| 90 | // | lead byte | not valid | next: ASCII/lead | not valid | 91 | // | (192-207) | | context: 0 - 1 | | 92 | // |-------------|-------------------|---------------------|------------------| 93 | // | lead byte | not valid | next: cont. | not valid | 94 | // | (208-) | | context: 2 - 3 | | 95 | // |-------------|-------------------|---------------------|------------------| 96 | static const uint8_t kUTF8ContextLookup[512] = { 97 | // Last byte. 98 | // 99 | // ASCII range. 100 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0, 101 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102 | 8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12, 103 | 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12, 104 | 12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48, 105 | 52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12, 106 | 12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56, 107 | 60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0, 108 | // UTF8 continuation byte range. 109 | 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 110 | 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 111 | 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 112 | 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 113 | // UTF8 lead byte range. 114 | 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 115 | 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 116 | 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 117 | 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 118 | // Second last byte. 119 | // 120 | // ASCII range. 121 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123 | 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 124 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 125 | 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 126 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 127 | 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 128 | 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0, 129 | // UTF8 continuation byte range. 130 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135 | // UTF8 lead byte range. 136 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 138 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 139 | }; 140 | 141 | // Context lookup table for small signed integers. 142 | static const uint8_t kSigned3BitContextLookup[] = { 143 | 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 144 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 145 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 146 | 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 147 | 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 148 | 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 149 | 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 150 | 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 151 | 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 152 | 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 153 | 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 154 | 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 155 | 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 156 | 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 157 | 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 158 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 159 | }; 160 | 161 | enum ContextType { 162 | CONTEXT_LSB6 = 0, 163 | CONTEXT_MSB6 = 1, 164 | CONTEXT_UTF8 = 2, 165 | CONTEXT_SIGNED = 3 166 | }; 167 | 168 | static inline uint8_t Context(uint8_t p1, uint8_t p2, int mode) { 169 | switch (mode) { 170 | case CONTEXT_LSB6: 171 | return p1 & 0x3f; 172 | case CONTEXT_MSB6: 173 | return p1 >> 2; 174 | case CONTEXT_UTF8: 175 | return kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256]; 176 | case CONTEXT_SIGNED: 177 | return (kSigned3BitContextLookup[p1] << 3) + kSigned3BitContextLookup[p2]; 178 | default: 179 | return 0; 180 | } 181 | } 182 | 183 | } // namespace brotli 184 | 185 | #endif // BROTLI_ENC_CONTEXT_H_ 186 | -------------------------------------------------------------------------------- /csrc/enc/encode.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // API for Brotli compression 16 | 17 | #ifndef BROTLI_ENC_ENCODE_H_ 18 | #define BROTLI_ENC_ENCODE_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "./command.h" 25 | #include "./hash.h" 26 | #include "./ringbuffer.h" 27 | #include "./static_dict.h" 28 | #include "./streams.h" 29 | 30 | namespace brotli { 31 | 32 | static const int kMaxWindowBits = 24; 33 | static const int kMinWindowBits = 16; 34 | static const int kMinInputBlockBits = 16; 35 | static const int kMaxInputBlockBits = 24; 36 | 37 | struct BrotliParams { 38 | BrotliParams() 39 | : mode(MODE_TEXT), 40 | quality(11), 41 | lgwin(22), 42 | lgblock(0), 43 | enable_dictionary(true), 44 | enable_transforms(false), 45 | greedy_block_split(false), 46 | enable_context_modeling(true) {} 47 | 48 | enum Mode { 49 | MODE_TEXT = 0, 50 | MODE_FONT = 1, 51 | }; 52 | Mode mode; 53 | 54 | // Controls the compression-speed vs compression-density tradeoffs. The higher 55 | // the quality, the slower the compression. Range is 0 to 11. 56 | int quality; 57 | // Base 2 logarithm of the sliding window size. Range is 16 to 24. 58 | int lgwin; 59 | // Base 2 logarithm of the maximum input block size. Range is 16 to 24. 60 | // If set to 0, the value will be set based on the quality. 61 | int lgblock; 62 | 63 | // These settings will be respected only if quality > 9. 64 | bool enable_dictionary; 65 | bool enable_transforms; 66 | bool greedy_block_split; 67 | bool enable_context_modeling; 68 | }; 69 | 70 | // An instance can not be reused for multiple brotli streams. 71 | class BrotliCompressor { 72 | public: 73 | explicit BrotliCompressor(BrotliParams params); 74 | ~BrotliCompressor(); 75 | 76 | // The maximum input size that can be processed at once. 77 | size_t input_block_size() const { return 1 << params_.lgblock; } 78 | 79 | // Encodes the data in input_buffer as a meta-block and writes it to 80 | // encoded_buffer (*encoded_size should be set to the size of 81 | // encoded_buffer) and sets *encoded_size to the number of bytes that 82 | // was written. Returns 0 if there was an error and 1 otherwise. 83 | bool WriteMetaBlock(const size_t input_size, 84 | const uint8_t* input_buffer, 85 | const bool is_last, 86 | size_t* encoded_size, 87 | uint8_t* encoded_buffer); 88 | 89 | // Writes a metadata meta-block containing the given input to encoded_buffer. 90 | // *encoded_size should be set to the size of the encoded_buffer. 91 | // Sets *encoded_size to the number of bytes that was written. 92 | // Note that the given input data will not be part of the sliding window and 93 | // thus no backward references can be made to this data from subsequent 94 | // metablocks. 95 | bool WriteMetadata(const size_t input_size, 96 | const uint8_t* input_buffer, 97 | const bool is_last, 98 | size_t* encoded_size, 99 | uint8_t* encoded_buffer); 100 | 101 | // Writes a zero-length meta-block with end-of-input bit set to the 102 | // internal output buffer and copies the output buffer to encoded_buffer 103 | // (*encoded_size should be set to the size of encoded_buffer) and sets 104 | // *encoded_size to the number of bytes written. Returns false if there was 105 | // an error and true otherwise. 106 | bool FinishStream(size_t* encoded_size, uint8_t* encoded_buffer); 107 | 108 | // Copies the given input data to the internal ring buffer of the compressor. 109 | // No processing of the data occurs at this time and this function can be 110 | // called multiple times before calling WriteBrotliData() to process the 111 | // accumulated input. At most input_block_size() bytes of input data can be 112 | // copied to the ring buffer, otherwise the next WriteBrotliData() will fail. 113 | void CopyInputToRingBuffer(const size_t input_size, 114 | const uint8_t* input_buffer); 115 | 116 | // Processes the accumulated input data and sets *out_size to the length of 117 | // the new output meta-block, or to zero if no new output meta-block was 118 | // created (in this case the processed input data is buffered internally). 119 | // If *out_size is positive, *output points to the start of the output data. 120 | // Returns false if the size of the input data is larger than 121 | // input_block_size() or if there was an error during writing the output. 122 | // If is_last or force_flush is true, an output meta-block is always created. 123 | bool WriteBrotliData(const bool is_last, const bool force_flush, 124 | size_t* out_size, uint8_t** output); 125 | 126 | // No-op, but we keep it here for API backward-compatibility. 127 | void WriteStreamHeader() {} 128 | 129 | private: 130 | // Initializes the hasher with the hashes of dictionary words. 131 | void StoreDictionaryWordHashes(bool enable_transforms); 132 | 133 | uint8_t* GetBrotliStorage(size_t size); 134 | 135 | bool WriteMetaBlockInternal(const bool is_last, 136 | const bool utf8_mode, 137 | size_t* out_size, 138 | uint8_t** output); 139 | 140 | BrotliParams params_; 141 | int max_backward_distance_; 142 | std::unique_ptr hashers_; 143 | int hash_type_; 144 | size_t input_pos_; 145 | std::unique_ptr ringbuffer_; 146 | std::unique_ptr literal_cost_; 147 | size_t literal_cost_mask_; 148 | size_t cmd_buffer_size_; 149 | std::unique_ptr commands_; 150 | int num_commands_; 151 | int last_insert_len_; 152 | size_t last_flush_pos_; 153 | size_t last_processed_pos_; 154 | int dist_cache_[4]; 155 | uint8_t last_byte_; 156 | uint8_t last_byte_bits_; 157 | uint8_t prev_byte_; 158 | uint8_t prev_byte2_; 159 | int storage_size_; 160 | std::unique_ptr storage_; 161 | static StaticDictionary *static_dictionary_; 162 | }; 163 | 164 | // Compresses the data in input_buffer into encoded_buffer, and sets 165 | // *encoded_size to the compressed length. 166 | // Returns 0 if there was an error and 1 otherwise. 167 | int BrotliCompressBuffer(BrotliParams params, 168 | size_t input_size, 169 | const uint8_t* input_buffer, 170 | size_t* encoded_size, 171 | uint8_t* encoded_buffer); 172 | 173 | // Same as above, but uses the specified input and output classes instead 174 | // of reading from and writing to pre-allocated memory buffers. 175 | int BrotliCompress(BrotliParams params, BrotliIn* in, BrotliOut* out); 176 | 177 | } // namespace brotli 178 | 179 | #endif // BROTLI_ENC_ENCODE_H_ 180 | -------------------------------------------------------------------------------- /csrc/enc/encode_parallel.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Implementation of parallel Brotli compressor. 16 | 17 | #include "./encode_parallel.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "./backward_references.h" 23 | #include "./bit_cost.h" 24 | #include "./block_splitter.h" 25 | #include "./brotli_bit_stream.h" 26 | #include "./cluster.h" 27 | #include "./context.h" 28 | #include "./metablock.h" 29 | #include "./transform.h" 30 | #include "./entropy_encode.h" 31 | #include "./fast_log.h" 32 | #include "./hash.h" 33 | #include "./histogram.h" 34 | #include "./literal_cost.h" 35 | #include "./prefix.h" 36 | #include "./write_bits.h" 37 | 38 | namespace brotli { 39 | 40 | namespace { 41 | 42 | int ParseAsUTF8(int* symbol, const uint8_t* input, int size) { 43 | // ASCII 44 | if ((input[0] & 0x80) == 0) { 45 | *symbol = input[0]; 46 | if (*symbol > 0) { 47 | return 1; 48 | } 49 | } 50 | // 2-byte UTF8 51 | if (size > 1 && 52 | (input[0] & 0xe0) == 0xc0 && 53 | (input[1] & 0xc0) == 0x80) { 54 | *symbol = (((input[0] & 0x1f) << 6) | 55 | (input[1] & 0x3f)); 56 | if (*symbol > 0x7f) { 57 | return 2; 58 | } 59 | } 60 | // 3-byte UFT8 61 | if (size > 2 && 62 | (input[0] & 0xf0) == 0xe0 && 63 | (input[1] & 0xc0) == 0x80 && 64 | (input[2] & 0xc0) == 0x80) { 65 | *symbol = (((input[0] & 0x0f) << 12) | 66 | ((input[1] & 0x3f) << 6) | 67 | (input[2] & 0x3f)); 68 | if (*symbol > 0x7ff) { 69 | return 3; 70 | } 71 | } 72 | // 4-byte UFT8 73 | if (size > 3 && 74 | (input[0] & 0xf8) == 0xf0 && 75 | (input[1] & 0xc0) == 0x80 && 76 | (input[2] & 0xc0) == 0x80 && 77 | (input[3] & 0xc0) == 0x80) { 78 | *symbol = (((input[0] & 0x07) << 18) | 79 | ((input[1] & 0x3f) << 12) | 80 | ((input[2] & 0x3f) << 6) | 81 | (input[3] & 0x3f)); 82 | if (*symbol > 0xffff && *symbol <= 0x10ffff) { 83 | return 4; 84 | } 85 | } 86 | // Not UTF8, emit a special symbol above the UTF8-code space 87 | *symbol = 0x110000 | input[0]; 88 | return 1; 89 | } 90 | 91 | // Returns true if at least min_fraction of the data is UTF8-encoded. 92 | bool IsMostlyUTF8(const uint8_t* data, size_t length, double min_fraction) { 93 | size_t size_utf8 = 0; 94 | for (size_t pos = 0; pos < length; ) { 95 | int symbol; 96 | int bytes_read = ParseAsUTF8(&symbol, data + pos, length - pos); 97 | pos += bytes_read; 98 | if (symbol < 0x110000) size_utf8 += bytes_read; 99 | } 100 | return size_utf8 > min_fraction * length; 101 | } 102 | 103 | void RecomputeDistancePrefixes(std::vector* cmds, 104 | int num_direct_distance_codes, 105 | int distance_postfix_bits) { 106 | if (num_direct_distance_codes == 0 && 107 | distance_postfix_bits == 0) { 108 | return; 109 | } 110 | for (int i = 0; i < cmds->size(); ++i) { 111 | Command* cmd = &(*cmds)[i]; 112 | if (cmd->copy_len_ > 0 && cmd->cmd_prefix_ >= 128) { 113 | PrefixEncodeCopyDistance(cmd->DistanceCode(), 114 | num_direct_distance_codes, 115 | distance_postfix_bits, 116 | &cmd->dist_prefix_, 117 | &cmd->dist_extra_); 118 | } 119 | } 120 | } 121 | 122 | bool WriteMetaBlockParallel(const BrotliParams& params, 123 | const size_t block_size, 124 | const uint8_t* input_buffer, 125 | const size_t prefix_size, 126 | const uint8_t* prefix_buffer, 127 | const StaticDictionary* static_dict, 128 | const bool is_first, 129 | const bool is_last, 130 | size_t* encoded_size, 131 | uint8_t* encoded_buffer) { 132 | if (block_size == 0) { 133 | return false; 134 | } 135 | const size_t input_size = block_size; 136 | 137 | // Copy prefix + next input block into a continuous area. 138 | size_t input_pos = prefix_size; 139 | std::vector input(prefix_size + input_size); 140 | memcpy(&input[0], prefix_buffer, prefix_size); 141 | memcpy(&input[input_pos], input_buffer, input_size); 142 | // Since we don't have a ringbuffer, masking is a no-op. 143 | // We use one less bit than the full range because some of the code uses 144 | // mask + 1 as the size of the ringbuffer. 145 | const size_t mask = std::numeric_limits::max() >> 1; 146 | 147 | uint8_t prev_byte = input_pos > 0 ? input[(input_pos - 1) & mask] : 0; 148 | uint8_t prev_byte2 = input_pos > 1 ? input[(input_pos - 2) & mask] : 0; 149 | 150 | // Decide about UTF8 mode. 151 | static const double kMinUTF8Ratio = 0.75; 152 | bool utf8_mode = IsMostlyUTF8(&input[input_pos], input_size, kMinUTF8Ratio); 153 | 154 | // Compute literal costs. 155 | std::vector literal_cost(prefix_size + input_size); 156 | if (utf8_mode) { 157 | EstimateBitCostsForLiteralsUTF8(input_pos, input_size, mask, mask, 158 | &input[0], &literal_cost[0]); 159 | } else { 160 | EstimateBitCostsForLiterals(input_pos, input_size, mask, mask, 161 | &input[0], &literal_cost[0]); 162 | } 163 | 164 | // Initialize hashers. 165 | int hash_type = 9; 166 | switch (params.mode) { 167 | case BrotliParams::MODE_TEXT: hash_type = 8; break; 168 | case BrotliParams::MODE_FONT: hash_type = 9; break; 169 | default: break; 170 | } 171 | std::unique_ptr hashers(new Hashers()); 172 | hashers->Init(hash_type); 173 | hashers->SetStaticDictionary(static_dict); 174 | 175 | // Compute backward references. 176 | int last_insert_len = 0; 177 | int num_commands = 0; 178 | double base_min_score = 8.115; 179 | int max_backward_distance = (1 << params.lgwin) - 16; 180 | int dist_cache[4] = { -4, -4, -4, -4 }; 181 | std::vector commands((input_size + 1) >> 1); 182 | CreateBackwardReferences( 183 | input_size, input_pos, 184 | &input[0], mask, 185 | &literal_cost[0], mask, 186 | max_backward_distance, 187 | base_min_score, 188 | params.quality, 189 | hashers.get(), 190 | hash_type, 191 | dist_cache, 192 | &last_insert_len, 193 | &commands[0], 194 | &num_commands); 195 | commands.resize(num_commands); 196 | if (last_insert_len > 0) { 197 | commands.push_back(Command(last_insert_len)); 198 | } 199 | 200 | // Build the meta-block. 201 | MetaBlockSplit mb; 202 | int num_direct_distance_codes = 203 | params.mode == BrotliParams::MODE_FONT ? 12 : 0; 204 | int distance_postfix_bits = params.mode == BrotliParams::MODE_FONT ? 1 : 0; 205 | int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED; 206 | RecomputeDistancePrefixes(&commands, 207 | num_direct_distance_codes, 208 | distance_postfix_bits); 209 | if (params.greedy_block_split) { 210 | BuildMetaBlockGreedy(&input[0], input_pos, mask, 211 | commands.data(), commands.size(), 212 | &mb); 213 | } else { 214 | BuildMetaBlock(&input[0], input_pos, mask, 215 | prev_byte, prev_byte2, 216 | commands.data(), commands.size(), 217 | literal_context_mode, 218 | true, 219 | &mb); 220 | } 221 | 222 | // Set up the temporary output storage. 223 | const size_t max_out_size = 2 * input_size + 500; 224 | std::vector storage(max_out_size); 225 | int first_byte = 0; 226 | int first_byte_bits = 0; 227 | if (is_first) { 228 | if (params.lgwin == 16) { 229 | first_byte = 0; 230 | first_byte_bits = 1; 231 | } else { 232 | first_byte = ((params.lgwin - 17) << 1) | 1; 233 | first_byte_bits = 4; 234 | } 235 | } 236 | storage[0] = first_byte; 237 | int storage_ix = first_byte_bits; 238 | 239 | // Store the meta-block to the temporary output. 240 | if (!StoreMetaBlock(&input[0], input_pos, input_size, mask, 241 | prev_byte, prev_byte2, 242 | is_last, 243 | num_direct_distance_codes, 244 | distance_postfix_bits, 245 | literal_context_mode, 246 | commands.data(), commands.size(), 247 | mb, 248 | &storage_ix, &storage[0])) { 249 | return false; 250 | } 251 | 252 | // If this is not the last meta-block, store an empty metadata 253 | // meta-block so that the meta-block will end at a byte boundary. 254 | if (!is_last) { 255 | StoreSyncMetaBlock(&storage_ix, &storage[0]); 256 | } 257 | 258 | // If the compressed data is too large, fall back to an uncompressed 259 | // meta-block. 260 | size_t output_size = storage_ix >> 3; 261 | if (input_size + 4 < output_size) { 262 | storage[0] = first_byte; 263 | storage_ix = first_byte_bits; 264 | if (!StoreUncompressedMetaBlock(is_last, &input[0], input_pos, mask, 265 | input_size, 266 | &storage_ix, &storage[0])) { 267 | return false; 268 | } 269 | output_size = storage_ix >> 3; 270 | } 271 | 272 | // Copy the temporary output with size-check to the output. 273 | if (output_size > *encoded_size) { 274 | return false; 275 | } 276 | memcpy(encoded_buffer, &storage[0], output_size); 277 | *encoded_size = output_size; 278 | return true; 279 | } 280 | 281 | } // namespace 282 | 283 | int BrotliCompressBufferParallel(BrotliParams params, 284 | size_t input_size, 285 | const uint8_t* input_buffer, 286 | size_t* encoded_size, 287 | uint8_t* encoded_buffer) { 288 | if (*encoded_size == 0) { 289 | // Output buffer needs at least one byte. 290 | return 0; 291 | } else if (input_size == 0) { 292 | encoded_buffer[0] = 6; 293 | *encoded_size = 1; 294 | return 1; 295 | } 296 | 297 | // Sanitize params. 298 | if (params.lgwin < kMinWindowBits) { 299 | params.lgwin = kMinWindowBits; 300 | } else if (params.lgwin > kMaxWindowBits) { 301 | params.lgwin = kMaxWindowBits; 302 | } 303 | if (params.lgblock == 0) { 304 | params.lgblock = 16; 305 | if (params.quality >= 9 && params.lgwin > params.lgblock) { 306 | params.lgblock = std::min(21, params.lgwin); 307 | } 308 | } else if (params.lgblock < kMinInputBlockBits) { 309 | params.lgblock = kMinInputBlockBits; 310 | } else if (params.lgblock > kMaxInputBlockBits) { 311 | params.lgblock = kMaxInputBlockBits; 312 | } 313 | size_t max_input_block_size = 1 << params.lgblock; 314 | 315 | std::vector > compressed_pieces; 316 | StaticDictionary dict; 317 | dict.Fill(params.enable_transforms); 318 | 319 | // Compress block-by-block independently. 320 | for (size_t pos = 0; pos < input_size; ) { 321 | size_t input_block_size = std::min(max_input_block_size, input_size - pos); 322 | size_t out_size = 1.2 * input_block_size + 1024; 323 | std::vector out(out_size); 324 | if (!WriteMetaBlockParallel(params, 325 | input_block_size, 326 | &input_buffer[pos], 327 | pos, 328 | input_buffer, 329 | &dict, 330 | pos == 0, 331 | pos + input_block_size == input_size, 332 | &out_size, 333 | &out[0])) { 334 | return false; 335 | } 336 | out.resize(out_size); 337 | compressed_pieces.push_back(out); 338 | pos += input_block_size; 339 | } 340 | 341 | // Piece together the output. 342 | size_t out_pos = 0; 343 | for (int i = 0; i < compressed_pieces.size(); ++i) { 344 | const std::vector& out = compressed_pieces[i]; 345 | if (out_pos + out.size() > *encoded_size) { 346 | return false; 347 | } 348 | memcpy(&encoded_buffer[out_pos], &out[0], out.size()); 349 | out_pos += out.size(); 350 | } 351 | *encoded_size = out_pos; 352 | 353 | return true; 354 | } 355 | 356 | } // namespace brotli 357 | -------------------------------------------------------------------------------- /csrc/enc/encode_parallel.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // API for parallel Brotli compression 16 | // Note that this is only a proof of concept currently and not part of the 17 | // final API yet. 18 | 19 | #ifndef BROTLI_ENC_ENCODE_PARALLEL_H_ 20 | #define BROTLI_ENC_ENCODE_PARALLEL_H_ 21 | 22 | #include 23 | #include 24 | 25 | #include "./encode.h" 26 | 27 | namespace brotli { 28 | 29 | int BrotliCompressBufferParallel(BrotliParams params, 30 | size_t input_size, 31 | const uint8_t* input_buffer, 32 | size_t* encoded_size, 33 | uint8_t* encoded_buffer); 34 | 35 | } // namespace brotli 36 | 37 | #endif // BROTLI_ENC_ENCODE_PARALLEL_H_ 38 | -------------------------------------------------------------------------------- /csrc/enc/entropy_encode.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Entropy encoding (Huffman) utilities. 16 | 17 | #ifndef BROTLI_ENC_ENTROPY_ENCODE_H_ 18 | #define BROTLI_ENC_ENTROPY_ENCODE_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include "./histogram.h" 24 | #include "./prefix.h" 25 | 26 | namespace brotli { 27 | 28 | // This function will create a Huffman tree. 29 | // 30 | // The (data,length) contains the population counts. 31 | // The tree_limit is the maximum bit depth of the Huffman codes. 32 | // 33 | // The depth contains the tree, i.e., how many bits are used for 34 | // the symbol. 35 | // 36 | // See http://en.wikipedia.org/wiki/Huffman_coding 37 | void CreateHuffmanTree(const int *data, 38 | const int length, 39 | const int tree_limit, 40 | uint8_t *depth); 41 | 42 | // Change the population counts in a way that the consequent 43 | // Hufmann tree compression, especially its rle-part will be more 44 | // likely to compress this data more efficiently. 45 | // 46 | // length contains the size of the histogram. 47 | // counts contains the population counts. 48 | int OptimizeHuffmanCountsForRle(int length, int* counts); 49 | 50 | // Write a huffman tree from bit depths into the bitstream representation 51 | // of a Huffman tree. The generated Huffman tree is to be compressed once 52 | // more using a Huffman tree 53 | void WriteHuffmanTree(const uint8_t* depth, 54 | uint32_t num, 55 | std::vector *tree, 56 | std::vector *extra_bits_data); 57 | 58 | // Get the actual bit values for a tree of bit depths. 59 | void ConvertBitDepthsToSymbols(const uint8_t *depth, int len, uint16_t *bits); 60 | 61 | template 62 | struct EntropyCode { 63 | // How many bits for symbol. 64 | uint8_t depth_[kSize]; 65 | // Actual bits used to represent the symbol. 66 | uint16_t bits_[kSize]; 67 | // How many non-zero depth. 68 | int count_; 69 | // First four symbols with non-zero depth. 70 | int symbols_[4]; 71 | }; 72 | 73 | static const int kCodeLengthCodes = 18; 74 | 75 | // Literal entropy code. 76 | typedef EntropyCode<256> EntropyCodeLiteral; 77 | // Prefix entropy codes. 78 | typedef EntropyCode EntropyCodeCommand; 79 | typedef EntropyCode EntropyCodeDistance; 80 | typedef EntropyCode EntropyCodeBlockLength; 81 | // Context map entropy code, 256 Huffman tree indexes + 16 run length codes. 82 | typedef EntropyCode<272> EntropyCodeContextMap; 83 | // Block type entropy code, 256 block types + 2 special symbols. 84 | typedef EntropyCode<258> EntropyCodeBlockType; 85 | 86 | } // namespace brotli 87 | 88 | #endif // BROTLI_ENC_ENTROPY_ENCODE_H_ 89 | -------------------------------------------------------------------------------- /csrc/enc/fast_log.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Utilities for fast computation of logarithms. 16 | 17 | #ifndef BROTLI_ENC_FAST_LOG_H_ 18 | #define BROTLI_ENC_FAST_LOG_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | namespace brotli { 25 | 26 | // Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0. 27 | inline int Log2Floor(uint32_t n) { 28 | #if defined(__clang__) || \ 29 | (defined(__GNUC__) && \ 30 | ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4)) 31 | return n == 0 ? -1 : 31 ^ __builtin_clz(n); 32 | #else 33 | if (n == 0) 34 | return -1; 35 | int log = 0; 36 | uint32_t value = n; 37 | for (int i = 4; i >= 0; --i) { 38 | int shift = (1 << i); 39 | uint32_t x = value >> shift; 40 | if (x != 0) { 41 | value = x; 42 | log += shift; 43 | } 44 | } 45 | assert(value == 1); 46 | return log; 47 | #endif 48 | } 49 | 50 | static inline int Log2FloorNonZero(uint32_t n) { 51 | #ifdef __GNUC__ 52 | return 31 ^ __builtin_clz(n); 53 | #else 54 | unsigned int result = 0; 55 | while (n >>= 1) result++; 56 | return result; 57 | #endif 58 | } 59 | 60 | // Return ceiling(log2(n)) for positive integer n. Returns -1 iff n == 0. 61 | inline int Log2Ceiling(uint32_t n) { 62 | int floor = Log2Floor(n); 63 | if (n == (n &~ (n - 1))) // zero or a power of two 64 | return floor; 65 | else 66 | return floor + 1; 67 | } 68 | 69 | // A lookup table for small values of log2(int) to be used in entropy 70 | // computation. 71 | // 72 | // ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) 73 | static const float kLog2Table[] = { 74 | 0.0000000000000000f, 0.0000000000000000f, 1.0000000000000000f, 75 | 1.5849625007211563f, 2.0000000000000000f, 2.3219280948873622f, 76 | 2.5849625007211561f, 2.8073549220576042f, 3.0000000000000000f, 77 | 3.1699250014423126f, 3.3219280948873626f, 3.4594316186372978f, 78 | 3.5849625007211565f, 3.7004397181410922f, 3.8073549220576037f, 79 | 3.9068905956085187f, 4.0000000000000000f, 4.0874628412503400f, 80 | 4.1699250014423122f, 4.2479275134435852f, 4.3219280948873626f, 81 | 4.3923174227787607f, 4.4594316186372973f, 4.5235619560570131f, 82 | 4.5849625007211570f, 4.6438561897747244f, 4.7004397181410926f, 83 | 4.7548875021634691f, 4.8073549220576037f, 4.8579809951275728f, 84 | 4.9068905956085187f, 4.9541963103868758f, 5.0000000000000000f, 85 | 5.0443941193584534f, 5.0874628412503400f, 5.1292830169449664f, 86 | 5.1699250014423122f, 5.2094533656289501f, 5.2479275134435852f, 87 | 5.2854022188622487f, 5.3219280948873626f, 5.3575520046180838f, 88 | 5.3923174227787607f, 5.4262647547020979f, 5.4594316186372973f, 89 | 5.4918530963296748f, 5.5235619560570131f, 5.5545888516776376f, 90 | 5.5849625007211570f, 5.6147098441152083f, 5.6438561897747244f, 91 | 5.6724253419714961f, 5.7004397181410926f, 5.7279204545631996f, 92 | 5.7548875021634691f, 5.7813597135246599f, 5.8073549220576046f, 93 | 5.8328900141647422f, 5.8579809951275719f, 5.8826430493618416f, 94 | 5.9068905956085187f, 5.9307373375628867f, 5.9541963103868758f, 95 | 5.9772799234999168f, 6.0000000000000000f, 6.0223678130284544f, 96 | 6.0443941193584534f, 6.0660891904577721f, 6.0874628412503400f, 97 | 6.1085244567781700f, 6.1292830169449672f, 6.1497471195046822f, 98 | 6.1699250014423122f, 6.1898245588800176f, 6.2094533656289510f, 99 | 6.2288186904958804f, 6.2479275134435861f, 6.2667865406949019f, 100 | 6.2854022188622487f, 6.3037807481771031f, 6.3219280948873617f, 101 | 6.3398500028846252f, 6.3575520046180847f, 6.3750394313469254f, 102 | 6.3923174227787598f, 6.4093909361377026f, 6.4262647547020979f, 103 | 6.4429434958487288f, 6.4594316186372982f, 6.4757334309663976f, 104 | 6.4918530963296748f, 6.5077946401986964f, 6.5235619560570131f, 105 | 6.5391588111080319f, 6.5545888516776376f, 6.5698556083309478f, 106 | 6.5849625007211561f, 6.5999128421871278f, 6.6147098441152092f, 107 | 6.6293566200796095f, 6.6438561897747253f, 6.6582114827517955f, 108 | 6.6724253419714952f, 6.6865005271832185f, 6.7004397181410917f, 109 | 6.7142455176661224f, 6.7279204545631988f, 6.7414669864011465f, 110 | 6.7548875021634691f, 6.7681843247769260f, 6.7813597135246599f, 111 | 6.7944158663501062f, 6.8073549220576037f, 6.8201789624151887f, 112 | 6.8328900141647422f, 6.8454900509443757f, 6.8579809951275719f, 113 | 6.8703647195834048f, 6.8826430493618416f, 6.8948177633079437f, 114 | 6.9068905956085187f, 6.9188632372745955f, 6.9307373375628867f, 115 | 6.9425145053392399f, 6.9541963103868758f, 6.9657842846620879f, 116 | 6.9772799234999168f, 6.9886846867721664f, 7.0000000000000000f, 117 | 7.0112272554232540f, 7.0223678130284544f, 7.0334230015374501f, 118 | 7.0443941193584534f, 7.0552824355011898f, 7.0660891904577721f, 119 | 7.0768155970508317f, 7.0874628412503400f, 7.0980320829605272f, 120 | 7.1085244567781700f, 7.1189410727235076f, 7.1292830169449664f, 121 | 7.1395513523987937f, 7.1497471195046822f, 7.1598713367783891f, 122 | 7.1699250014423130f, 7.1799090900149345f, 7.1898245588800176f, 123 | 7.1996723448363644f, 7.2094533656289492f, 7.2191685204621621f, 124 | 7.2288186904958804f, 7.2384047393250794f, 7.2479275134435861f, 125 | 7.2573878426926521f, 7.2667865406949019f, 7.2761244052742384f, 126 | 7.2854022188622487f, 7.2946207488916270f, 7.3037807481771031f, 127 | 7.3128829552843557f, 7.3219280948873617f, 7.3309168781146177f, 128 | 7.3398500028846243f, 7.3487281542310781f, 7.3575520046180847f, 129 | 7.3663222142458151f, 7.3750394313469254f, 7.3837042924740528f, 130 | 7.3923174227787607f, 7.4008794362821844f, 7.4093909361377026f, 131 | 7.4178525148858991f, 7.4262647547020979f, 7.4346282276367255f, 132 | 7.4429434958487288f, 7.4512111118323299f, 7.4594316186372973f, 133 | 7.4676055500829976f, 7.4757334309663976f, 7.4838157772642564f, 134 | 7.4918530963296748f, 7.4998458870832057f, 7.5077946401986964f, 135 | 7.5156998382840436f, 7.5235619560570131f, 7.5313814605163119f, 136 | 7.5391588111080319f, 7.5468944598876373f, 7.5545888516776376f, 137 | 7.5622424242210728f, 7.5698556083309478f, 7.5774288280357487f, 138 | 7.5849625007211561f, 7.5924570372680806f, 7.5999128421871278f, 139 | 7.6073303137496113f, 7.6147098441152075f, 7.6220518194563764f, 140 | 7.6293566200796095f, 7.6366246205436488f, 7.6438561897747244f, 141 | 7.6510516911789290f, 7.6582114827517955f, 7.6653359171851765f, 142 | 7.6724253419714952f, 7.6794800995054464f, 7.6865005271832185f, 143 | 7.6934869574993252f, 7.7004397181410926f, 7.7073591320808825f, 144 | 7.7142455176661224f, 7.7210991887071856f, 7.7279204545631996f, 145 | 7.7347096202258392f, 7.7414669864011465f, 7.7481928495894596f, 146 | 7.7548875021634691f, 7.7615512324444795f, 7.7681843247769260f, 147 | 7.7747870596011737f, 7.7813597135246608f, 7.7879025593914317f, 148 | 7.7944158663501062f, 7.8008998999203047f, 7.8073549220576037f, 149 | 7.8137811912170374f, 7.8201789624151887f, 7.8265484872909159f, 150 | 7.8328900141647422f, 7.8392037880969445f, 7.8454900509443757f, 151 | 7.8517490414160571f, 7.8579809951275719f, 7.8641861446542798f, 152 | 7.8703647195834048f, 7.8765169465650002f, 7.8826430493618425f, 153 | 7.8887432488982601f, 7.8948177633079446f, 7.9008668079807496f, 154 | 7.9068905956085187f, 7.9128893362299619f, 7.9188632372745955f, 155 | 7.9248125036057813f, 7.9307373375628867f, 7.9366379390025719f, 156 | 7.9425145053392399f, 7.9483672315846778f, 7.9541963103868758f, 157 | 7.9600019320680806f, 7.9657842846620870f, 7.9715435539507720f, 158 | 7.9772799234999168f, 7.9829935746943104f, 7.9886846867721664f, 159 | 7.9943534368588578f 160 | }; 161 | 162 | // Faster logarithm for small integers, with the property of log2(0) == 0. 163 | static inline double FastLog2(int v) { 164 | if (v < (int)(sizeof(kLog2Table) / sizeof(kLog2Table[0]))) { 165 | return kLog2Table[v]; 166 | } 167 | #if defined(_MSC_VER) && _MSC_VER <= 1600 168 | // Visual Studio 2010 does not have the log2() function defined, so we use 169 | // log() and a multiplication instead. 170 | static const double kLog2Inv = 1.4426950408889634f; 171 | return log(static_cast(v)) * kLog2Inv; 172 | #else 173 | return log2(static_cast(v)); 174 | #endif 175 | } 176 | 177 | } // namespace brotli 178 | 179 | #endif // BROTLI_ENC_FAST_LOG_H_ 180 | -------------------------------------------------------------------------------- /csrc/enc/find_match_length.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Function to find maximal matching prefixes of strings. 16 | 17 | #ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_ 18 | #define BROTLI_ENC_FIND_MATCH_LENGTH_H_ 19 | 20 | #include 21 | 22 | #include 23 | 24 | #include "./port.h" 25 | 26 | namespace brotli { 27 | 28 | // Separate implementation for little-endian 64-bit targets, for speed. 29 | #if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN) 30 | 31 | static inline int FindMatchLengthWithLimit(const uint8_t* s1, 32 | const uint8_t* s2, 33 | size_t limit) { 34 | int matched = 0; 35 | size_t limit2 = (limit >> 3) + 1; // + 1 is for pre-decrement in while 36 | while (PREDICT_TRUE(--limit2)) { 37 | if (PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) == 38 | BROTLI_UNALIGNED_LOAD64(s1 + matched))) { 39 | s2 += 8; 40 | matched += 8; 41 | } else { 42 | uint64_t x = 43 | BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched); 44 | int matching_bits = __builtin_ctzll(x); 45 | matched += matching_bits >> 3; 46 | return matched; 47 | } 48 | } 49 | limit = (limit & 7) + 1; // + 1 is for pre-decrement in while 50 | while (--limit) { 51 | if (PREDICT_TRUE(s1[matched] == *s2)) { 52 | ++s2; 53 | ++matched; 54 | } else { 55 | return matched; 56 | } 57 | } 58 | return matched; 59 | } 60 | #else 61 | static inline int FindMatchLengthWithLimit(const uint8_t* s1, 62 | const uint8_t* s2, 63 | size_t limit) { 64 | int matched = 0; 65 | const uint8_t* s2_limit = s2 + limit; 66 | const uint8_t* s2_ptr = s2; 67 | // Find out how long the match is. We loop over the data 32 bits at a 68 | // time until we find a 32-bit block that doesn't match; then we find 69 | // the first non-matching bit and use that to calculate the total 70 | // length of the match. 71 | while (s2_ptr <= s2_limit - 4 && 72 | BROTLI_UNALIGNED_LOAD32(s2_ptr) == 73 | BROTLI_UNALIGNED_LOAD32(s1 + matched)) { 74 | s2_ptr += 4; 75 | matched += 4; 76 | } 77 | while ((s2_ptr < s2_limit) && (s1[matched] == *s2_ptr)) { 78 | ++s2_ptr; 79 | ++matched; 80 | } 81 | return matched; 82 | } 83 | #endif 84 | 85 | } // namespace brotli 86 | 87 | #endif // BROTLI_ENC_FIND_MATCH_LENGTH_H_ 88 | -------------------------------------------------------------------------------- /csrc/enc/histogram.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Build per-context histograms of literals, commands and distance codes. 16 | 17 | #include "./histogram.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "./block_splitter.h" 23 | #include "./command.h" 24 | #include "./context.h" 25 | #include "./prefix.h" 26 | 27 | namespace brotli { 28 | 29 | void BuildHistograms( 30 | const Command* cmds, 31 | const size_t num_commands, 32 | const BlockSplit& literal_split, 33 | const BlockSplit& insert_and_copy_split, 34 | const BlockSplit& dist_split, 35 | const uint8_t* ringbuffer, 36 | size_t start_pos, 37 | size_t mask, 38 | uint8_t prev_byte, 39 | uint8_t prev_byte2, 40 | const std::vector& context_modes, 41 | std::vector* literal_histograms, 42 | std::vector* insert_and_copy_histograms, 43 | std::vector* copy_dist_histograms) { 44 | size_t pos = start_pos; 45 | BlockSplitIterator literal_it(literal_split); 46 | BlockSplitIterator insert_and_copy_it(insert_and_copy_split); 47 | BlockSplitIterator dist_it(dist_split); 48 | for (int i = 0; i < num_commands; ++i) { 49 | const Command &cmd = cmds[i]; 50 | insert_and_copy_it.Next(); 51 | (*insert_and_copy_histograms)[insert_and_copy_it.type_].Add( 52 | cmd.cmd_prefix_); 53 | for (int j = 0; j < cmd.insert_len_; ++j) { 54 | literal_it.Next(); 55 | int context = (literal_it.type_ << kLiteralContextBits) + 56 | Context(prev_byte, prev_byte2, context_modes[literal_it.type_]); 57 | (*literal_histograms)[context].Add(ringbuffer[pos & mask]); 58 | prev_byte2 = prev_byte; 59 | prev_byte = ringbuffer[pos & mask]; 60 | ++pos; 61 | } 62 | pos += cmd.copy_len_; 63 | if (cmd.copy_len_ > 0) { 64 | prev_byte2 = ringbuffer[(pos - 2) & mask]; 65 | prev_byte = ringbuffer[(pos - 1) & mask]; 66 | if (cmd.cmd_prefix_ >= 128) { 67 | dist_it.Next(); 68 | int context = (dist_it.type_ << kDistanceContextBits) + 69 | cmd.DistanceContext(); 70 | (*copy_dist_histograms)[context].Add(cmd.dist_prefix_); 71 | } 72 | } 73 | } 74 | } 75 | 76 | } // namespace brotli 77 | -------------------------------------------------------------------------------- /csrc/enc/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Models the histograms of literals, commands and distance codes. 16 | 17 | #ifndef BROTLI_ENC_HISTOGRAM_H_ 18 | #define BROTLI_ENC_HISTOGRAM_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "./command.h" 25 | #include "./fast_log.h" 26 | #include "./prefix.h" 27 | 28 | namespace brotli { 29 | 30 | class BlockSplit; 31 | 32 | // A simple container for histograms of data in blocks. 33 | template 34 | struct Histogram { 35 | Histogram() { 36 | Clear(); 37 | } 38 | void Clear() { 39 | memset(data_, 0, sizeof(data_)); 40 | total_count_ = 0; 41 | } 42 | void Add(int val) { 43 | ++data_[val]; 44 | ++total_count_; 45 | } 46 | void Remove(int val) { 47 | --data_[val]; 48 | --total_count_; 49 | } 50 | template 51 | void Add(const DataType *p, size_t n) { 52 | total_count_ += n; 53 | n += 1; 54 | while(--n) ++data_[*p++]; 55 | } 56 | void AddHistogram(const Histogram& v) { 57 | total_count_ += v.total_count_; 58 | for (int i = 0; i < kDataSize; ++i) { 59 | data_[i] += v.data_[i]; 60 | } 61 | } 62 | double EntropyBitCost() const { 63 | double retval = total_count_ * FastLog2(total_count_); 64 | for (int i = 0; i < kDataSize; ++i) { 65 | retval -= data_[i] * FastLog2(data_[i]); 66 | } 67 | return retval; 68 | } 69 | 70 | int data_[kDataSize]; 71 | int total_count_; 72 | double bit_cost_; 73 | }; 74 | 75 | // Literal histogram. 76 | typedef Histogram<256> HistogramLiteral; 77 | // Prefix histograms. 78 | typedef Histogram HistogramCommand; 79 | typedef Histogram HistogramDistance; 80 | typedef Histogram HistogramBlockLength; 81 | // Context map histogram, 256 Huffman tree indexes + 16 run length codes. 82 | typedef Histogram<272> HistogramContextMap; 83 | // Block type histogram, 256 block types + 2 special symbols. 84 | typedef Histogram<258> HistogramBlockType; 85 | 86 | static const int kLiteralContextBits = 6; 87 | static const int kDistanceContextBits = 2; 88 | 89 | void BuildHistograms( 90 | const Command* cmds, 91 | const size_t num_commands, 92 | const BlockSplit& literal_split, 93 | const BlockSplit& insert_and_copy_split, 94 | const BlockSplit& dist_split, 95 | const uint8_t* ringbuffer, 96 | size_t pos, 97 | size_t mask, 98 | uint8_t prev_byte, 99 | uint8_t prev_byte2, 100 | const std::vector& context_modes, 101 | std::vector* literal_histograms, 102 | std::vector* insert_and_copy_histograms, 103 | std::vector* copy_dist_histograms); 104 | 105 | } // namespace brotli 106 | 107 | #endif // BROTLI_ENC_HISTOGRAM_H_ 108 | -------------------------------------------------------------------------------- /csrc/enc/literal_cost.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Literal cost model to allow backward reference replacement to be efficient. 16 | 17 | #include "./literal_cost.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "./fast_log.h" 24 | 25 | namespace brotli { 26 | 27 | static int UTF8Position(int last, int c, int clamp) { 28 | if (c < 128) { 29 | return 0; // Next one is the 'Byte 1' again. 30 | } else if (c >= 192) { 31 | return std::min(1, clamp); // Next one is the 'Byte 2' of utf-8 encoding. 32 | } else { 33 | // Let's decide over the last byte if this ends the sequence. 34 | if (last < 0xe0) { 35 | return 0; // Completed two or three byte coding. 36 | } else { 37 | return std::min(2, clamp); // Next one is the 'Byte 3' of utf-8 encoding. 38 | } 39 | } 40 | } 41 | 42 | static int DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask, 43 | const uint8_t *data) { 44 | int counts[3] = { 0 }; 45 | int max_utf8 = 1; // should be 2, but 1 compresses better. 46 | int last_c = 0; 47 | int utf8_pos = 0; 48 | for (int i = 0; i < len; ++i) { 49 | int c = data[(pos + i) & mask]; 50 | utf8_pos = UTF8Position(last_c, c, 2); 51 | ++counts[utf8_pos]; 52 | last_c = c; 53 | } 54 | if (counts[2] < 500) { 55 | max_utf8 = 1; 56 | } 57 | if (counts[1] + counts[2] < 25) { 58 | max_utf8 = 0; 59 | } 60 | return max_utf8; 61 | } 62 | 63 | void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, 64 | size_t cost_mask, const uint8_t *data, 65 | float *cost) { 66 | 67 | // max_utf8 is 0 (normal ascii single byte modeling), 68 | // 1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). 69 | const int max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); 70 | int histogram[3][256] = { { 0 } }; 71 | int window_half = 495; 72 | int in_window = std::min(static_cast(window_half), len); 73 | int in_window_utf8[3] = { 0 }; 74 | 75 | // Bootstrap histograms. 76 | int last_c = 0; 77 | int utf8_pos = 0; 78 | for (int i = 0; i < in_window; ++i) { 79 | int c = data[(pos + i) & mask]; 80 | ++histogram[utf8_pos][c]; 81 | ++in_window_utf8[utf8_pos]; 82 | utf8_pos = UTF8Position(last_c, c, max_utf8); 83 | last_c = c; 84 | } 85 | 86 | // Compute bit costs with sliding window. 87 | for (int i = 0; i < len; ++i) { 88 | if (i - window_half >= 0) { 89 | // Remove a byte in the past. 90 | int c = (i - window_half - 1) < 0 ? 91 | 0 : data[(pos + i - window_half - 1) & mask]; 92 | int last_c = (i - window_half - 2) < 0 ? 93 | 0 : data[(pos + i - window_half - 2) & mask]; 94 | int utf8_pos2 = UTF8Position(last_c, c, max_utf8); 95 | --histogram[utf8_pos2][data[(pos + i - window_half) & mask]]; 96 | --in_window_utf8[utf8_pos2]; 97 | } 98 | if (i + window_half < len) { 99 | // Add a byte in the future. 100 | int c = (i + window_half - 1) < 0 ? 101 | 0 : data[(pos + i + window_half - 1) & mask]; 102 | int last_c = (i + window_half - 2) < 0 ? 103 | 0 : data[(pos + i + window_half - 2) & mask]; 104 | int utf8_pos2 = UTF8Position(last_c, c, max_utf8); 105 | ++histogram[utf8_pos2][data[(pos + i + window_half) & mask]]; 106 | ++in_window_utf8[utf8_pos2]; 107 | } 108 | int c = i < 1 ? 0 : data[(pos + i - 1) & mask]; 109 | int last_c = i < 2 ? 0 : data[(pos + i - 2) & mask]; 110 | int utf8_pos = UTF8Position(last_c, c, max_utf8); 111 | int masked_pos = (pos + i) & mask; 112 | int histo = histogram[utf8_pos][data[masked_pos]]; 113 | if (histo == 0) { 114 | histo = 1; 115 | } 116 | float lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo); 117 | lit_cost += 0.02905; 118 | if (lit_cost < 1.0) { 119 | lit_cost *= 0.5; 120 | lit_cost += 0.5; 121 | } 122 | // Make the first bytes more expensive -- seems to help, not sure why. 123 | // Perhaps because the entropy source is changing its properties 124 | // rapidly in the beginning of the file, perhaps because the beginning 125 | // of the data is a statistical "anomaly". 126 | if (i < 2000) { 127 | lit_cost += 0.7 - ((2000 - i) / 2000.0 * 0.35); 128 | } 129 | cost[(pos + i) & cost_mask] = lit_cost; 130 | } 131 | } 132 | 133 | void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, 134 | size_t cost_mask, const uint8_t *data, 135 | float *cost) { 136 | int histogram[256] = { 0 }; 137 | int window_half = 2000; 138 | int in_window = std::min(static_cast(window_half), len); 139 | 140 | // Bootstrap histogram. 141 | for (int i = 0; i < in_window; ++i) { 142 | ++histogram[data[(pos + i) & mask]]; 143 | } 144 | 145 | // Compute bit costs with sliding window. 146 | for (int i = 0; i < len; ++i) { 147 | if (i - window_half >= 0) { 148 | // Remove a byte in the past. 149 | --histogram[data[(pos + i - window_half) & mask]]; 150 | --in_window; 151 | } 152 | if (i + window_half < len) { 153 | // Add a byte in the future. 154 | ++histogram[data[(pos + i + window_half) & mask]]; 155 | ++in_window; 156 | } 157 | int histo = histogram[data[(pos + i) & mask]]; 158 | if (histo == 0) { 159 | histo = 1; 160 | } 161 | float lit_cost = FastLog2(in_window) - FastLog2(histo); 162 | lit_cost += 0.029; 163 | if (lit_cost < 1.0) { 164 | lit_cost *= 0.5; 165 | lit_cost += 0.5; 166 | } 167 | cost[(pos + i) & cost_mask] = lit_cost; 168 | } 169 | } 170 | 171 | 172 | } // namespace brotli 173 | -------------------------------------------------------------------------------- /csrc/enc/literal_cost.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Literal cost model to allow backward reference replacement to be efficient. 16 | 17 | #ifndef BROTLI_ENC_LITERAL_COST_H_ 18 | #define BROTLI_ENC_LITERAL_COST_H_ 19 | 20 | #include 21 | #include 22 | 23 | namespace brotli { 24 | 25 | // Estimates how many bits the literals in the interval [pos, pos + len) in the 26 | // ringbuffer (data, mask) will take entropy coded and writes these estimates 27 | // to the ringbuffer (cost, mask). 28 | void EstimateBitCostsForLiterals(size_t pos, size_t len, size_t mask, 29 | size_t cost_mask, const uint8_t *data, 30 | float *cost); 31 | 32 | void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, 33 | size_t cost_mask, const uint8_t *data, 34 | float *cost); 35 | 36 | } // namespace brotli 37 | 38 | #endif // BROTLI_ENC_LITERAL_COST_H_ 39 | -------------------------------------------------------------------------------- /csrc/enc/metablock.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Algorithms for distributing the literals and commands of a metablock between 16 | // block types and contexts. 17 | 18 | #ifndef BROTLI_ENC_METABLOCK_H_ 19 | #define BROTLI_ENC_METABLOCK_H_ 20 | 21 | #include 22 | 23 | #include "./command.h" 24 | #include "./histogram.h" 25 | 26 | namespace brotli { 27 | 28 | struct BlockSplit { 29 | BlockSplit() : num_types(0) {} 30 | 31 | int num_types; 32 | std::vector types; 33 | std::vector lengths; 34 | }; 35 | 36 | struct MetaBlockSplit { 37 | BlockSplit literal_split; 38 | BlockSplit command_split; 39 | BlockSplit distance_split; 40 | std::vector literal_context_map; 41 | std::vector distance_context_map; 42 | std::vector literal_histograms; 43 | std::vector command_histograms; 44 | std::vector distance_histograms; 45 | }; 46 | 47 | void BuildMetaBlock(const uint8_t* ringbuffer, 48 | const size_t pos, 49 | const size_t mask, 50 | uint8_t prev_byte, 51 | uint8_t prev_byte2, 52 | const Command* cmds, 53 | size_t num_commands, 54 | int literal_context_mode, 55 | bool enable_context_modleing, 56 | MetaBlockSplit* mb); 57 | 58 | void BuildMetaBlockGreedy(const uint8_t* ringbuffer, 59 | size_t pos, 60 | size_t mask, 61 | const Command *commands, 62 | size_t n_commands, 63 | MetaBlockSplit* mb); 64 | 65 | void OptimizeHistograms(int num_direct_distance_codes, 66 | int distance_postfix_bits, 67 | MetaBlockSplit* mb); 68 | 69 | } // namespace brotli 70 | 71 | #endif // BROTLI_ENC_METABLOCK_H_ 72 | -------------------------------------------------------------------------------- /csrc/enc/port.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Macros for endianness, branch prediction and unaligned loads and stores. 16 | 17 | #ifndef BROTLI_ENC_PORT_H_ 18 | #define BROTLI_ENC_PORT_H_ 19 | 20 | #if defined OS_LINUX || defined OS_CYGWIN 21 | #include 22 | #elif defined OS_FREEBSD 23 | #include 24 | #elif defined OS_MACOSX 25 | #include 26 | /* Let's try and follow the Linux convention */ 27 | #define __BYTE_ORDER BYTE_ORDER 28 | #define __LITTLE_ENDIAN LITTLE_ENDIAN 29 | #define __BIG_ENDIAN BIG_ENDIAN 30 | #endif 31 | 32 | // define the macros IS_LITTLE_ENDIAN or IS_BIG_ENDIAN 33 | // using the above endian definitions from endian.h if 34 | // endian.h was included 35 | #ifdef __BYTE_ORDER 36 | #if __BYTE_ORDER == __LITTLE_ENDIAN 37 | #define IS_LITTLE_ENDIAN 38 | #endif 39 | 40 | #if __BYTE_ORDER == __BIG_ENDIAN 41 | #define IS_BIG_ENDIAN 42 | #endif 43 | 44 | #else 45 | 46 | #if defined(__LITTLE_ENDIAN__) 47 | #define IS_LITTLE_ENDIAN 48 | #elif defined(__BIG_ENDIAN__) 49 | #define IS_BIG_ENDIAN 50 | #endif 51 | #endif // __BYTE_ORDER 52 | 53 | // Enable little-endian optimization for x64 architecture on Windows. 54 | #if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64) 55 | #define IS_LITTLE_ENDIAN 56 | #endif 57 | 58 | #if defined(COMPILER_GCC3) 59 | #define PREDICT_FALSE(x) (__builtin_expect(x, 0)) 60 | #define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) 61 | #else 62 | #define PREDICT_FALSE(x) x 63 | #define PREDICT_TRUE(x) x 64 | #endif 65 | 66 | // Portable handling of unaligned loads, stores, and copies. 67 | // On some platforms, like ARM, the copy functions can be more efficient 68 | // then a load and a store. 69 | 70 | #if defined(ARCH_PIII) || defined(ARCH_ATHLON) || \ 71 | defined(ARCH_K8) || defined(_ARCH_PPC) 72 | 73 | // x86 and x86-64 can perform unaligned loads/stores directly; 74 | // modern PowerPC hardware can also do unaligned integer loads and stores; 75 | // but note: the FPU still sends unaligned loads and stores to a trap handler! 76 | 77 | #define BROTLI_UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) 78 | #define BROTLI_UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) 79 | 80 | #define BROTLI_UNALIGNED_STORE32(_p, _val) \ 81 | (*reinterpret_cast(_p) = (_val)) 82 | #define BROTLI_UNALIGNED_STORE64(_p, _val) \ 83 | (*reinterpret_cast(_p) = (_val)) 84 | 85 | #elif defined(__arm__) && \ 86 | !defined(__ARM_ARCH_5__) && \ 87 | !defined(__ARM_ARCH_5T__) && \ 88 | !defined(__ARM_ARCH_5TE__) && \ 89 | !defined(__ARM_ARCH_5TEJ__) && \ 90 | !defined(__ARM_ARCH_6__) && \ 91 | !defined(__ARM_ARCH_6J__) && \ 92 | !defined(__ARM_ARCH_6K__) && \ 93 | !defined(__ARM_ARCH_6Z__) && \ 94 | !defined(__ARM_ARCH_6ZK__) && \ 95 | !defined(__ARM_ARCH_6T2__) 96 | 97 | // ARMv7 and newer support native unaligned accesses, but only of 16-bit 98 | // and 32-bit values (not 64-bit); older versions either raise a fatal signal, 99 | // do an unaligned read and rotate the words around a bit, or do the reads very 100 | // slowly (trip through kernel mode). 101 | 102 | #define BROTLI_UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) 103 | #define BROTLI_UNALIGNED_STORE32(_p, _val) \ 104 | (*reinterpret_cast(_p) = (_val)) 105 | 106 | inline uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) { 107 | uint64_t t; 108 | memcpy(&t, p, sizeof t); 109 | return t; 110 | } 111 | 112 | inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) { 113 | memcpy(p, &v, sizeof v); 114 | } 115 | 116 | #else 117 | 118 | // These functions are provided for architectures that don't support 119 | // unaligned loads and stores. 120 | 121 | inline uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) { 122 | uint32_t t; 123 | memcpy(&t, p, sizeof t); 124 | return t; 125 | } 126 | 127 | inline uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) { 128 | uint64_t t; 129 | memcpy(&t, p, sizeof t); 130 | return t; 131 | } 132 | 133 | inline void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) { 134 | memcpy(p, &v, sizeof v); 135 | } 136 | 137 | inline void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) { 138 | memcpy(p, &v, sizeof v); 139 | } 140 | 141 | #endif 142 | 143 | #endif // BROTLI_ENC_PORT_H_ 144 | -------------------------------------------------------------------------------- /csrc/enc/prefix.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions for encoding of integers into prefix codes the amount of extra 16 | // bits, and the actual values of the extra bits. 17 | 18 | #ifndef BROTLI_ENC_PREFIX_H_ 19 | #define BROTLI_ENC_PREFIX_H_ 20 | 21 | #include 22 | #include "./fast_log.h" 23 | 24 | namespace brotli { 25 | 26 | static const int kNumInsertLenPrefixes = 24; 27 | static const int kNumCopyLenPrefixes = 24; 28 | static const int kNumCommandPrefixes = 704; 29 | static const int kNumBlockLenPrefixes = 26; 30 | static const int kNumDistanceShortCodes = 16; 31 | static const int kNumDistancePrefixes = 520; 32 | 33 | // Represents the range of values belonging to a prefix code: 34 | // [offset, offset + 2^nbits) 35 | struct PrefixCodeRange { 36 | int offset; 37 | int nbits; 38 | }; 39 | 40 | static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = { 41 | { 1, 2}, { 5, 2}, { 9, 2}, { 13, 2}, 42 | { 17, 3}, { 25, 3}, { 33, 3}, { 41, 3}, 43 | { 49, 4}, { 65, 4}, { 81, 4}, { 97, 4}, 44 | { 113, 5}, { 145, 5}, { 177, 5}, { 209, 5}, 45 | { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8}, 46 | { 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12}, 47 | {8433, 13}, {16625, 24} 48 | }; 49 | 50 | inline void GetBlockLengthPrefixCode(int len, 51 | int* code, int* n_extra, int* extra) { 52 | *code = 0; 53 | while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) { 54 | ++(*code); 55 | } 56 | *n_extra = kBlockLengthPrefixCode[*code].nbits; 57 | *extra = len - kBlockLengthPrefixCode[*code].offset; 58 | } 59 | 60 | inline void PrefixEncodeCopyDistance(int distance_code, 61 | int num_direct_codes, 62 | int postfix_bits, 63 | uint16_t* code, 64 | uint32_t* extra_bits) { 65 | distance_code -= 1; 66 | if (distance_code < kNumDistanceShortCodes + num_direct_codes) { 67 | *code = distance_code; 68 | *extra_bits = 0; 69 | return; 70 | } 71 | distance_code -= kNumDistanceShortCodes + num_direct_codes; 72 | distance_code += (1 << (postfix_bits + 2)); 73 | int bucket = Log2Floor(distance_code) - 1; 74 | int postfix_mask = (1 << postfix_bits) - 1; 75 | int postfix = distance_code & postfix_mask; 76 | int prefix = (distance_code >> bucket) & 1; 77 | int offset = (2 + prefix) << bucket; 78 | int nbits = bucket - postfix_bits; 79 | *code = kNumDistanceShortCodes + num_direct_codes + 80 | ((2 * (nbits - 1) + prefix) << postfix_bits) + postfix; 81 | *extra_bits = (nbits << 24) | ((distance_code - offset) >> postfix_bits); 82 | } 83 | 84 | } // namespace brotli 85 | 86 | #endif // BROTLI_ENC_PREFIX_H_ 87 | -------------------------------------------------------------------------------- /csrc/enc/ringbuffer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Sliding window over the input data. 16 | 17 | #ifndef BROTLI_ENC_RINGBUFFER_H_ 18 | #define BROTLI_ENC_RINGBUFFER_H_ 19 | 20 | #include 21 | #include 22 | 23 | #include "./port.h" 24 | 25 | namespace brotli { 26 | 27 | // A RingBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of 28 | // data in a circular manner: writing a byte writes it to 29 | // `position() % (1 << window_bits)'. For convenience, the RingBuffer array 30 | // contains another copy of the first `1 << tail_bits' bytes: 31 | // buffer_[i] == buffer_[i + (1 << window_bits)] if i < (1 << tail_bits). 32 | class RingBuffer { 33 | public: 34 | RingBuffer(int window_bits, int tail_bits) 35 | : window_bits_(window_bits), 36 | mask_((1 << window_bits) - 1), 37 | tail_size_(1 << tail_bits), 38 | pos_(0) { 39 | static const int kSlackForFourByteHashingEverywhere = 3; 40 | const int buflen = (1 << window_bits_) + tail_size_; 41 | buffer_ = new uint8_t[buflen + kSlackForFourByteHashingEverywhere]; 42 | for (int i = 0; i < kSlackForFourByteHashingEverywhere; ++i) { 43 | buffer_[buflen + i] = 0; 44 | } 45 | } 46 | ~RingBuffer() { 47 | delete [] buffer_; 48 | } 49 | 50 | // Push bytes into the ring buffer. 51 | void Write(const uint8_t *bytes, size_t n) { 52 | const size_t masked_pos = pos_ & mask_; 53 | // The length of the writes is limited so that we do not need to worry 54 | // about a write 55 | WriteTail(bytes, n); 56 | if (PREDICT_TRUE(masked_pos + n <= (1 << window_bits_))) { 57 | // A single write fits. 58 | memcpy(&buffer_[masked_pos], bytes, n); 59 | } else { 60 | // Split into two writes. 61 | // Copy into the end of the buffer, including the tail buffer. 62 | memcpy(&buffer_[masked_pos], bytes, 63 | std::min(n, ((1 << window_bits_) + tail_size_) - masked_pos)); 64 | // Copy into the begining of the buffer 65 | memcpy(&buffer_[0], bytes + ((1 << window_bits_) - masked_pos), 66 | n - ((1 << window_bits_) - masked_pos)); 67 | } 68 | pos_ += n; 69 | } 70 | 71 | void Reset() { 72 | pos_ = 0; 73 | } 74 | 75 | // Logical cursor position in the ring buffer. 76 | size_t position() const { return pos_; } 77 | 78 | // Bit mask for getting the physical position for a logical position. 79 | size_t mask() const { return mask_; } 80 | 81 | uint8_t *start() { return &buffer_[0]; } 82 | const uint8_t *start() const { return &buffer_[0]; } 83 | 84 | private: 85 | void WriteTail(const uint8_t *bytes, size_t n) { 86 | const size_t masked_pos = pos_ & mask_; 87 | if (PREDICT_FALSE(masked_pos < tail_size_)) { 88 | // Just fill the tail buffer with the beginning data. 89 | const size_t p = (1 << window_bits_) + masked_pos; 90 | memcpy(&buffer_[p], bytes, std::min(n, tail_size_ - masked_pos)); 91 | } 92 | } 93 | 94 | // Size of the ringbuffer is (1 << window_bits) + tail_size_. 95 | const int window_bits_; 96 | const size_t mask_; 97 | const size_t tail_size_; 98 | 99 | // Position to write in the ring buffer. 100 | size_t pos_; 101 | // The actual ring buffer containing the data and the copy of the beginning 102 | // as a tail. 103 | uint8_t *buffer_; 104 | }; 105 | 106 | } // namespace brotli 107 | 108 | #endif // BROTLI_ENC_RINGBUFFER_H_ 109 | -------------------------------------------------------------------------------- /csrc/enc/static_dict.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Class to model the static dictionary. 16 | 17 | #ifndef BROTLI_ENC_STATIC_DICT_H_ 18 | #define BROTLI_ENC_STATIC_DICT_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "./dictionary.h" 25 | #include "./transform.h" 26 | 27 | namespace brotli { 28 | 29 | class StaticDictionary { 30 | public: 31 | StaticDictionary() {} 32 | void Fill(bool enable_transforms) { 33 | const int num_transforms = enable_transforms ? kNumTransforms : 1; 34 | for (int t = num_transforms - 1; t >= 0; --t) { 35 | for (int i = kMaxDictionaryWordLength; 36 | i >= kMinDictionaryWordLength; --i) { 37 | const int num_words = 1 << kBrotliDictionarySizeBitsByLength[i]; 38 | for (int j = num_words - 1; j >= 0; --j) { 39 | int word_id = t * num_words + j; 40 | std::string word = GetTransformedDictionaryWord(i, word_id); 41 | if (word.size() >= 4) { 42 | Insert(word, i, word_id); 43 | } 44 | } 45 | } 46 | } 47 | } 48 | void Insert(const std::string &str, int len, int dist) { 49 | int ix = (dist << 6) + len; 50 | std::unordered_map::const_iterator it = map_.find(str); 51 | if (it != map_.end() && ix >= it->second) { 52 | return; 53 | } 54 | map_[str] = ix; 55 | uint32_t v = 0; 56 | for (int i = 0; i < 4 && i < str.size(); ++i) { 57 | v += static_cast(str[i]) << (8 * i); 58 | } 59 | if (prefix_map_[v] < str.size()) { 60 | prefix_map_[v] = str.size(); 61 | } 62 | } 63 | int GetLength(uint32_t v) const { 64 | std::unordered_map::const_iterator it = prefix_map_.find(v); 65 | if (it == prefix_map_.end()) { 66 | return 0; 67 | } 68 | return it->second; 69 | } 70 | bool Get(const std::string &str, int *len, int *dist) const { 71 | std::unordered_map::const_iterator it = map_.find(str); 72 | if (it == map_.end()) { 73 | return false; 74 | } 75 | int v = it->second; 76 | *len = v & 63; 77 | *dist = v >> 6; 78 | return true; 79 | } 80 | private: 81 | std::unordered_map map_; 82 | std::unordered_map prefix_map_; 83 | }; 84 | 85 | } // namespace brotli 86 | 87 | #endif // BROTLI_ENC_STATIC_DICT_H_ 88 | -------------------------------------------------------------------------------- /csrc/enc/streams.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2009 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Convience routines to make Brotli I/O classes from some memory containers and 16 | // files. 17 | 18 | #include "./streams.h" 19 | 20 | #include 21 | #include 22 | 23 | namespace brotli { 24 | 25 | BrotliMemOut::BrotliMemOut(void* buf, int len) 26 | : buf_(buf), 27 | len_(len), 28 | pos_(0) {} 29 | 30 | void BrotliMemOut::Reset(void* buf, int len) { 31 | buf_ = buf; 32 | len_ = len; 33 | pos_ = 0; 34 | } 35 | 36 | // Brotli output routine: copy n bytes to the output buffer. 37 | bool BrotliMemOut::Write(const void *buf, size_t n) { 38 | if (n + pos_ > len_) 39 | return false; 40 | char* p = reinterpret_cast(buf_) + pos_; 41 | memcpy(p, buf, n); 42 | pos_ += n; 43 | return true; 44 | } 45 | 46 | BrotliStringOut::BrotliStringOut(std::string* buf, int max_size) 47 | : buf_(buf), 48 | max_size_(max_size) { 49 | assert(buf->empty()); 50 | } 51 | 52 | void BrotliStringOut::Reset(std::string* buf, int max_size) { 53 | buf_ = buf; 54 | max_size_ = max_size; 55 | } 56 | 57 | // Brotli output routine: add n bytes to a string. 58 | bool BrotliStringOut::Write(const void *buf, size_t n) { 59 | if (buf_->size() + n > max_size_) 60 | return false; 61 | buf_->append(static_cast(buf), n); 62 | return true; 63 | } 64 | 65 | BrotliMemIn::BrotliMemIn(const void* buf, int len) 66 | : buf_(buf), 67 | len_(len), 68 | pos_(0) {} 69 | 70 | void BrotliMemIn::Reset(const void* buf, int len) { 71 | buf_ = buf; 72 | len_ = len; 73 | pos_ = 0; 74 | } 75 | 76 | // Brotli input routine: read the next chunk of memory. 77 | const void* BrotliMemIn::Read(size_t n, size_t* output) { 78 | if (pos_ == len_) { 79 | return NULL; 80 | } 81 | if (n > len_ - pos_) 82 | n = len_ - pos_; 83 | const char* p = reinterpret_cast(buf_) + pos_; 84 | pos_ += n; 85 | *output = n; 86 | return p; 87 | } 88 | 89 | BrotliFileIn::BrotliFileIn(FILE* f, size_t max_read_size) 90 | : f_(f), 91 | buf_(malloc(max_read_size)), 92 | buf_size_(max_read_size) {} 93 | 94 | BrotliFileIn::~BrotliFileIn() { 95 | if (buf_) free(buf_); 96 | } 97 | 98 | const void* BrotliFileIn::Read(size_t n, size_t* bytes_read) { 99 | if (buf_ == NULL) { 100 | *bytes_read = 0; 101 | return NULL; 102 | } 103 | if (n > buf_size_) { 104 | n = buf_size_; 105 | } else if (n == 0) { 106 | return feof(f_) ? NULL : buf_; 107 | } 108 | *bytes_read = fread(buf_, 1, n, f_); 109 | if (*bytes_read == 0) { 110 | return NULL; 111 | } else { 112 | return buf_; 113 | } 114 | } 115 | 116 | BrotliFileOut::BrotliFileOut(FILE* f) : f_(f) {} 117 | 118 | bool BrotliFileOut::Write(const void* buf, size_t n) { 119 | if (fwrite(buf, n, 1, f_) != 1) { 120 | return false; 121 | } 122 | return true; 123 | } 124 | 125 | 126 | } // namespace brotli 127 | -------------------------------------------------------------------------------- /csrc/enc/streams.h: -------------------------------------------------------------------------------- 1 | // Copyright 2009 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Input and output classes for streaming brotli compression. 16 | 17 | #ifndef BROTLI_ENC_STREAMS_H_ 18 | #define BROTLI_ENC_STREAMS_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace brotli { 26 | 27 | // Input interface for the compression routines. 28 | class BrotliIn { 29 | public: 30 | virtual ~BrotliIn() {} 31 | 32 | // Return a pointer to the next block of input of at most n bytes. 33 | // Return the actual length in *nread. 34 | // At end of data, return NULL. Don't return NULL if there is more data 35 | // to read, even if called with n == 0. 36 | // Read will only be called if some of its bytes are needed. 37 | virtual const void* Read(size_t n, size_t* nread) = 0; 38 | }; 39 | 40 | // Output interface for the compression routines. 41 | class BrotliOut { 42 | public: 43 | virtual ~BrotliOut() {} 44 | 45 | // Write n bytes of data from buf. 46 | // Return true if all written, false otherwise. 47 | virtual bool Write(const void *buf, size_t n) = 0; 48 | }; 49 | 50 | // Adapter class to make BrotliIn objects from raw memory. 51 | class BrotliMemIn : public BrotliIn { 52 | public: 53 | BrotliMemIn(const void* buf, int len); 54 | 55 | void Reset(const void* buf, int len); 56 | 57 | // returns the amount of data consumed 58 | int position() const { return pos_; } 59 | 60 | const void* Read(size_t n, size_t* OUTPUT) override; 61 | 62 | private: 63 | const void* buf_; // start of input buffer 64 | int len_; // length of input 65 | int pos_; // current read position within input 66 | }; 67 | 68 | // Adapter class to make BrotliOut objects from raw memory. 69 | class BrotliMemOut : public BrotliOut { 70 | public: 71 | BrotliMemOut(void* buf, int len); 72 | 73 | void Reset(void* buf, int len); 74 | 75 | // returns the amount of data written 76 | int position() const { return pos_; } 77 | 78 | bool Write(const void* buf, size_t n) override; 79 | 80 | private: 81 | void* buf_; // start of output buffer 82 | int len_; // length of output 83 | int pos_; // current write position within output 84 | }; 85 | 86 | // Adapter class to make BrotliOut objects from a string. 87 | class BrotliStringOut : public BrotliOut { 88 | public: 89 | // Create a writer that appends its data to buf. 90 | // buf->size() will grow to at most max_size 91 | // buf is expected to be empty when constructing BrotliStringOut. 92 | BrotliStringOut(std::string* buf, int max_size); 93 | 94 | void Reset(std::string* buf, int max_len); 95 | 96 | bool Write(const void* buf, size_t n) override; 97 | 98 | private: 99 | std::string* buf_; // start of output buffer 100 | int max_size_; // max length of output 101 | }; 102 | 103 | // Adapter class to make BrotliIn object from a file. 104 | class BrotliFileIn : public BrotliIn { 105 | public: 106 | BrotliFileIn(FILE* f, size_t max_read_size); 107 | ~BrotliFileIn(); 108 | 109 | const void* Read(size_t n, size_t* bytes_read) override; 110 | 111 | private: 112 | FILE* f_; 113 | void* buf_; 114 | size_t buf_size_; 115 | }; 116 | 117 | // Adapter class to make BrotliOut object from a file. 118 | class BrotliFileOut : public BrotliOut { 119 | public: 120 | explicit BrotliFileOut(FILE* f); 121 | 122 | bool Write(const void* buf, size_t n) override; 123 | private: 124 | FILE* f_; 125 | }; 126 | 127 | 128 | } // namespace brotli 129 | 130 | #endif // BROTLI_ENC_STREAMS_H_ 131 | -------------------------------------------------------------------------------- /csrc/enc/streams.h.gch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfroidure/ttf2woff2/3a9b14734fe166d3f7db65d8d8407a7f08540da5/csrc/enc/streams.h.gch -------------------------------------------------------------------------------- /csrc/enc/transform.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Transformations on dictionary words. 16 | 17 | #ifndef BROTLI_ENC_TRANSFORM_H_ 18 | #define BROTLI_ENC_TRANSFORM_H_ 19 | 20 | #include 21 | 22 | #include "./dictionary.h" 23 | 24 | namespace brotli { 25 | 26 | enum WordTransformType { 27 | kIdentity = 0, 28 | kOmitLast1 = 1, 29 | kOmitLast2 = 2, 30 | kOmitLast3 = 3, 31 | kOmitLast4 = 4, 32 | kOmitLast5 = 5, 33 | kOmitLast6 = 6, 34 | kOmitLast7 = 7, 35 | kOmitLast8 = 8, 36 | kOmitLast9 = 9, 37 | kUppercaseFirst = 10, 38 | kUppercaseAll = 11, 39 | kOmitFirst1 = 12, 40 | kOmitFirst2 = 13, 41 | kOmitFirst3 = 14, 42 | kOmitFirst4 = 15, 43 | kOmitFirst5 = 16, 44 | kOmitFirst6 = 17, 45 | kOmitFirst7 = 18, 46 | kOmitFirst8 = 19, 47 | kOmitFirst9 = 20, 48 | }; 49 | 50 | struct Transform { 51 | const char* prefix; 52 | WordTransformType word_transform; 53 | const char* suffix; 54 | }; 55 | 56 | static const Transform kTransforms[] = { 57 | { "", kIdentity, "" }, 58 | { "", kIdentity, " " }, 59 | { " ", kIdentity, " " }, 60 | { "", kOmitFirst1, "" }, 61 | { "", kUppercaseFirst, " " }, 62 | { "", kIdentity, " the " }, 63 | { " ", kIdentity, "" }, 64 | { "s ", kIdentity, " " }, 65 | { "", kIdentity, " of " }, 66 | { "", kUppercaseFirst, "" }, 67 | { "", kIdentity, " and " }, 68 | { "", kOmitFirst2, "" }, 69 | { "", kOmitLast1, "" }, 70 | { ", ", kIdentity, " " }, 71 | { "", kIdentity, ", " }, 72 | { " ", kUppercaseFirst, " " }, 73 | { "", kIdentity, " in " }, 74 | { "", kIdentity, " to " }, 75 | { "e ", kIdentity, " " }, 76 | { "", kIdentity, "\"" }, 77 | { "", kIdentity, "." }, 78 | { "", kIdentity, "\">" }, 79 | { "", kIdentity, "\n" }, 80 | { "", kOmitLast3, "" }, 81 | { "", kIdentity, "]" }, 82 | { "", kIdentity, " for " }, 83 | { "", kOmitFirst3, "" }, 84 | { "", kOmitLast2, "" }, 85 | { "", kIdentity, " a " }, 86 | { "", kIdentity, " that " }, 87 | { " ", kUppercaseFirst, "" }, 88 | { "", kIdentity, ". " }, 89 | { ".", kIdentity, "" }, 90 | { " ", kIdentity, ", " }, 91 | { "", kOmitFirst4, "" }, 92 | { "", kIdentity, " with " }, 93 | { "", kIdentity, "'" }, 94 | { "", kIdentity, " from " }, 95 | { "", kIdentity, " by " }, 96 | { "", kOmitFirst5, "" }, 97 | { "", kOmitFirst6, "" }, 98 | { " the ", kIdentity, "" }, 99 | { "", kOmitLast4, "" }, 100 | { "", kIdentity, ". The " }, 101 | { "", kUppercaseAll, "" }, 102 | { "", kIdentity, " on " }, 103 | { "", kIdentity, " as " }, 104 | { "", kIdentity, " is " }, 105 | { "", kOmitLast7, "" }, 106 | { "", kOmitLast1, "ing " }, 107 | { "", kIdentity, "\n\t" }, 108 | { "", kIdentity, ":" }, 109 | { " ", kIdentity, ". " }, 110 | { "", kIdentity, "ed " }, 111 | { "", kOmitFirst9, "" }, 112 | { "", kOmitFirst7, "" }, 113 | { "", kOmitLast6, "" }, 114 | { "", kIdentity, "(" }, 115 | { "", kUppercaseFirst, ", " }, 116 | { "", kOmitLast8, "" }, 117 | { "", kIdentity, " at " }, 118 | { "", kIdentity, "ly " }, 119 | { " the ", kIdentity, " of " }, 120 | { "", kOmitLast5, "" }, 121 | { "", kOmitLast9, "" }, 122 | { " ", kUppercaseFirst, ", " }, 123 | { "", kUppercaseFirst, "\"" }, 124 | { ".", kIdentity, "(" }, 125 | { "", kUppercaseAll, " " }, 126 | { "", kUppercaseFirst, "\">" }, 127 | { "", kIdentity, "=\"" }, 128 | { " ", kIdentity, "." }, 129 | { ".com/", kIdentity, "" }, 130 | { " the ", kIdentity, " of the " }, 131 | { "", kUppercaseFirst, "'" }, 132 | { "", kIdentity, ". This " }, 133 | { "", kIdentity, "," }, 134 | { ".", kIdentity, " " }, 135 | { "", kUppercaseFirst, "(" }, 136 | { "", kUppercaseFirst, "." }, 137 | { "", kIdentity, " not " }, 138 | { " ", kIdentity, "=\"" }, 139 | { "", kIdentity, "er " }, 140 | { " ", kUppercaseAll, " " }, 141 | { "", kIdentity, "al " }, 142 | { " ", kUppercaseAll, "" }, 143 | { "", kIdentity, "='" }, 144 | { "", kUppercaseAll, "\"" }, 145 | { "", kUppercaseFirst, ". " }, 146 | { " ", kIdentity, "(" }, 147 | { "", kIdentity, "ful " }, 148 | { " ", kUppercaseFirst, ". " }, 149 | { "", kIdentity, "ive " }, 150 | { "", kIdentity, "less " }, 151 | { "", kUppercaseAll, "'" }, 152 | { "", kIdentity, "est " }, 153 | { " ", kUppercaseFirst, "." }, 154 | { "", kUppercaseAll, "\">" }, 155 | { " ", kIdentity, "='" }, 156 | { "", kUppercaseFirst, "," }, 157 | { "", kIdentity, "ize " }, 158 | { "", kUppercaseAll, "." }, 159 | { "\xc2\xa0", kIdentity, "" }, 160 | { " ", kIdentity, "," }, 161 | { "", kUppercaseFirst, "=\"" }, 162 | { "", kUppercaseAll, "=\"" }, 163 | { "", kIdentity, "ous " }, 164 | { "", kUppercaseAll, ", " }, 165 | { "", kUppercaseFirst, "='" }, 166 | { " ", kUppercaseFirst, "," }, 167 | { " ", kUppercaseAll, "=\"" }, 168 | { " ", kUppercaseAll, ", " }, 169 | { "", kUppercaseAll, "," }, 170 | { "", kUppercaseAll, "(" }, 171 | { "", kUppercaseAll, ". " }, 172 | { " ", kUppercaseAll, "." }, 173 | { "", kUppercaseAll, "='" }, 174 | { " ", kUppercaseAll, ". " }, 175 | { " ", kUppercaseFirst, "=\"" }, 176 | { " ", kUppercaseAll, "='" }, 177 | { " ", kUppercaseFirst, "='" }, 178 | }; 179 | 180 | static const int kNumTransforms = sizeof(kTransforms) / sizeof(kTransforms[0]); 181 | 182 | static int ToUpperCase(uint8_t *p, int len) { 183 | if (len == 1 || p[0] < 0xc0) { 184 | if (p[0] >= 'a' && p[0] <= 'z') { 185 | p[0] ^= 32; 186 | } 187 | return 1; 188 | } 189 | if (p[0] < 0xe0) { 190 | p[1] ^= 32; 191 | return 2; 192 | } 193 | if (len == 2) { 194 | return 2; 195 | } 196 | p[2] ^= 5; 197 | return 3; 198 | } 199 | 200 | inline std::string ApplyTransform( 201 | const Transform& t, const uint8_t* word, int len) { 202 | std::string ret(t.prefix); 203 | if (t.word_transform <= kOmitLast9) { 204 | len -= t.word_transform; 205 | } 206 | if (len > 0) { 207 | if (t.word_transform >= kOmitFirst1) { 208 | const int skip = t.word_transform - (kOmitFirst1 - 1); 209 | if (len > skip) { 210 | ret += std::string(word + skip, word + len); 211 | } 212 | } else { 213 | ret += std::string(word, word + len); 214 | uint8_t *uppercase = reinterpret_cast(&ret[ret.size() - len]); 215 | if (t.word_transform == kUppercaseFirst) { 216 | ToUpperCase(uppercase, len); 217 | } else if (t.word_transform == kUppercaseAll) { 218 | while (len > 0) { 219 | int step = ToUpperCase(uppercase, len); 220 | uppercase += step; 221 | len -= step; 222 | } 223 | } 224 | } 225 | } 226 | ret += std::string(t.suffix); 227 | return ret; 228 | } 229 | 230 | inline std::string GetTransformedDictionaryWord(int len_code, int word_id) { 231 | int num_words = 1 << kBrotliDictionarySizeBitsByLength[len_code]; 232 | int offset = kBrotliDictionaryOffsetsByLength[len_code]; 233 | int t = word_id / num_words; 234 | int word_idx = word_id % num_words; 235 | offset += len_code * word_idx; 236 | const uint8_t* word = &kBrotliDictionary[offset]; 237 | return ApplyTransform(kTransforms[t], word, len_code); 238 | } 239 | 240 | } // namespace brotli 241 | 242 | #endif // BROTLI_ENC_TRANSFORM_H_ 243 | -------------------------------------------------------------------------------- /csrc/enc/write_bits.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Write bits into a byte array. 16 | 17 | #ifndef BROTLI_ENC_WRITE_BITS_H_ 18 | #define BROTLI_ENC_WRITE_BITS_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include "./port.h" 25 | 26 | namespace brotli { 27 | 28 | //#define BIT_WRITER_DEBUG 29 | 30 | // This function writes bits into bytes in increasing addresses, and within 31 | // a byte least-significant-bit first. 32 | // 33 | // The function can write up to 56 bits in one go with WriteBits 34 | // Example: let's assume that 3 bits (Rs below) have been written already: 35 | // 36 | // BYTE-0 BYTE+1 BYTE+2 37 | // 38 | // 0000 0RRR 0000 0000 0000 0000 39 | // 40 | // Now, we could write 5 or less bits in MSB by just sifting by 3 41 | // and OR'ing to BYTE-0. 42 | // 43 | // For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, 44 | // and locate the rest in BYTE+1, BYTE+2, etc. 45 | inline void WriteBits(int n_bits, 46 | uint64_t bits, 47 | int * __restrict pos, 48 | uint8_t * __restrict array) { 49 | #ifdef BIT_WRITER_DEBUG 50 | printf("WriteBits %2d 0x%016llx %10d\n", n_bits, bits, *pos); 51 | #endif 52 | assert(bits < 1ULL << n_bits); 53 | #ifdef IS_LITTLE_ENDIAN 54 | // This branch of the code can write up to 56 bits at a time, 55 | // 7 bits are lost by being perhaps already in *p and at least 56 | // 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 57 | // bits are in *p and we write 57 bits, then the next write will 58 | // access a byte that was never initialized). 59 | uint8_t *p = &array[*pos >> 3]; 60 | uint64_t v = *p; 61 | v |= bits << (*pos & 7); 62 | BROTLI_UNALIGNED_STORE64(p, v); // Set some bits. 63 | *pos += n_bits; 64 | #else 65 | // implicit & 0xff is assumed for uint8_t arithmetics 66 | uint8_t *array_pos = &array[*pos >> 3]; 67 | const int bits_reserved_in_first_byte = (*pos & 7); 68 | bits <<= bits_reserved_in_first_byte; 69 | *array_pos++ |= bits; 70 | for (int bits_left_to_write = n_bits - 8 + bits_reserved_in_first_byte; 71 | bits_left_to_write >= 1; 72 | bits_left_to_write -= 8) { 73 | bits >>= 8; 74 | *array_pos++ = bits; 75 | } 76 | *array_pos = 0; 77 | *pos += n_bits; 78 | #endif 79 | } 80 | 81 | inline void WriteBitsPrepareStorage(int pos, uint8_t *array) { 82 | #ifdef BIT_WRITER_DEBUG 83 | printf("WriteBitsPrepareStorage %10d\n", pos); 84 | #endif 85 | assert((pos & 7) == 0); 86 | array[pos >> 3] = 0; 87 | } 88 | 89 | } // namespace brotli 90 | 91 | #endif // BROTLI_ENC_WRITE_BITS_H_ 92 | -------------------------------------------------------------------------------- /csrc/fallback.cc: -------------------------------------------------------------------------------- 1 | // Emscripten wrapper 2 | #include 3 | #include 4 | #include "./woff2/woff2_enc.h" 5 | 6 | using namespace emscripten; 7 | using std::string; 8 | 9 | int getSizePtr() { 10 | int* sizePtr = reinterpret_cast(calloc(1, sizeof(int))); 11 | return reinterpret_cast(sizePtr); 12 | } 13 | 14 | int convert(int inputDataAddress, int inputLength, int outputSizePtrAddress) { 15 | int* outputSizePtr = reinterpret_cast(outputSizePtrAddress); 16 | char* inputData = reinterpret_cast(inputDataAddress); 17 | 18 | size_t outputSize = woff2::MaxWOFF2CompressedSize( 19 | reinterpret_cast(inputData), 20 | inputLength 21 | ); 22 | 23 | uint8_t* outputData = reinterpret_cast(calloc(outputSize, sizeof(uint8_t))); 24 | 25 | 26 | if(!woff2::ConvertTTFToWOFF2( 27 | reinterpret_cast(inputData), 28 | inputLength, 29 | outputData, 30 | &outputSize 31 | )) { 32 | // throw an error 33 | } 34 | 35 | *outputSizePtr = outputSize; 36 | 37 | return reinterpret_cast(outputData); 38 | } 39 | 40 | void freePtrs(int outputDataAddress, int sizePtrAddress) { 41 | int* sizePtr = reinterpret_cast(sizePtrAddress); 42 | char* outputData = reinterpret_cast(outputDataAddress); 43 | free(outputData); 44 | free(sizePtr); 45 | } 46 | 47 | EMSCRIPTEN_BINDINGS(ttf2woff2_fallback) { 48 | function("getSizePtr", &getSizePtr, allow_raw_pointers()); 49 | function("convert", &convert, allow_raw_pointers()); 50 | function("freePtrs", &freePtrs, allow_raw_pointers()); 51 | } 52 | -------------------------------------------------------------------------------- /csrc/woff2/buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // The parts of ots.h & opentype-sanitiser.h that we need, taken from the 16 | // https://code.google.com/p/ots/ project. 17 | 18 | #ifndef WOFF2_BUFFER_H_ 19 | #define WOFF2_BUFFER_H_ 20 | 21 | #if defined(_WIN32) 22 | #include 23 | typedef signed char int8_t; 24 | typedef unsigned char uint8_t; 25 | typedef short int16_t; 26 | typedef unsigned short uint16_t; 27 | typedef int int32_t; 28 | typedef unsigned int uint32_t; 29 | typedef __int64 int64_t; 30 | typedef unsigned __int64 uint64_t; 31 | #define ntohl(x) _byteswap_ulong (x) 32 | #define ntohs(x) _byteswap_ushort (x) 33 | #define htonl(x) _byteswap_ulong (x) 34 | #define htons(x) _byteswap_ushort (x) 35 | #else 36 | #include 37 | #include 38 | #endif 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | namespace woff2 { 46 | 47 | #if defined(_MSC_VER) || !defined(FONT_COMPRESSION_DEBUG) 48 | #define FONT_COMPRESSION_FAILURE() false 49 | #else 50 | #define FONT_COMPRESSION_FAILURE() \ 51 | woff2::Failure(__FILE__, __LINE__, __PRETTY_FUNCTION__) 52 | inline bool Failure(const char *f, int l, const char *fn) { 53 | fprintf(stderr, "ERROR at %s:%d (%s)\n", f, l, fn); 54 | fflush(stderr); 55 | return false; 56 | } 57 | #endif 58 | 59 | // ----------------------------------------------------------------------------- 60 | // Buffer helper class 61 | // 62 | // This class perform some trival buffer operations while checking for 63 | // out-of-bounds errors. As a family they return false if anything is amiss, 64 | // updating the current offset otherwise. 65 | // ----------------------------------------------------------------------------- 66 | class Buffer { 67 | public: 68 | Buffer(const uint8_t *buffer, size_t len) 69 | : buffer_(buffer), 70 | length_(len), 71 | offset_(0) { } 72 | 73 | bool Skip(size_t n_bytes) { 74 | return Read(NULL, n_bytes); 75 | } 76 | 77 | bool Read(uint8_t *buffer, size_t n_bytes) { 78 | if (n_bytes > 1024 * 1024 * 1024) { 79 | return FONT_COMPRESSION_FAILURE(); 80 | } 81 | if ((offset_ + n_bytes > length_) || 82 | (offset_ > length_ - n_bytes)) { 83 | return FONT_COMPRESSION_FAILURE(); 84 | } 85 | if (buffer) { 86 | std::memcpy(buffer, buffer_ + offset_, n_bytes); 87 | } 88 | offset_ += n_bytes; 89 | return true; 90 | } 91 | 92 | inline bool ReadU8(uint8_t *value) { 93 | if (offset_ + 1 > length_) { 94 | return FONT_COMPRESSION_FAILURE(); 95 | } 96 | *value = buffer_[offset_]; 97 | ++offset_; 98 | return true; 99 | } 100 | 101 | bool ReadU16(uint16_t *value) { 102 | if (offset_ + 2 > length_) { 103 | return FONT_COMPRESSION_FAILURE(); 104 | } 105 | std::memcpy(value, buffer_ + offset_, sizeof(uint16_t)); 106 | *value = ntohs(*value); 107 | offset_ += 2; 108 | return true; 109 | } 110 | 111 | bool ReadS16(int16_t *value) { 112 | return ReadU16(reinterpret_cast(value)); 113 | } 114 | 115 | bool ReadU24(uint32_t *value) { 116 | if (offset_ + 3 > length_) { 117 | return FONT_COMPRESSION_FAILURE(); 118 | } 119 | *value = static_cast(buffer_[offset_]) << 16 | 120 | static_cast(buffer_[offset_ + 1]) << 8 | 121 | static_cast(buffer_[offset_ + 2]); 122 | offset_ += 3; 123 | return true; 124 | } 125 | 126 | bool ReadU32(uint32_t *value) { 127 | if (offset_ + 4 > length_) { 128 | return FONT_COMPRESSION_FAILURE(); 129 | } 130 | std::memcpy(value, buffer_ + offset_, sizeof(uint32_t)); 131 | *value = ntohl(*value); 132 | offset_ += 4; 133 | return true; 134 | } 135 | 136 | bool ReadS32(int32_t *value) { 137 | return ReadU32(reinterpret_cast(value)); 138 | } 139 | 140 | bool ReadTag(uint32_t *value) { 141 | if (offset_ + 4 > length_) { 142 | return FONT_COMPRESSION_FAILURE(); 143 | } 144 | std::memcpy(value, buffer_ + offset_, sizeof(uint32_t)); 145 | offset_ += 4; 146 | return true; 147 | } 148 | 149 | bool ReadR64(uint64_t *value) { 150 | if (offset_ + 8 > length_) { 151 | return FONT_COMPRESSION_FAILURE(); 152 | } 153 | std::memcpy(value, buffer_ + offset_, sizeof(uint64_t)); 154 | offset_ += 8; 155 | return true; 156 | } 157 | 158 | const uint8_t *buffer() const { return buffer_; } 159 | size_t offset() const { return offset_; } 160 | size_t length() const { return length_; } 161 | 162 | void set_offset(size_t newoffset) { offset_ = newoffset; } 163 | 164 | private: 165 | const uint8_t * const buffer_; 166 | const size_t length_; 167 | size_t offset_; 168 | }; 169 | 170 | } // namespace woff2 171 | 172 | #endif // WOFF2_BUFFER_H_ 173 | -------------------------------------------------------------------------------- /csrc/woff2/font.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Data model for a font file in sfnt format, reading and writing functions and 16 | // accessors for the glyph data. 17 | 18 | #ifndef WOFF2_FONT_H_ 19 | #define WOFF2_FONT_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace woff2 { 27 | 28 | // Represents an sfnt font file. Only the table directory is parsed, for the 29 | // table data we only store a raw pointer, therefore a font object is valid only 30 | // as long the data from which it was parsed is around. 31 | struct Font { 32 | uint32_t flavor; 33 | uint16_t num_tables; 34 | 35 | struct Table { 36 | uint32_t tag; 37 | uint32_t checksum; 38 | uint32_t offset; 39 | uint32_t length; 40 | const uint8_t* data; 41 | 42 | // Buffer used to mutate the data before writing out. 43 | std::vector buffer; 44 | 45 | // If we've seen this tag/offset before, pointer to the first time we saw it 46 | // If this is the first time we've seen this table, NULL 47 | // Intended use is to bypass re-processing tables 48 | Font::Table* reuse_of; 49 | 50 | // Is this table reused by a TTC 51 | bool IsReused() const; 52 | }; 53 | std::map tables; 54 | std::vector OutputOrderedTags() const; 55 | 56 | Table* FindTable(uint32_t tag); 57 | const Table* FindTable(uint32_t tag) const; 58 | }; 59 | 60 | // Accomodates both singular (OTF, TTF) and collection (TTC) fonts 61 | struct FontCollection { 62 | uint32_t header_version; 63 | // (offset, first use of table*) pairs 64 | std::map tables; 65 | std::vector fonts; 66 | }; 67 | 68 | // Parses the font from the given data. Returns false on parsing failure or 69 | // buffer overflow. The font is valid only so long the input data pointer is 70 | // valid. Does NOT support collections. 71 | bool ReadFont(const uint8_t* data, size_t len, Font* font); 72 | 73 | // Parses the font from the given data. Returns false on parsing failure or 74 | // buffer overflow. The font is valid only so long the input data pointer is 75 | // valid. Supports collections. 76 | bool ReadFontCollection(const uint8_t* data, size_t len, FontCollection* fonts); 77 | 78 | // Returns the file size of the font. 79 | size_t FontFileSize(const Font& font); 80 | size_t FontCollectionFileSize(const FontCollection& font); 81 | 82 | // Writes the font into the specified dst buffer. The dst_size should be the 83 | // same as returned by FontFileSize(). Returns false upon buffer overflow (which 84 | // should not happen if dst_size was computed by FontFileSize()). 85 | bool WriteFont(const Font& font, uint8_t* dst, size_t dst_size); 86 | // Write the font at a specific offset 87 | bool WriteFont(const Font& font, size_t* offset, uint8_t* dst, size_t dst_size); 88 | 89 | bool WriteFontCollection(const FontCollection& font_collection, uint8_t* dst, 90 | size_t dst_size); 91 | 92 | // Returns the number of glyphs in the font. 93 | // NOTE: Currently this works only for TrueType-flavored fonts, will return 94 | // zero for CFF-flavored fonts. 95 | int NumGlyphs(const Font& font); 96 | 97 | // Returns the index format of the font 98 | int IndexFormat(const Font& font); 99 | 100 | // Sets *glyph_data and *glyph_size to point to the location of the glyph data 101 | // with the given index. Returns false if the glyph is not found. 102 | bool GetGlyphData(const Font& font, int glyph_index, 103 | const uint8_t** glyph_data, size_t* glyph_size); 104 | 105 | // Removes the digital signature (DSIG) table 106 | bool RemoveDigitalSignature(Font* font); 107 | 108 | } // namespace woff2 109 | 110 | #endif // WOFF2_FONT_H_ 111 | -------------------------------------------------------------------------------- /csrc/woff2/glyph.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Data model and I/O for glyph data within sfnt format files for the purpose of 16 | // performing the preprocessing step of the WOFF 2.0 conversion. 17 | 18 | #ifndef WOFF2_GLYPH_H_ 19 | #define WOFF2_GLYPH_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace woff2 { 26 | 27 | // Represents a parsed simple or composite glyph. The composite glyph data and 28 | // instructions are un-parsed and we keep only pointers to the raw data, 29 | // therefore the glyph is valid only so long the data from which it was parsed 30 | // is around. 31 | class Glyph { 32 | public: 33 | Glyph() : instructions_size(0), composite_data_size(0) {} 34 | 35 | // Bounding box. 36 | int16_t x_min; 37 | int16_t x_max; 38 | int16_t y_min; 39 | int16_t y_max; 40 | 41 | // Instructions. 42 | uint16_t instructions_size; 43 | const uint8_t* instructions_data; 44 | 45 | // Data model for simple glyphs. 46 | struct Point { 47 | int x; 48 | int y; 49 | bool on_curve; 50 | }; 51 | std::vector > contours; 52 | 53 | // Data for composite glyphs. 54 | const uint8_t* composite_data; 55 | uint32_t composite_data_size; 56 | bool have_instructions; 57 | }; 58 | 59 | // Parses the glyph from the given data. Returns false on parsing failure or 60 | // buffer overflow. The glyph is valid only so long the input data pointer is 61 | // valid. 62 | bool ReadGlyph(const uint8_t* data, size_t len, Glyph* glyph); 63 | 64 | // Stores the glyph into the specified dst buffer. The *dst_size is the buffer 65 | // size on entry and is set to the actual (unpadded) stored size on exit. 66 | // Returns false on buffer overflow. 67 | bool StoreGlyph(const Glyph& glyph, uint8_t* dst, size_t* dst_size); 68 | 69 | } // namespace woff2 70 | 71 | #endif // WOFF2_GLYPH_H_ 72 | -------------------------------------------------------------------------------- /csrc/woff2/normalize.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Glyph normalization 16 | 17 | #include "./normalize.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "./buffer.h" 23 | #include "./port.h" 24 | #include "./font.h" 25 | #include "./glyph.h" 26 | #include "./round.h" 27 | #include "./store_bytes.h" 28 | #include "./table_tags.h" 29 | #include "./woff2_common.h" 30 | 31 | namespace woff2 { 32 | 33 | namespace { 34 | 35 | void StoreLoca(int index_fmt, uint32_t value, size_t* offset, uint8_t* dst) { 36 | if (index_fmt == 0) { 37 | Store16(value >> 1, offset, dst); 38 | } else { 39 | StoreU32(value, offset, dst); 40 | } 41 | } 42 | 43 | } // namespace 44 | 45 | namespace { 46 | 47 | bool WriteNormalizedLoca(int index_fmt, int num_glyphs, Font* font) { 48 | Font::Table* glyf_table = font->FindTable(kGlyfTableTag); 49 | Font::Table* loca_table = font->FindTable(kLocaTableTag); 50 | 51 | int glyph_sz = index_fmt == 0 ? 2 : 4; 52 | loca_table->buffer.resize(Round4(num_glyphs + 1) * glyph_sz); 53 | loca_table->length = (num_glyphs + 1) * glyph_sz; 54 | 55 | uint8_t* glyf_dst = &glyf_table->buffer[0]; 56 | uint8_t* loca_dst = &loca_table->buffer[0]; 57 | uint32_t glyf_offset = 0; 58 | size_t loca_offset = 0; 59 | 60 | for (int i = 0; i < num_glyphs; ++i) { 61 | StoreLoca(index_fmt, glyf_offset, &loca_offset, loca_dst); 62 | Glyph glyph; 63 | const uint8_t* glyph_data; 64 | size_t glyph_size; 65 | if (!GetGlyphData(*font, i, &glyph_data, &glyph_size) || 66 | (glyph_size > 0 && !ReadGlyph(glyph_data, glyph_size, &glyph))) { 67 | return FONT_COMPRESSION_FAILURE(); 68 | } 69 | size_t glyf_dst_size = glyf_table->buffer.size() - glyf_offset; 70 | if (!StoreGlyph(glyph, glyf_dst + glyf_offset, &glyf_dst_size)) { 71 | return FONT_COMPRESSION_FAILURE(); 72 | } 73 | glyf_dst_size = Round4(glyf_dst_size); 74 | if (glyf_dst_size > std::numeric_limits::max() || 75 | glyf_offset + static_cast(glyf_dst_size) < glyf_offset || 76 | (index_fmt == 0 && glyf_offset + glyf_dst_size >= (1UL << 17))) { 77 | return FONT_COMPRESSION_FAILURE(); 78 | } 79 | glyf_offset += glyf_dst_size; 80 | } 81 | if (glyf_offset == 0) { 82 | return false; 83 | } 84 | 85 | StoreLoca(index_fmt, glyf_offset, &loca_offset, loca_dst); 86 | 87 | glyf_table->buffer.resize(glyf_offset); 88 | glyf_table->data = &glyf_table->buffer[0]; 89 | glyf_table->length = glyf_offset; 90 | loca_table->data = &loca_table->buffer[0]; 91 | 92 | return true; 93 | } 94 | 95 | } // namespace 96 | 97 | namespace { 98 | 99 | bool MakeEditableBuffer(Font* font, int tableTag) { 100 | Font::Table* table = font->FindTable(tableTag); 101 | if (table == NULL) { 102 | return FONT_COMPRESSION_FAILURE(); 103 | } 104 | if (table->IsReused()) { 105 | return true; 106 | } 107 | int sz = Round4(table->length); 108 | table->buffer.resize(sz); 109 | uint8_t* buf = &table->buffer[0]; 110 | memcpy(buf, table->data, sz); 111 | table->data = buf; 112 | return true; 113 | } 114 | 115 | } // namespace 116 | 117 | bool NormalizeGlyphs(Font* font) { 118 | Font::Table* cff_table = font->FindTable(kCffTableTag); 119 | Font::Table* head_table = font->FindTable(kHeadTableTag); 120 | Font::Table* glyf_table = font->FindTable(kGlyfTableTag); 121 | Font::Table* loca_table = font->FindTable(kLocaTableTag); 122 | if (head_table == NULL) { 123 | return FONT_COMPRESSION_FAILURE(); 124 | } 125 | // CFF, no loca, no glyf is OK for CFF. If so, don't normalize. 126 | if (cff_table != NULL && loca_table == NULL && glyf_table == NULL) { 127 | return true; 128 | } 129 | if (loca_table == NULL || glyf_table == NULL) { 130 | return FONT_COMPRESSION_FAILURE(); 131 | } 132 | 133 | // Must share neither or both loca & glyf 134 | if (loca_table->IsReused() != glyf_table->IsReused()) { 135 | return FONT_COMPRESSION_FAILURE(); 136 | } 137 | if (loca_table->IsReused()) { 138 | return true; 139 | } 140 | 141 | int index_fmt = head_table->data[51]; 142 | int num_glyphs = NumGlyphs(*font); 143 | 144 | // We need to allocate a bit more than its original length for the normalized 145 | // glyf table, since it can happen that the glyphs in the original table are 146 | // 2-byte aligned, while in the normalized table they are 4-byte aligned. 147 | // That gives a maximum of 2 bytes increase per glyph. However, there is no 148 | // theoretical guarantee that the total size of the flags plus the coordinates 149 | // is the smallest possible in the normalized version, so we have to allow 150 | // some general overhead. 151 | // TODO(user) Figure out some more precise upper bound on the size of 152 | // the overhead. 153 | size_t max_normalized_glyf_size = 1.1 * glyf_table->length + 2 * num_glyphs; 154 | 155 | glyf_table->buffer.resize(max_normalized_glyf_size); 156 | 157 | // if we can't write a loca using short's (index_fmt 0) 158 | // try again using longs (index_fmt 1) 159 | if (!WriteNormalizedLoca(index_fmt, num_glyphs, font)) { 160 | if (index_fmt != 0) { 161 | return FONT_COMPRESSION_FAILURE(); 162 | } 163 | 164 | // Rewrite loca with 4-byte entries & update head to match 165 | index_fmt = 1; 166 | if (!WriteNormalizedLoca(index_fmt, num_glyphs, font)) { 167 | return FONT_COMPRESSION_FAILURE(); 168 | } 169 | head_table->buffer[51] = 1; 170 | } 171 | 172 | return true; 173 | } 174 | 175 | bool NormalizeOffsets(Font* font) { 176 | uint32_t offset = 12 + 16 * font->num_tables; 177 | for (auto tag : font->OutputOrderedTags()) { 178 | auto& table = font->tables[tag]; 179 | table.offset = offset; 180 | offset += Round4(table.length); 181 | } 182 | return true; 183 | } 184 | 185 | namespace { 186 | 187 | uint32_t ComputeHeaderChecksum(const Font& font) { 188 | uint32_t checksum = font.flavor; 189 | uint16_t max_pow2 = font.num_tables ? Log2Floor(font.num_tables) : 0; 190 | uint16_t search_range = max_pow2 ? 1 << (max_pow2 + 4) : 0; 191 | uint16_t range_shift = (font.num_tables << 4) - search_range; 192 | checksum += (font.num_tables << 16 | search_range); 193 | checksum += (max_pow2 << 16 | range_shift); 194 | for (const auto& i : font.tables) { 195 | const Font::Table* table = &i.second; 196 | if (table->IsReused()) { 197 | table = table->reuse_of; 198 | } 199 | checksum += table->tag; 200 | checksum += table->checksum; 201 | checksum += table->offset; 202 | checksum += table->length; 203 | } 204 | return checksum; 205 | } 206 | 207 | } // namespace 208 | 209 | bool FixChecksums(Font* font) { 210 | Font::Table* head_table = font->FindTable(kHeadTableTag); 211 | if (head_table == NULL) { 212 | return FONT_COMPRESSION_FAILURE(); 213 | } 214 | if (head_table->reuse_of != NULL) { 215 | head_table = head_table->reuse_of; 216 | } 217 | if (head_table->length < 12) { 218 | return FONT_COMPRESSION_FAILURE(); 219 | } 220 | 221 | uint8_t* head_buf = &head_table->buffer[0]; 222 | size_t offset = 8; 223 | StoreU32(0, &offset, head_buf); 224 | uint32_t file_checksum = 0; 225 | uint32_t head_checksum = 0; 226 | for (auto& i : font->tables) { 227 | Font::Table* table = &i.second; 228 | if (table->IsReused()) { 229 | table = table->reuse_of; 230 | } 231 | table->checksum = ComputeULongSum(table->data, table->length); 232 | file_checksum += table->checksum; 233 | 234 | if (table->tag == kHeadTableTag) { 235 | head_checksum = table->checksum; 236 | } 237 | } 238 | 239 | file_checksum += ComputeHeaderChecksum(*font); 240 | offset = 8; 241 | StoreU32(0xb1b0afba - file_checksum, &offset, head_buf); 242 | 243 | return true; 244 | } 245 | 246 | namespace { 247 | bool MarkTransformed(Font* font) { 248 | Font::Table* head_table = font->FindTable(kHeadTableTag); 249 | if (head_table == NULL) { 250 | return FONT_COMPRESSION_FAILURE(); 251 | } 252 | if (head_table->reuse_of != NULL) { 253 | head_table = head_table->reuse_of; 254 | } 255 | if (head_table->length < 17) { 256 | return FONT_COMPRESSION_FAILURE(); 257 | } 258 | // set bit 11 of head table 'flags' to indicate that font has undergone 259 | // lossless modifying transform 260 | int head_flags = head_table->data[16]; 261 | head_table->buffer[16] = head_flags | 0x08; 262 | return true; 263 | } 264 | } // namespace 265 | 266 | 267 | bool NormalizeWithoutFixingChecksums(Font* font) { 268 | return (MakeEditableBuffer(font, kHeadTableTag) && 269 | RemoveDigitalSignature(font) && 270 | MarkTransformed(font) && 271 | NormalizeGlyphs(font) && 272 | NormalizeOffsets(font)); 273 | } 274 | 275 | bool NormalizeFont(Font* font) { 276 | return (NormalizeWithoutFixingChecksums(font) && 277 | FixChecksums(font)); 278 | } 279 | 280 | bool NormalizeFontCollection(FontCollection* font_collection) { 281 | if (font_collection->fonts.size() == 1) { 282 | return NormalizeFont(&font_collection->fonts[0]); 283 | } 284 | 285 | uint32_t offset = CollectionHeaderSize(font_collection->header_version, 286 | font_collection->fonts.size()); 287 | for (auto& font : font_collection->fonts) { 288 | if (!NormalizeWithoutFixingChecksums(&font)) { 289 | fprintf(stderr, "Font normalization failed.\n"); 290 | return false; 291 | } 292 | offset += kSfntHeaderSize + kSfntEntrySize * font.num_tables; 293 | } 294 | 295 | // Start table offsets after TTC Header and Sfnt Headers 296 | for (auto& font : font_collection->fonts) { 297 | for (auto tag : font.OutputOrderedTags()) { 298 | Font::Table& table = font.tables[tag]; 299 | if (table.IsReused()) { 300 | table.offset = table.reuse_of->offset; 301 | } else { 302 | table.offset = offset; 303 | offset += Round4(table.length); 304 | } 305 | } 306 | } 307 | 308 | // Now we can fix the checksums 309 | for (auto& font : font_collection->fonts) { 310 | if (!FixChecksums(&font)) { 311 | fprintf(stderr, "Failed to fix checksums\n"); 312 | return false; 313 | } 314 | } 315 | 316 | return true; 317 | } 318 | 319 | } // namespace woff2 320 | -------------------------------------------------------------------------------- /csrc/woff2/normalize.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Functions for normalizing fonts. Since the WOFF 2.0 decoder creates font 16 | // files in normalized form, the WOFF 2.0 conversion is guaranteed to be 17 | // lossless (in a bitwise sense) only for normalized font files. 18 | 19 | #ifndef WOFF2_NORMALIZE_H_ 20 | #define WOFF2_NORMALIZE_H_ 21 | 22 | namespace woff2 { 23 | 24 | struct Font; 25 | struct FontCollection; 26 | 27 | // Changes the offset fields of the table headers so that the data for the 28 | // tables will be written in order of increasing tag values, without any gaps 29 | // other than the 4-byte padding. 30 | bool NormalizeOffsets(Font* font); 31 | 32 | // Changes the checksum fields of the table headers and the checksum field of 33 | // the head table so that it matches the current data. 34 | bool FixChecksums(Font* font); 35 | 36 | // Parses each of the glyphs in the font and writes them again to the glyf 37 | // table in normalized form, as defined by the StoreGlyph() function. Changes 38 | // the loca table accordigly. 39 | bool NormalizeGlyphs(Font* font); 40 | 41 | // Performs all of the normalization steps above. 42 | bool NormalizeFont(Font* font); 43 | bool NormalizeFontCollection(FontCollection* font_collection); 44 | 45 | } // namespace woff2 46 | 47 | #endif // WOFF2_NORMALIZE_H_ 48 | -------------------------------------------------------------------------------- /csrc/woff2/port.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helper function for bit twiddling 16 | 17 | #ifndef WOFF2_PORT_H_ 18 | #define WOFF2_PORT_H_ 19 | 20 | #include 21 | 22 | namespace woff2 { 23 | 24 | typedef unsigned int uint32; 25 | 26 | inline int Log2Floor(uint32 n) { 27 | #if defined(__GNUC__) 28 | return n == 0 ? -1 : 31 ^ __builtin_clz(n); 29 | #else 30 | if (n == 0) 31 | return -1; 32 | int log = 0; 33 | uint32 value = n; 34 | for (int i = 4; i >= 0; --i) { 35 | int shift = (1 << i); 36 | uint32 x = value >> shift; 37 | if (x != 0) { 38 | value = x; 39 | log += shift; 40 | } 41 | } 42 | assert(value == 1); 43 | return log; 44 | #endif 45 | } 46 | 47 | } // namespace woff2 48 | #endif // WOFF2_PORT_H_ 49 | -------------------------------------------------------------------------------- /csrc/woff2/round.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helper for rounding 16 | 17 | #ifndef WOFF2_ROUND_H_ 18 | #define WOFF2_ROUND_H_ 19 | 20 | #include 21 | 22 | namespace woff2 { 23 | 24 | // Round a value up to the nearest multiple of 4. Don't round the value in the 25 | // case that rounding up overflows. 26 | template T Round4(T value) { 27 | if (std::numeric_limits::max() - value < 3) { 28 | return value; 29 | } 30 | return (value + 3) & ~3; 31 | } 32 | 33 | } // namespace woff2 34 | 35 | #endif // WOFF2_ROUND_H_ 36 | -------------------------------------------------------------------------------- /csrc/woff2/store_bytes.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helper functions for storing integer values into byte streams. 16 | // No bounds checking is performed, that is the responsibility of the caller. 17 | 18 | #ifndef WOFF2_STORE_BYTES_H_ 19 | #define WOFF2_STORE_BYTES_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | namespace woff2 { 26 | 27 | inline size_t StoreU32(uint8_t* dst, size_t offset, uint32_t x) { 28 | dst[offset] = x >> 24; 29 | dst[offset + 1] = x >> 16; 30 | dst[offset + 2] = x >> 8; 31 | dst[offset + 3] = x; 32 | return offset + 4; 33 | } 34 | 35 | inline size_t Store16(uint8_t* dst, size_t offset, int x) { 36 | dst[offset] = x >> 8; 37 | dst[offset + 1] = x; 38 | return offset + 2; 39 | } 40 | 41 | inline void StoreU32(uint32_t val, size_t* offset, uint8_t* dst) { 42 | dst[(*offset)++] = val >> 24; 43 | dst[(*offset)++] = val >> 16; 44 | dst[(*offset)++] = val >> 8; 45 | dst[(*offset)++] = val; 46 | } 47 | 48 | inline void Store16(int val, size_t* offset, uint8_t* dst) { 49 | dst[(*offset)++] = val >> 8; 50 | dst[(*offset)++] = val; 51 | } 52 | 53 | inline void StoreBytes(const uint8_t* data, size_t len, 54 | size_t* offset, uint8_t* dst) { 55 | memcpy(&dst[*offset], data, len); 56 | *offset += len; 57 | } 58 | 59 | } // namespace woff2 60 | 61 | #endif // WOFF2_STORE_BYTES_H_ 62 | -------------------------------------------------------------------------------- /csrc/woff2/table_tags.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Font table tags 16 | 17 | #include "./table_tags.h" 18 | 19 | namespace woff2 { 20 | 21 | // Note that the byte order is big-endian, not the same as ots.cc 22 | #define TAG(a, b, c, d) ((a << 24) | (b << 16) | (c << 8) | d) 23 | 24 | const uint32_t kKnownTags[63] = { 25 | TAG('c', 'm', 'a', 'p'), // 0 26 | TAG('h', 'e', 'a', 'd'), // 1 27 | TAG('h', 'h', 'e', 'a'), // 2 28 | TAG('h', 'm', 't', 'x'), // 3 29 | TAG('m', 'a', 'x', 'p'), // 4 30 | TAG('n', 'a', 'm', 'e'), // 5 31 | TAG('O', 'S', '/', '2'), // 6 32 | TAG('p', 'o', 's', 't'), // 7 33 | TAG('c', 'v', 't', ' '), // 8 34 | TAG('f', 'p', 'g', 'm'), // 9 35 | TAG('g', 'l', 'y', 'f'), // 10 36 | TAG('l', 'o', 'c', 'a'), // 11 37 | TAG('p', 'r', 'e', 'p'), // 12 38 | TAG('C', 'F', 'F', ' '), // 13 39 | TAG('V', 'O', 'R', 'G'), // 14 40 | TAG('E', 'B', 'D', 'T'), // 15 41 | TAG('E', 'B', 'L', 'C'), // 16 42 | TAG('g', 'a', 's', 'p'), // 17 43 | TAG('h', 'd', 'm', 'x'), // 18 44 | TAG('k', 'e', 'r', 'n'), // 19 45 | TAG('L', 'T', 'S', 'H'), // 20 46 | TAG('P', 'C', 'L', 'T'), // 21 47 | TAG('V', 'D', 'M', 'X'), // 22 48 | TAG('v', 'h', 'e', 'a'), // 23 49 | TAG('v', 'm', 't', 'x'), // 24 50 | TAG('B', 'A', 'S', 'E'), // 25 51 | TAG('G', 'D', 'E', 'F'), // 26 52 | TAG('G', 'P', 'O', 'S'), // 27 53 | TAG('G', 'S', 'U', 'B'), // 28 54 | TAG('E', 'B', 'S', 'C'), // 29 55 | TAG('J', 'S', 'T', 'F'), // 30 56 | TAG('M', 'A', 'T', 'H'), // 31 57 | TAG('C', 'B', 'D', 'T'), // 32 58 | TAG('C', 'B', 'L', 'C'), // 33 59 | TAG('C', 'O', 'L', 'R'), // 34 60 | TAG('C', 'P', 'A', 'L'), // 35 61 | TAG('S', 'V', 'G', ' '), // 36 62 | TAG('s', 'b', 'i', 'x'), // 37 63 | TAG('a', 'c', 'n', 't'), // 38 64 | TAG('a', 'v', 'a', 'r'), // 39 65 | TAG('b', 'd', 'a', 't'), // 40 66 | TAG('b', 'l', 'o', 'c'), // 41 67 | TAG('b', 's', 'l', 'n'), // 42 68 | TAG('c', 'v', 'a', 'r'), // 43 69 | TAG('f', 'd', 's', 'c'), // 44 70 | TAG('f', 'e', 'a', 't'), // 45 71 | TAG('f', 'm', 't', 'x'), // 46 72 | TAG('f', 'v', 'a', 'r'), // 47 73 | TAG('g', 'v', 'a', 'r'), // 48 74 | TAG('h', 's', 't', 'y'), // 49 75 | TAG('j', 'u', 's', 't'), // 50 76 | TAG('l', 'c', 'a', 'r'), // 51 77 | TAG('m', 'o', 'r', 't'), // 52 78 | TAG('m', 'o', 'r', 'x'), // 53 79 | TAG('o', 'p', 'b', 'd'), // 54 80 | TAG('p', 'r', 'o', 'p'), // 55 81 | TAG('t', 'r', 'a', 'k'), // 56 82 | TAG('Z', 'a', 'p', 'f'), // 57 83 | TAG('S', 'i', 'l', 'f'), // 58 84 | TAG('G', 'l', 'a', 't'), // 59 85 | TAG('G', 'l', 'o', 'c'), // 60 86 | TAG('F', 'e', 'a', 't'), // 61 87 | TAG('S', 'i', 'l', 'l'), // 62 88 | }; 89 | 90 | } // namespace woff2 91 | -------------------------------------------------------------------------------- /csrc/woff2/table_tags.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Font table tags 16 | 17 | #ifndef WOFF2_TABLE_TAGS_H_ 18 | #define WOFF2_TABLE_TAGS_H_ 19 | 20 | #include 21 | 22 | namespace woff2 { 23 | 24 | // Tags of popular tables. 25 | static const uint32_t kGlyfTableTag = 0x676c7966; 26 | static const uint32_t kHeadTableTag = 0x68656164; 27 | static const uint32_t kLocaTableTag = 0x6c6f6361; 28 | static const uint32_t kDsigTableTag = 0x44534947; 29 | static const uint32_t kCffTableTag = 0x43464620; 30 | 31 | extern const uint32_t kKnownTags[]; 32 | 33 | } // namespace woff2 34 | 35 | #endif // WOFF2_TABLE_TAGS_H_ 36 | -------------------------------------------------------------------------------- /csrc/woff2/transform.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Library for preprocessing fonts as part of the WOFF 2.0 conversion. 16 | 17 | #include "./transform.h" 18 | 19 | #include // for std::abs 20 | 21 | #include "./buffer.h" 22 | #include "./font.h" 23 | #include "./glyph.h" 24 | #include "./table_tags.h" 25 | #include "./variable_length.h" 26 | 27 | namespace woff2 { 28 | 29 | namespace { 30 | 31 | const int FLAG_ARG_1_AND_2_ARE_WORDS = 1 << 0; 32 | const int FLAG_WE_HAVE_INSTRUCTIONS = 1 << 8; 33 | 34 | void WriteBytes(std::vector* out, const uint8_t* data, size_t len) { 35 | if (len == 0) return; 36 | size_t offset = out->size(); 37 | out->resize(offset + len); 38 | memcpy(&(*out)[offset], data, len); 39 | } 40 | 41 | void WriteBytes(std::vector* out, const std::vector& in) { 42 | for (int i = 0; i < in.size(); ++i) { 43 | out->push_back(in[i]); 44 | } 45 | } 46 | 47 | void WriteUShort(std::vector* out, int value) { 48 | out->push_back(value >> 8); 49 | out->push_back(value & 255); 50 | } 51 | 52 | void WriteLong(std::vector* out, int value) { 53 | out->push_back((value >> 24) & 255); 54 | out->push_back((value >> 16) & 255); 55 | out->push_back((value >> 8) & 255); 56 | out->push_back(value & 255); 57 | } 58 | 59 | // Glyf table preprocessing, based on 60 | // GlyfEncoder.java 61 | class GlyfEncoder { 62 | public: 63 | explicit GlyfEncoder(int num_glyphs) 64 | : n_glyphs_(num_glyphs) { 65 | bbox_bitmap_.resize(((num_glyphs + 31) >> 5) << 2); 66 | } 67 | 68 | bool Encode(int glyph_id, const Glyph& glyph) { 69 | if (glyph.composite_data_size > 0) { 70 | WriteCompositeGlyph(glyph_id, glyph); 71 | } else if (glyph.contours.size() > 0) { 72 | WriteSimpleGlyph(glyph_id, glyph); 73 | } else { 74 | WriteUShort(&n_contour_stream_, 0); 75 | } 76 | return true; 77 | } 78 | 79 | void GetTransformedGlyfBytes(std::vector* result) { 80 | WriteLong(result, 0); // version 81 | WriteUShort(result, n_glyphs_); 82 | WriteUShort(result, 0); // index_format, will be set later 83 | WriteLong(result, n_contour_stream_.size()); 84 | WriteLong(result, n_points_stream_.size()); 85 | WriteLong(result, flag_byte_stream_.size()); 86 | WriteLong(result, glyph_stream_.size()); 87 | WriteLong(result, composite_stream_.size()); 88 | WriteLong(result, bbox_bitmap_.size() + bbox_stream_.size()); 89 | WriteLong(result, instruction_stream_.size()); 90 | WriteBytes(result, n_contour_stream_); 91 | WriteBytes(result, n_points_stream_); 92 | WriteBytes(result, flag_byte_stream_); 93 | WriteBytes(result, glyph_stream_); 94 | WriteBytes(result, composite_stream_); 95 | WriteBytes(result, bbox_bitmap_); 96 | WriteBytes(result, bbox_stream_); 97 | WriteBytes(result, instruction_stream_); 98 | } 99 | 100 | private: 101 | void WriteInstructions(const Glyph& glyph) { 102 | Write255UShort(&glyph_stream_, glyph.instructions_size); 103 | WriteBytes(&instruction_stream_, 104 | glyph.instructions_data, glyph.instructions_size); 105 | } 106 | 107 | bool ShouldWriteSimpleGlyphBbox(const Glyph& glyph) { 108 | if (glyph.contours.empty() || glyph.contours[0].empty()) { 109 | return glyph.x_min || glyph.y_min || glyph.x_max || glyph.y_max; 110 | } 111 | 112 | int16_t x_min = glyph.contours[0][0].x; 113 | int16_t y_min = glyph.contours[0][0].y; 114 | int16_t x_max = x_min; 115 | int16_t y_max = y_min; 116 | for (const auto& contour : glyph.contours) { 117 | for (const auto& point : contour) { 118 | if (point.x < x_min) x_min = point.x; 119 | if (point.x > x_max) x_max = point.x; 120 | if (point.y < y_min) y_min = point.y; 121 | if (point.y > y_max) y_max = point.y; 122 | } 123 | } 124 | 125 | if (glyph.x_min != x_min) 126 | return true; 127 | if (glyph.y_min != y_min) 128 | return true; 129 | if (glyph.x_max != x_max) 130 | return true; 131 | if (glyph.y_max != y_max) 132 | return true; 133 | 134 | return false; 135 | } 136 | 137 | void WriteSimpleGlyph(int glyph_id, const Glyph& glyph) { 138 | int num_contours = glyph.contours.size(); 139 | WriteUShort(&n_contour_stream_, num_contours); 140 | if (ShouldWriteSimpleGlyphBbox(glyph)) { 141 | WriteBbox(glyph_id, glyph); 142 | } 143 | for (int i = 0; i < num_contours; i++) { 144 | Write255UShort(&n_points_stream_, glyph.contours[i].size()); 145 | } 146 | int lastX = 0; 147 | int lastY = 0; 148 | for (int i = 0; i < num_contours; i++) { 149 | int num_points = glyph.contours[i].size(); 150 | for (int j = 0; j < num_points; j++) { 151 | int x = glyph.contours[i][j].x; 152 | int y = glyph.contours[i][j].y; 153 | int dx = x - lastX; 154 | int dy = y - lastY; 155 | WriteTriplet(glyph.contours[i][j].on_curve, dx, dy); 156 | lastX = x; 157 | lastY = y; 158 | } 159 | } 160 | if (num_contours > 0) { 161 | WriteInstructions(glyph); 162 | } 163 | } 164 | 165 | void WriteCompositeGlyph(int glyph_id, const Glyph& glyph) { 166 | WriteUShort(&n_contour_stream_, -1); 167 | WriteBbox(glyph_id, glyph); 168 | WriteBytes(&composite_stream_, 169 | glyph.composite_data, 170 | glyph.composite_data_size); 171 | if (glyph.have_instructions) { 172 | WriteInstructions(glyph); 173 | } 174 | } 175 | 176 | void WriteBbox(int glyph_id, const Glyph& glyph) { 177 | bbox_bitmap_[glyph_id >> 3] |= 0x80 >> (glyph_id & 7); 178 | WriteUShort(&bbox_stream_, glyph.x_min); 179 | WriteUShort(&bbox_stream_, glyph.y_min); 180 | WriteUShort(&bbox_stream_, glyph.x_max); 181 | WriteUShort(&bbox_stream_, glyph.y_max); 182 | } 183 | 184 | void WriteTriplet(bool on_curve, int x, int y) { 185 | int abs_x = std::abs(x); 186 | int abs_y = std::abs(y); 187 | int on_curve_bit = on_curve ? 0 : 128; 188 | int x_sign_bit = (x < 0) ? 0 : 1; 189 | int y_sign_bit = (y < 0) ? 0 : 1; 190 | int xy_sign_bits = x_sign_bit + 2 * y_sign_bit; 191 | if (x == 0 && abs_y < 1280) { 192 | flag_byte_stream_.push_back(on_curve_bit + 193 | ((abs_y & 0xf00) >> 7) + y_sign_bit); 194 | glyph_stream_.push_back(abs_y & 0xff); 195 | } else if (y == 0 && abs_x < 1280) { 196 | flag_byte_stream_.push_back(on_curve_bit + 10 + 197 | ((abs_x & 0xf00) >> 7) + x_sign_bit); 198 | glyph_stream_.push_back(abs_x & 0xff); 199 | } else if (abs_x < 65 && abs_y < 65) { 200 | flag_byte_stream_.push_back(on_curve_bit + 20 + 201 | ((abs_x - 1) & 0x30) + 202 | (((abs_y - 1) & 0x30) >> 2) + 203 | xy_sign_bits); 204 | glyph_stream_.push_back((((abs_x - 1) & 0xf) << 4) | ((abs_y - 1) & 0xf)); 205 | } else if (abs_x < 769 && abs_y < 769) { 206 | flag_byte_stream_.push_back(on_curve_bit + 84 + 207 | 12 * (((abs_x - 1) & 0x300) >> 8) + 208 | (((abs_y - 1) & 0x300) >> 6) + xy_sign_bits); 209 | glyph_stream_.push_back((abs_x - 1) & 0xff); 210 | glyph_stream_.push_back((abs_y - 1) & 0xff); 211 | } else if (abs_x < 4096 && abs_y < 4096) { 212 | flag_byte_stream_.push_back(on_curve_bit + 120 + xy_sign_bits); 213 | glyph_stream_.push_back(abs_x >> 4); 214 | glyph_stream_.push_back(((abs_x & 0xf) << 4) | (abs_y >> 8)); 215 | glyph_stream_.push_back(abs_y & 0xff); 216 | } else { 217 | flag_byte_stream_.push_back(on_curve_bit + 124 + xy_sign_bits); 218 | glyph_stream_.push_back(abs_x >> 8); 219 | glyph_stream_.push_back(abs_x & 0xff); 220 | glyph_stream_.push_back(abs_y >> 8); 221 | glyph_stream_.push_back(abs_y & 0xff); 222 | } 223 | } 224 | 225 | std::vector n_contour_stream_; 226 | std::vector n_points_stream_; 227 | std::vector flag_byte_stream_; 228 | std::vector composite_stream_; 229 | std::vector bbox_bitmap_; 230 | std::vector bbox_stream_; 231 | std::vector glyph_stream_; 232 | std::vector instruction_stream_; 233 | int n_glyphs_; 234 | }; 235 | 236 | } // namespace 237 | 238 | bool TransformGlyfAndLocaTables(Font* font) { 239 | // no transform for CFF 240 | const Font::Table* glyf_table = font->FindTable(kGlyfTableTag); 241 | const Font::Table* loca_table = font->FindTable(kLocaTableTag); 242 | if (font->FindTable(kCffTableTag) != NULL 243 | && glyf_table == NULL 244 | && loca_table == NULL) { 245 | return true; 246 | } 247 | // Must share neither or both loca/glyf 248 | if (glyf_table->IsReused() != loca_table->IsReused()) { 249 | return FONT_COMPRESSION_FAILURE(); 250 | } 251 | if (glyf_table->IsReused()) { 252 | return true; 253 | } 254 | Font::Table* transformed_glyf = &font->tables[kGlyfTableTag ^ 0x80808080]; 255 | Font::Table* transformed_loca = &font->tables[kLocaTableTag ^ 0x80808080]; 256 | 257 | int num_glyphs = NumGlyphs(*font); 258 | GlyfEncoder encoder(num_glyphs); 259 | for (int i = 0; i < num_glyphs; ++i) { 260 | Glyph glyph; 261 | const uint8_t* glyph_data; 262 | size_t glyph_size; 263 | if (!GetGlyphData(*font, i, &glyph_data, &glyph_size) || 264 | (glyph_size > 0 && !ReadGlyph(glyph_data, glyph_size, &glyph))) { 265 | return FONT_COMPRESSION_FAILURE(); 266 | } 267 | encoder.Encode(i, glyph); 268 | } 269 | encoder.GetTransformedGlyfBytes(&transformed_glyf->buffer); 270 | 271 | const Font::Table* head_table = font->FindTable(kHeadTableTag); 272 | if (head_table == NULL || head_table->length < 52) { 273 | return FONT_COMPRESSION_FAILURE(); 274 | } 275 | transformed_glyf->buffer[7] = head_table->data[51]; // index_format 276 | 277 | transformed_glyf->tag = kGlyfTableTag ^ 0x80808080; 278 | transformed_glyf->length = transformed_glyf->buffer.size(); 279 | transformed_glyf->data = transformed_glyf->buffer.data(); 280 | 281 | transformed_loca->tag = kLocaTableTag ^ 0x80808080; 282 | transformed_loca->length = 0; 283 | transformed_loca->data = NULL; 284 | 285 | return true; 286 | } 287 | 288 | } // namespace woff2 289 | -------------------------------------------------------------------------------- /csrc/woff2/transform.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Library for preprocessing fonts as part of the WOFF 2.0 conversion. 16 | 17 | #ifndef WOFF2_TRANSFORM_H_ 18 | #define WOFF2_TRANSFORM_H_ 19 | 20 | #include "./font.h" 21 | 22 | namespace woff2 { 23 | 24 | // Adds the transformed versions of the glyf and loca tables to the font. The 25 | // transformed loca table has zero length. The tag of the transformed tables is 26 | // derived from the original tag by flipping the MSBs of every byte. 27 | bool TransformGlyfAndLocaTables(Font* font); 28 | 29 | } // namespace woff2 30 | 31 | #endif // WOFF2_TRANSFORM_H_ 32 | -------------------------------------------------------------------------------- /csrc/woff2/variable_length.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helper functions for woff2 variable length types: 255UInt16 and UIntBase128 16 | 17 | #include "./variable_length.h" 18 | 19 | namespace woff2 { 20 | 21 | size_t Size255UShort(uint16_t value) { 22 | size_t result = 3; 23 | if (value < 253) { 24 | result = 1; 25 | } else if (value < 762) { 26 | result = 2; 27 | } else { 28 | result = 3; 29 | } 30 | return result; 31 | } 32 | 33 | void Write255UShort(std::vector* out, int value) { 34 | if (value < 253) { 35 | out->push_back(value); 36 | } else if (value < 506) { 37 | out->push_back(255); 38 | out->push_back(value - 253); 39 | } else if (value < 762) { 40 | out->push_back(254); 41 | out->push_back(value - 506); 42 | } else { 43 | out->push_back(253); 44 | out->push_back(value >> 8); 45 | out->push_back(value & 0xff); 46 | } 47 | } 48 | 49 | void Store255UShort(int val, size_t* offset, uint8_t* dst) { 50 | std::vector packed; 51 | Write255UShort(&packed, val); 52 | for (uint8_t val : packed) { 53 | dst[(*offset)++] = val; 54 | } 55 | } 56 | 57 | // Based on section 6.1.1 of MicroType Express draft spec 58 | bool Read255UShort(Buffer* buf, unsigned int* value) { 59 | static const int kWordCode = 253; 60 | static const int kOneMoreByteCode2 = 254; 61 | static const int kOneMoreByteCode1 = 255; 62 | static const int kLowestUCode = 253; 63 | uint8_t code = 0; 64 | if (!buf->ReadU8(&code)) { 65 | return FONT_COMPRESSION_FAILURE(); 66 | } 67 | if (code == kWordCode) { 68 | uint16_t result = 0; 69 | if (!buf->ReadU16(&result)) { 70 | return FONT_COMPRESSION_FAILURE(); 71 | } 72 | *value = result; 73 | return true; 74 | } else if (code == kOneMoreByteCode1) { 75 | uint8_t result = 0; 76 | if (!buf->ReadU8(&result)) { 77 | return FONT_COMPRESSION_FAILURE(); 78 | } 79 | *value = result + kLowestUCode; 80 | return true; 81 | } else if (code == kOneMoreByteCode2) { 82 | uint8_t result = 0; 83 | if (!buf->ReadU8(&result)) { 84 | return FONT_COMPRESSION_FAILURE(); 85 | } 86 | *value = result + kLowestUCode * 2; 87 | return true; 88 | } else { 89 | *value = code; 90 | return true; 91 | } 92 | } 93 | 94 | bool ReadBase128(Buffer* buf, uint32_t* value) { 95 | uint32_t result = 0; 96 | for (size_t i = 0; i < 5; ++i) { 97 | uint8_t code = 0; 98 | if (!buf->ReadU8(&code)) { 99 | return FONT_COMPRESSION_FAILURE(); 100 | } 101 | // If any of the top seven bits are set then we're about to overflow. 102 | if (result & 0xfe000000) { 103 | return FONT_COMPRESSION_FAILURE(); 104 | } 105 | result = (result << 7) | (code & 0x7f); 106 | if ((code & 0x80) == 0) { 107 | *value = result; 108 | return true; 109 | } 110 | } 111 | // Make sure not to exceed the size bound 112 | return FONT_COMPRESSION_FAILURE(); 113 | } 114 | 115 | size_t Base128Size(size_t n) { 116 | size_t size = 1; 117 | for (; n >= 128; n >>= 7) ++size; 118 | return size; 119 | } 120 | 121 | void StoreBase128(size_t len, size_t* offset, uint8_t* dst) { 122 | size_t size = Base128Size(len); 123 | for (int i = 0; i < size; ++i) { 124 | int b = static_cast((len >> (7 * (size - i - 1))) & 0x7f); 125 | if (i < size - 1) { 126 | b |= 0x80; 127 | } 128 | dst[(*offset)++] = b; 129 | } 130 | } 131 | 132 | } // namespace woff2 133 | 134 | -------------------------------------------------------------------------------- /csrc/woff2/variable_length.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helper functions for woff2 variable length types: 255UInt16 and UIntBase128 16 | 17 | #ifndef WOFF2_VARIABLE_LENGTH_H_ 18 | #define WOFF2_VARIABLE_LENGTH_H_ 19 | 20 | #include 21 | #include 22 | #include "./buffer.h" 23 | 24 | namespace woff2 { 25 | 26 | size_t Size255UShort(uint16_t value); 27 | bool Read255UShort(Buffer* buf, unsigned int* value); 28 | void Write255UShort(std::vector* out, int value); 29 | void Store255UShort(int val, size_t* offset, uint8_t* dst); 30 | 31 | size_t Base128Size(size_t n); 32 | bool ReadBase128(Buffer* buf, uint32_t* value); 33 | void StoreBase128(size_t len, size_t* offset, uint8_t* dst); 34 | 35 | } // namespace woff2 36 | 37 | #endif // WOFF2_VARIABLE_LENGTH_H_ 38 | 39 | -------------------------------------------------------------------------------- /csrc/woff2/woff2_common.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Helpers common across multiple parts of woff2 16 | 17 | #include 18 | 19 | #include "./woff2_common.h" 20 | 21 | namespace woff2 { 22 | 23 | 24 | uint32_t ComputeULongSum(const uint8_t* buf, size_t size) { 25 | uint32_t checksum = 0; 26 | for (size_t i = 0; i < size; i += 4) { 27 | // We assume the addition is mod 2^32, which is valid because unsigned 28 | checksum += (buf[i] << 24) | (buf[i + 1] << 16) | 29 | (buf[i + 2] << 8) | buf[i + 3]; 30 | } 31 | return checksum; 32 | } 33 | 34 | size_t CollectionHeaderSize(uint32_t header_version, uint32_t num_fonts) { 35 | size_t size = 0; 36 | if (header_version == 0x00020000) { 37 | size += 12; // ulDsig{Tag,Length,Offset} 38 | } 39 | if (header_version == 0x00010000 || header_version == 0x00020000) { 40 | size += 12 // TTCTag, Version, numFonts 41 | + 4 * num_fonts; // OffsetTable[numFonts] 42 | } 43 | return size; 44 | } 45 | 46 | } // namespace woff2 47 | -------------------------------------------------------------------------------- /csrc/woff2/woff2_common.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Common definition for WOFF2 encoding/decoding 16 | 17 | #ifndef WOFF2_WOFF2_COMMON_H_ 18 | #define WOFF2_WOFF2_COMMON_H_ 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | namespace woff2 { 26 | 27 | static const uint32_t kWoff2Signature = 0x774f4632; // "wOF2" 28 | 29 | const unsigned int kWoff2FlagsContinueStream = 1 << 4; 30 | const unsigned int kWoff2FlagsTransform = 1 << 5; 31 | 32 | // TrueType Collection ID string: 'ttcf' 33 | static const uint32_t kTtcFontFlavor = 0x74746366; 34 | 35 | static const size_t kSfntHeaderSize = 12; 36 | static const size_t kSfntEntrySize = 16; 37 | 38 | struct Point { 39 | int x; 40 | int y; 41 | bool on_curve; 42 | }; 43 | 44 | struct Table { 45 | uint32_t tag; 46 | uint32_t flags; 47 | uint32_t src_offset; 48 | uint32_t src_length; 49 | 50 | uint32_t transform_length; 51 | 52 | uint32_t dst_offset; 53 | uint32_t dst_length; 54 | const uint8_t* dst_data; 55 | 56 | bool operator<(const Table& other) const { 57 | return tag < other.tag; 58 | } 59 | }; 60 | 61 | 62 | // Size of the collection header. 0 if version indicates this isn't a 63 | // collection. Ref http://www.microsoft.com/typography/otspec/otff.htm, 64 | // True Type Collections 65 | size_t CollectionHeaderSize(uint32_t header_version, uint32_t num_fonts); 66 | 67 | // Compute checksum over size bytes of buf 68 | uint32_t ComputeULongSum(const uint8_t* buf, size_t size); 69 | 70 | } // namespace woff2 71 | 72 | #endif // WOFF2_WOFF2_COMMON_H_ 73 | -------------------------------------------------------------------------------- /csrc/woff2/woff2_dec.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Library for converting WOFF2 format font files to their TTF versions. 16 | 17 | #ifndef WOFF2_WOFF2_DEC_H_ 18 | #define WOFF2_WOFF2_DEC_H_ 19 | 20 | #include 21 | #include 22 | 23 | namespace woff2 { 24 | 25 | // Compute the size of the final uncompressed font, or 0 on error. 26 | size_t ComputeWOFF2FinalSize(const uint8_t *data, size_t length); 27 | 28 | // Decompresses the font into the target buffer. The result_length should 29 | // be the same as determined by ComputeFinalSize(). Returns true on successful 30 | // decompression. 31 | bool ConvertWOFF2ToTTF(uint8_t *result, size_t result_length, 32 | const uint8_t *data, size_t length); 33 | 34 | } // namespace woff2 35 | 36 | #endif // WOFF2_WOFF2_DEC_H_ 37 | -------------------------------------------------------------------------------- /csrc/woff2/woff2_enc.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // Library for converting WOFF2 format font files to their TTF versions. 16 | 17 | #ifndef WOFF2_WOFF2_ENC_H_ 18 | #define WOFF2_WOFF2_ENC_H_ 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | using std::string; 25 | 26 | 27 | namespace woff2 { 28 | 29 | // Returns an upper bound on the size of the compressed file. 30 | size_t MaxWOFF2CompressedSize(const uint8_t* data, size_t length); 31 | size_t MaxWOFF2CompressedSize(const uint8_t* data, size_t length, 32 | const string& extended_metadata); 33 | 34 | // Compresses the font into the target buffer. *result_length should be at least 35 | // the value returned by MaxWOFF2CompressedSize(), upon return, it is set to the 36 | // actual compressed size. Returns true on successful compression. 37 | bool ConvertTTFToWOFF2(const uint8_t *data, size_t length, 38 | uint8_t *result, size_t *result_length); 39 | 40 | bool ConvertTTFToWOFF2(const uint8_t *data, size_t length, 41 | uint8_t *result, size_t *result_length, 42 | const string& extended_metadata); 43 | 44 | } // namespace woff2 45 | 46 | #endif // WOFF2_WOFF2_ENC_H_ 47 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // This file is automatically generated by a `metapak` 3 | // module. Do not change it elsewhere, changes would 4 | // be overridden. 5 | 6 | import eslint from '@eslint/js'; 7 | import tseslint from 'typescript-eslint'; 8 | import eslintConfigPrettier from 'eslint-config-prettier'; 9 | import eslintPluginJest from 'eslint-plugin-jest'; 10 | 11 | export default tseslint.config( 12 | { 13 | files: ['**/*.ts'], 14 | ignores: ['**/*.d.ts'], 15 | extends: [ 16 | eslint.configs.recommended, 17 | ...tseslint.configs.recommended, 18 | ], 19 | }, 20 | { 21 | files: ['*.test.ts'], 22 | ...eslintPluginJest.configs['flat/recommended'], 23 | }, 24 | eslintConfigPrettier, 25 | { 26 | name: 'Project config', 27 | languageOptions: { 28 | ecmaVersion: 2018, 29 | sourceType: 'module', 30 | }, 31 | ignores: ['*.d.ts'], 32 | }, 33 | ); 34 | 35 | -------------------------------------------------------------------------------- /fixtures/iconsfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfroidure/ttf2woff2/3a9b14734fe166d3f7db65d8d8407a7f08540da5/fixtures/iconsfont.ttf -------------------------------------------------------------------------------- /fixtures/iconsfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfroidure/ttf2woff2/3a9b14734fe166d3f7db65d8d8407a7f08540da5/fixtures/iconsfont.woff2 -------------------------------------------------------------------------------- /jssrc/index.js: -------------------------------------------------------------------------------- 1 | import theTTFToWOFF2Module from './ttf2woff2.cjs'; 2 | 3 | export default function ttf2woff2(inputContent) { 4 | // Prepare input 5 | const inputBuffer = theTTFToWOFF2Module._malloc(inputContent.length + 1); 6 | const outputSizePtr = theTTFToWOFF2Module._malloc(4); // eslint-disable-line 7 | let outputBufferPtr; 8 | let outputSize; 9 | let outputContent; 10 | 11 | theTTFToWOFF2Module.writeArrayToMemory(inputContent, inputBuffer); 12 | 13 | // Run 14 | outputBufferPtr = theTTFToWOFF2Module.convert( 15 | inputBuffer, 16 | inputContent.length, 17 | outputSizePtr, 18 | ); 19 | 20 | // Retrieve output 21 | outputSize = theTTFToWOFF2Module.getValue(outputSizePtr, 'i32'); 22 | outputContent = Buffer.alloc(outputSize); 23 | 24 | for (let i = 0; i < outputSize; i++) { 25 | outputContent[i] = theTTFToWOFF2Module.getValue(outputBufferPtr + i, 'i8'); 26 | } 27 | 28 | theTTFToWOFF2Module.freePtrs(outputBufferPtr, outputSizePtr); 29 | 30 | return outputContent; 31 | }; 32 | -------------------------------------------------------------------------------- /jssrc/post.js: -------------------------------------------------------------------------------- 1 | // This file need to be append to the build in order to work with browserify 2 | // Shamelessly stolen here: https://github.com/fabiosantoscode/require-emscripten/blob/master/post-js.postjs 3 | module.exports = Module; 4 | 5 | // Do not recurse into module and waste all day 6 | Module.inspect = function () { 7 | return '[Module]'; 8 | }; 9 | -------------------------------------------------------------------------------- /jssrc/ttf2woff2.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfroidure/ttf2woff2/3a9b14734fe166d3f7db65d8d8407a7f08540da5/jssrc/ttf2woff2.wasm -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "metapak": { 3 | "configs": [ 4 | "main", 5 | "readme", 6 | "tsesm", 7 | "jest", 8 | "eslint" 9 | ], 10 | "data": { 11 | "files": "'src/**/*.ts'", 12 | "testsFiles": "'src/**/*.tests.ts'", 13 | "distFiles": "'dist/**/*.js'", 14 | "ignore": [ 15 | "dist", 16 | "build" 17 | ], 18 | "bundleFiles": [ 19 | "dist", 20 | "src", 21 | "jssrc", 22 | "binding.gyp", 23 | "csrc" 24 | ] 25 | } 26 | }, 27 | "name": "ttf2woff2", 28 | "version": "7.0.0", 29 | "description": "Convert TTF files to WOFF2 ones.", 30 | "main": "dist/index.js", 31 | "browser": "jssrc/index.js", 32 | "engines": { 33 | "node": ">=20.11.1" 34 | }, 35 | "directories": { 36 | "test": "tests" 37 | }, 38 | "scripts": { 39 | "build": "rimraf 'dist' && tsc --outDir dist", 40 | "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md", 41 | "cli": "env NODE_ENV=${NODE_ENV:-cli}", 42 | "configure": "node-gyp configure", 43 | "cover": "npm run jest -- --coverage", 44 | "cz": "env NODE_ENV=${NODE_ENV:-cli} git cz", 45 | "emcc": "miniquery -p \"targets.#.sources.#\" ./binding.gyp | grep -v \"csrc/addon.cc\" | xargs emcc --bind -o jssrc/ttf2woff2.cjs -O2 -s \"TOTAL_MEMORY=536870912\" -s \"ALLOW_MEMORY_GROWTH=1\" -s BINARYEN_ASYNC_COMPILATION=0 -s EXPORTED_FUNCTIONS=[\"_malloc\"] -s 'EXPORTED_RUNTIME_METHODS=[\"getValue\", \"writeArrayToMemory\"]' -s NODEJS_CATCH_EXIT=0 -s NODEJS_CATCH_REJECTION=0 --post-js jssrc/post.js csrc/fallback.cc", 46 | "emcc-debug": "miniquery -p \"targets.#.sources.#\" ./binding.gyp | grep -v \"csrc/addon.cc\" | xargs emcc --bind -o jssrc/ttf2woff2.cjs -s \"ALLOW_MEMORY_GROWTH=1\" -s \"ASSERTIONS=1\" -s BINARYEN_ASYNC_COMPILATION=0 -s EXPORTED_FUNCTIONS=[\"_malloc\"] -s 'EXPORTED_RUNTIME_METHODS=[\"getValue\", \"writeArrayToMemory\"]' -s NODEJS_CATCH_EXIT=0 -s NODEJS_CATCH_REJECTION=0 --post-js jssrc/post.js csrc/fallback.cc", 47 | "format": "npm run prettier", 48 | "install": "((node-gyp configure && node-gyp build) > builderror.log) || (exit 0)", 49 | "jest": "NODE_OPTIONS=--experimental-vm-modules NODE_ENV=test jest", 50 | "lint": "eslint 'src/**/*.ts'", 51 | "make": "node-gyp build", 52 | "metapak": "metapak", 53 | "precz": "npm t && npm run lint && npm run build && npm run metapak -- -s", 54 | "prepare": "npm run build", 55 | "prettier": "prettier --write 'src/**/*.ts'", 56 | "preversion": "npm run lint && npm test && npm t && npm run build && npm run metapak -- -s", 57 | "rebuild": "swc ./src -s -d dist -C jsc.target=es2022", 58 | "test": "npm run jest", 59 | "type-check": "tsc --pretty --noEmit", 60 | "version": "npm run changelog" 61 | }, 62 | "repository": { 63 | "type": "git", 64 | "url": "git@github.com:nfroidure/ttf2woff2.git" 65 | }, 66 | "keywords": [ 67 | "ttf", 68 | "woff2", 69 | "fonts" 70 | ], 71 | "author": { 72 | "name": "Nicolas Froidure", 73 | "email": "nicolas.froidure@insertafter.com", 74 | "url": "https://insertafter.com/en/index.html" 75 | }, 76 | "license": "MIT", 77 | "bugs": { 78 | "url": "https://github.com/nfroidure/ttf2woff2/issues" 79 | }, 80 | "homepage": "https://github.com/nfroidure/ttf2woff2", 81 | "dependencies": { 82 | "bindings": "^1.5.0", 83 | "bufferstreams": "^4.0.0", 84 | "debug": "^4.4.1", 85 | "nan": "^2.22.2", 86 | "node-gyp": "^11.2.0", 87 | "yerror": "^8.0.0" 88 | }, 89 | "devDependencies": { 90 | "@eslint/js": "^9.16.0", 91 | "@swc/cli": "^0.5.2", 92 | "@swc/core": "^1.10.0", 93 | "@swc/helpers": "^0.5.15", 94 | "@swc/jest": "^0.2.37", 95 | "commitizen": "^4.3.1", 96 | "conventional-changelog-cli": "^5.0.0", 97 | "cz-conventional-changelog": "^3.3.0", 98 | "eslint": "^9.16.0", 99 | "eslint-config-prettier": "^9.1.0", 100 | "eslint-plugin-jest": "^28.9.0", 101 | "eslint-plugin-prettier": "^5.2.1", 102 | "jest": "^29.7.0", 103 | "metapak": "^6.0.2", 104 | "metapak-nfroidure": "^19.0.1", 105 | "miniquery": "^1.1.2", 106 | "prettier": "^3.4.2", 107 | "rimraf": "^6.0.1", 108 | "typescript": "^5.7.2", 109 | "typescript-eslint": "^8.17.0" 110 | }, 111 | "bin": { 112 | "ttf2woff2": "bin/ttf2woff2.js" 113 | }, 114 | "gypfile": true, 115 | "contributors": [ 116 | { 117 | "name": "Anders Kaseorg", 118 | "email": "andersk@mit.edu" 119 | } 120 | ], 121 | "files": [ 122 | "dist", 123 | "src", 124 | "jssrc", 125 | "binding.gyp", 126 | "csrc", 127 | "LICENSE", 128 | "README.md", 129 | "CHANGELOG.md" 130 | ], 131 | "config": { 132 | "commitizen": { 133 | "path": "./node_modules/cz-conventional-changelog" 134 | } 135 | }, 136 | "greenkeeper": { 137 | "ignore": [ 138 | "commitizen", 139 | "cz-conventional-changelog", 140 | "conventional-changelog-cli", 141 | "typescript", 142 | "rimraf", 143 | "@swc/cli", 144 | "@swc/core", 145 | "@swc/helpers", 146 | "jest", 147 | "@swc/jest", 148 | "eslint", 149 | "prettier", 150 | "eslint-config-prettier", 151 | "eslint-plugin-prettier", 152 | "typescript-eslint" 153 | ] 154 | }, 155 | "eslintIgnore": [ 156 | "jssrc/ttf2woff2.wasm", 157 | "jssrc/ttf2woff2.js", 158 | "jssrc/post.js" 159 | ], 160 | "prettier": { 161 | "semi": true, 162 | "printWidth": 80, 163 | "singleQuote": true, 164 | "trailingComma": "all", 165 | "proseWrap": "always" 166 | }, 167 | "type": "module", 168 | "types": "dist/index.d.ts", 169 | "jest": { 170 | "coverageReporters": [ 171 | "lcov" 172 | ], 173 | "testPathIgnorePatterns": [ 174 | "/node_modules/" 175 | ], 176 | "roots": [ 177 | "/src" 178 | ], 179 | "transform": { 180 | "^.+\\.tsx?$": [ 181 | "@swc/jest", 182 | {} 183 | ] 184 | }, 185 | "testEnvironment": "node", 186 | "moduleNameMapper": { 187 | "(.+)\\.js": "$1" 188 | }, 189 | "extensionsToTreatAsEsm": [ 190 | ".ts" 191 | ], 192 | "prettierPath": null 193 | }, 194 | "overrides": { 195 | "eslint": "^9.16.0" 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/cli.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect, jest } from '@jest/globals'; 2 | import { readFile } from 'node:fs/promises'; 3 | 4 | describe('Testing CLI', () => { 5 | test('should work', async () => { 6 | jest.setTimeout(5000); 7 | expect( 8 | (await import('child_process')).execSync('node ./bin/ttf2woff2.js', { 9 | input: await readFile('fixtures/iconsfont.ttf'), 10 | }), 11 | ).toEqual(await readFile('fixtures/iconsfont.woff2')); 12 | }); 13 | }); 14 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { YError, printStackTrace } from 'yerror'; 2 | import debug from 'debug'; 3 | import { env } from 'node:process'; 4 | import bindings from 'bindings' 5 | import ttf2woff2Loader from '../jssrc/index.js'; 6 | 7 | const doDebug = debug('ttf2woff2'); 8 | const ttf2woff2 = getExecutable(); 9 | 10 | function getExecutable() { 11 | if (!env.TTF2WOFF2_VERSION || env.TTF2WOFF2_VERSION?.toLowerCase() === 'native') { 12 | try { 13 | doDebug('✅ Using native version.'); 14 | return bindings.default('addon.node').convert; 15 | } catch (err) { 16 | doDebug('❌ Could not load the native version.', printStackTrace(err as Error)); 17 | } 18 | } 19 | 20 | if (!env.TTF2WOFF2_VERSION || env.TTF2WOFF2_VERSION?.toLowerCase() === 'wasm') { 21 | try { 22 | doDebug('✅ Using WASM version.'); 23 | return ttf2woff2Loader; 24 | } catch (err) { 25 | doDebug('❌ Could not load the WASM version.', printStackTrace(err as Error)); 26 | } 27 | } 28 | 29 | throw new YError('E_UNABLE_TO_LOAD_TTF2WOFF2', env.TTF2WOFF2_VERSION); 30 | } 31 | 32 | export default ttf2woff2 as NonNullable; 33 | -------------------------------------------------------------------------------- /src/tests.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect, jest } from '@jest/globals'; 2 | import { readFile } from 'node:fs/promises'; 3 | 4 | describe('ttf2woff2', () => { 5 | test('should work from the main endpoint', async () => { 6 | jest.setTimeout(5000); 7 | 8 | const ttf2woff2 = (await import('./index.js')).default; 9 | const inputContent = await readFile('fixtures/iconsfont.ttf'); 10 | const outputContent = ttf2woff2(inputContent); 11 | 12 | expect(outputContent.length).toEqual(1072); 13 | expect(outputContent[1071]).toEqual(0); 14 | expect(outputContent).toEqual(await readFile('fixtures/iconsfont.woff2')); 15 | }); 16 | 17 | test('should work from the native build', async () => { 18 | const ttf2woff2 = (await import('bindings')).default('addon.node').convert; 19 | const inputContent = await readFile('fixtures/iconsfont.ttf'); 20 | const outputContent = ttf2woff2(inputContent); 21 | 22 | expect(outputContent.length).toEqual(1072); 23 | expect(outputContent[1071]).toEqual(0); 24 | expect(outputContent).toEqual(await readFile('fixtures/iconsfont.woff2')); 25 | }); 26 | 27 | test('should work from the emscripten endpoint', async () => { 28 | jest.setTimeout(5000); 29 | 30 | const ttf2woff2 = (await import('../jssrc/index.js')).default; 31 | const inputContent = await readFile('fixtures/iconsfont.ttf'); 32 | const outputContent = ttf2woff2(inputContent); 33 | 34 | expect(outputContent.length).toEqual(1072); 35 | expect(outputContent[1071]).toEqual(0); 36 | expect(outputContent).toEqual(await readFile('fixtures/iconsfont.woff2')); 37 | }); 38 | }); 39 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "Node16", 4 | "moduleResolution": "Node16", 5 | "target": "es2022", 6 | "noImplicitAny": false, 7 | "removeComments": false, 8 | "preserveConstEnums": true, 9 | "allowSyntheticDefaultImports": true, 10 | "esModuleInterop": true, 11 | "strict": true, 12 | "declaration": true, 13 | "outDir": "dist", 14 | "sourceMap": true 15 | }, 16 | "include": [ 17 | "src/**/*.ts" 18 | ], 19 | "exclude": [ 20 | "node_modules" 21 | ] 22 | } --------------------------------------------------------------------------------