├── .github ├── stale.yml └── workflows │ ├── bench.yml │ ├── cargo-publish.yml │ └── rust.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Cargo.toml ├── LICENSE ├── README.md ├── benches ├── lorem-ipsum.txt └── lzzzz.rs ├── build.rs ├── justfile ├── lzzzz.png ├── rustfmt.toml ├── src ├── common │ ├── api.rs │ ├── binding.rs │ ├── error.rs │ └── mod.rs ├── lib.rs ├── lz4 │ ├── binding.rs │ ├── block │ │ ├── api.rs │ │ └── mod.rs │ ├── mod.rs │ └── stream │ │ ├── api.rs │ │ └── mod.rs ├── lz4_hc │ ├── binding.rs │ ├── block │ │ ├── api.rs │ │ └── mod.rs │ ├── mod.rs │ └── stream │ │ ├── api.rs │ │ └── mod.rs └── lz4f │ ├── api.rs │ ├── binding.rs │ ├── dictionary.rs │ ├── error.rs │ ├── frame.rs │ ├── frame_info.rs │ ├── mod.rs │ ├── preferences.rs │ └── stream │ ├── comp │ ├── bufread.rs │ ├── mod.rs │ ├── read.rs │ └── write.rs │ ├── decomp │ ├── bufread.rs │ ├── mod.rs │ ├── read.rs │ └── write.rs │ └── mod.rs ├── tests ├── common │ └── mod.rs ├── lz4.rs ├── lz4_hc.rs ├── lz4_hc_stream.rs ├── lz4_stream.rs ├── lz4f.rs └── lz4f_stream.rs └── vendor └── liblz4 ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── dll └── example │ ├── Makefile │ ├── README.md │ ├── fullbench-dll.sln │ └── fullbench-dll.vcxproj ├── liblz4-dll.rc.in ├── liblz4.pc.in ├── lz4.c ├── lz4.h ├── lz4file.c ├── lz4file.h ├── lz4frame.c ├── lz4frame.h ├── lz4frame_static.h ├── lz4hc.c ├── lz4hc.h ├── xxhash.c └── xxhash.h /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking an issue as stale 10 | staleLabel: wontfix 11 | # Comment to post when marking an issue as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false -------------------------------------------------------------------------------- /.github/workflows/bench.yml: -------------------------------------------------------------------------------- 1 | name: Bench 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | 8 | jobs: 9 | bench: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | os: [macos-latest, windows-latest, ubuntu-latest] 14 | toolchain: [stable, nightly] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/.rustup/toolchains/nightly-* 22 | ~/.cargo/registry 23 | ~/.cargo/git 24 | target 25 | key: ${{ runner.os }}-${{ matrix.toolchain }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} 26 | - name: Install nightly toolchain 27 | run: rustup install nightly 28 | if: matrix.toolchain == 'nightly' 29 | - name: Run bench 30 | run: cargo +${{ matrix.toolchain }} bench --all-features 31 | -------------------------------------------------------------------------------- /.github/workflows/cargo-publish.yml: -------------------------------------------------------------------------------- 1 | name: Cargo Publish 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | env: 8 | CARGO_TERM_COLOR: always 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Run tests 16 | run: cargo test --release --all-features 17 | 18 | cargo-publish: 19 | needs: test 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v2 23 | - run: cargo publish 24 | env: 25 | CARGO_REGISTRY_TOKEN: ${{secrets.cargo_registry_token}} 26 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | 8 | jobs: 9 | test: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | os: [macos-latest, windows-latest, ubuntu-latest] 14 | toolchain: [stable, nightly] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/.rustup/toolchains/nightly-* 22 | ~/.cargo/registry 23 | ~/.cargo/git 24 | target 25 | key: ${{ runner.os }}-${{ matrix.toolchain }}-cargo-${{ hashFiles('**/Cargo.lock') }} 26 | - name: Install nightly toolchain 27 | run: rustup install nightly 28 | if: matrix.toolchain == 'nightly' 29 | - name : Install clippy 30 | run: rustup component add clippy --toolchain ${{ matrix.toolchain }} 31 | - name: Run clippy 32 | run: cargo +${{ matrix.toolchain }} clippy --release --all-features --verbose 33 | - name: Run tests 34 | run: cargo +${{ matrix.toolchain }} test --release --all-features --verbose 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | 12 | 13 | #Added by cargo 14 | # 15 | #already existing elements were commented out 16 | 17 | /target 18 | #Cargo.lock 19 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | * **Be mindful of your language.** Any of the following behavior is unacceptable: 4 | * Offensive comments related to gender identity and expression, sexual orientation, race, ethnicity, language, neuro-type, size, ability, class, religion, culture, subculture, political opinion, age, skill level, occupation, or background 5 | * Threats of violence 6 | * Deliberate intimidation 7 | * Sexually explicit or violent material that is not contextualized and preceded by a considerate warning 8 | * Unwelcome sexual attention 9 | * Stalking or following 10 | * Or any other kinds of harassment 11 | 12 | Use your best judgement. If it will possibly make others uncomfortable, do not post it. 13 | 14 | * **Be respectful.** Disagreement is not an opportunity to attack someone else's thoughts or opinions. Although views may differ, remember to approach every situation with patience and care. 15 | * **Be considerate.** Think about how your contribution will affect others in the community. 16 | * **Be open minded.** Embrace new people and new ideas. Our community is continually evolving and we welcome positive change. 17 | 18 | If you believe someone is violating the code of conduct, we ask that you report it by emailing `[Github_Username]@outlook.com`. Please include your name and a description of the incident, and we will get back to you ASAP. 19 | 20 | Sometimes, participants violating the Code of Conduct are unaware that their behavior is harmful, and an open conversation clears things up to move forward. However, if a participant continues with the behavior, the Lzzzz team may take any action they deem appropriate, up to and including expulsion from all Lzzzz spaces and identification of the participant as a harasser to other Lzzzz members or the general public. 21 | 22 | --- 23 | This statement is based on [p5.js Code of Conduct](https://github.com/processing/p5.js/blob/master/CODE_OF_CONDUCT.md). 24 | Licensed under [CC BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/). 25 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lzzzz" 3 | version = "2.0.0" 4 | authors = ["picoHz "] 5 | edition = "2021" 6 | description = "Full-featured liblz4 binding for Rust." 7 | keywords = ["lz4", "lz4f", "lz4-hc", "compression", "decompression"] 8 | categories = ["compression", "api-bindings"] 9 | repository = "https://github.com/picoHz/lzzzz" 10 | homepage = "https://github.com/picoHz/lzzzz" 11 | documentation = "https://docs.rs/lzzzz" 12 | license = "MIT" 13 | readme = "README.md" 14 | include = ["src/**/*", "build.rs", "vendor/liblz4/*", "Cargo.toml", "LICENSE"] 15 | 16 | [package.metadata.docs.rs] 17 | all-features = true 18 | rustdoc-args = ["--cfg", "docsrs"] 19 | 20 | [dev-dependencies] 21 | assert_fs = "1.0.6" 22 | base64 = "0.22.1" 23 | bytes = "1.1.0" 24 | lazy_static = "1.4.0" 25 | rand = { version = "0.8.4", features = ["small_rng"] } 26 | rayon = "1.5.1" 27 | static_assertions = "1.1.0" 28 | criterion = "0.5.1" 29 | 30 | [build-dependencies] 31 | cc = { version = "1.0.72", features = ["parallel"] } 32 | 33 | [[bench]] 34 | name = "lzzzz" 35 | harness = false 36 | path = "benches/lzzzz.rs" 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 picoHz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | lzzzz 3 | 4 | Full-featured [liblz4](https://github.com/lz4/lz4) binding for Rust 5 | 6 | [![Crates.io](https://img.shields.io/crates/v/lzzzz.svg)](https://crates.io/crates/lzzzz) 7 | [![GitHub license](https://img.shields.io/github/license/picoHz/lzzzz.svg)](https://github.com/picoHz/lzzzz/blob/master/LICENSE) 8 | [![Rustdoc](https://img.shields.io/badge/doc-rustdoc-green.svg)](https://docs.rs/lzzzz) 9 | ![Rust](https://github.com/picoHz/lzzzz/workflows/Rust/badge.svg) 10 | 11 |
12 | 13 | --- 14 | 15 | ## About 16 | 17 | Rust APIs for the [LZ4](https://lz4.github.io/lz4/) compression algorithm. 18 | 19 | - Supports almost all liblz4 features 20 | - Zero dependencies except liblz4 21 | - Tested on Windows / macOS / Linux 22 | 23 | ## Usage 24 | 25 | Add this to your `Cargo.toml`: 26 | 27 | ```toml 28 | [dependencies] 29 | lzzzz = "2.0.0" 30 | ``` 31 | 32 | [API Documentation](https://docs.rs/lzzzz) 33 | 34 | ## Features 35 | 36 | - LZ4 37 | - Compression (Block / Streaming) 38 | - Decompression (Block / Streaming) 39 | - Partial Decompression 40 | - Custom Dictionary 41 | - LZ4_HC 42 | - Compression (Block / Streaming) 43 | - Partial Compression 44 | - Custom Dictionary 45 | - LZ4F 46 | - Compression 47 | - Decompression 48 | - Custom Dictionary 49 | - Streaming I/O (`Read` / `BufRead` / `Write`) 50 | 51 | ## Examples 52 | 53 | ### Block Mode 54 | 55 | ```rust 56 | use lzzzz::{lz4, lz4_hc, lz4f}; 57 | 58 | let data = b"The quick brown fox jumps over the lazy dog."; 59 | 60 | // LZ4 compression 61 | let mut comp = Vec::new(); 62 | lz4::compress_to_vec(data, &mut comp, lz4::ACC_LEVEL_DEFAULT)?; 63 | 64 | // LZ4_HC compression 65 | let mut comp = Vec::new(); 66 | lz4_hc::compress_to_vec(data, &mut comp, lz4_hc::CLEVEL_DEFAULT)?; 67 | 68 | // LZ4/LZ4_HC decompression 69 | let mut decomp = vec![0; data.len()]; 70 | lz4::decompress(&comp, &mut decomp)?; 71 | 72 | // LZ4F compression 73 | let prefs = lz4f::Preferences::default(); 74 | let mut comp = Vec::new(); 75 | lz4f::compress_to_vec(data, &mut comp, &prefs)?; 76 | 77 | // LZ4F decompression 78 | let mut decomp = Vec::new(); 79 | lz4f::decompress_to_vec(&comp, &mut decomp)?; 80 | ``` 81 | 82 | ### Streaming Mode 83 | 84 | ```rust 85 | use lzzzz::{lz4, lz4_hc}; 86 | 87 | let data = b"The quick brown fox jumps over the lazy dog."; 88 | 89 | // LZ4 compression 90 | let mut comp = lz4::Compressor::new()?; 91 | let mut buf = Vec::new(); 92 | comp.next_to_vec(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 93 | 94 | // LZ4_HC compression 95 | let mut comp = lz4_hc::Compressor::new()?; 96 | let mut buf = Vec::new(); 97 | comp.next_to_vec(data, &mut buf)?; 98 | 99 | // LZ4/LZ4_HC decompression 100 | let mut decomp = lz4::Decompressor::new()?; 101 | let result = decomp.next(&data, data.len())?; 102 | ``` 103 | 104 | ```rust 105 | use lzzzz::lz4f::{WriteCompressor, ReadDecompressor, Preferences}; 106 | use std::{fs::File, io::prelude::*}; 107 | 108 | // LZ4F Write-based compression 109 | let mut f = File::create("foo.lz4")?; 110 | let mut w = WriteCompressor::new(&mut f, Preferences::default())?; 111 | w.write_all(b"Hello world!")?; 112 | 113 | // LZ4F Read-based decompression 114 | let mut f = File::open("foo.lz4")?; 115 | let mut r = ReadDecompressor::new(&mut f)?; 116 | let mut buf = Vec::new(); 117 | r.read_to_end(&mut buf)?; 118 | ``` 119 | -------------------------------------------------------------------------------- /benches/lorem-ipsum.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, eros summo minim quo ex, recteque voluptaria ne sed. No vitae nostro quo, mazim tempor tincidunt ut sea, no mea utamur suscipit facilisis. Nec an stet possit moderatius, prodesset neglegentur vis et. Sit ut error petentium scriptorem, ne vim magna mediocrem dissentiunt. Quem constituto inciderint vix ut, eu option noluisse pri, vel an solet latine fabellas. Detraxit concludaturque ne cum, te vix eius vivendum, alia imperdiet qui ea. 2 | 3 | Ut mei eius appareat mnesarchum, eam amet percipit petentium et. Ius eu eruditi verterem conclusionemque, ei nec error interesset. Omnesque officiis id eum, eum ea quod harum, labitur praesent theophrastus pro te. Eirmod accumsan eum in. His ad agam deseruisse instructior. 4 | 5 | Ferri fabellas necessitatibus eos cu, te persius hendrerit vel. Cum tota velit no, et elitr munere eam, doming cotidieque duo ex. Te alterum volutpat similique mel, nec nostro virtute democritum te. Sale mucius sed et. 6 | 7 | His mazim virtute interesset cu, per et solet dignissim intellegam. Ne movet verterem aliquando pro. Cu vitae homero vel. Per dissentias disputando ei, has ei decore oporteat. Ei sea oportere temporibus, possim deseruisse mei ex, illum aeque at vix. 8 | 9 | Ne vocibus posidonium sed, eu vim nulla aliquid. Duo in solet legimus veritus, id essent ceteros atomorum vix. Nam purto habeo labores ea. Nam ea utamur laoreet. Id dicant deserunt intellegam duo, quo magna modus meliore ex. 10 | 11 | Mel diceret phaedrum sensibus te, vix habemus molestie te. An nam essent nostrud. Sea id ignota impetus urbanitas, vis an exerci regione corpora, cu debet labitur praesent quo. Autem adolescens reprehendunt cum ex. Te causae propriae concludaturque vel. Alia maiorum sit in. 12 | 13 | Nec nusquam fabellas ex, mei diam stet necessitatibus ex, erant ceteros interesset ut ius. Sit id nobis scaevola deseruisse. Nec copiosae elaboraret vituperatoribus id, vix et accusamus patrioque, docendi evertitur conceptam duo te. Altera intellegebat ut vim, eum in cibo consul. Error appetere volutpat et mea, sit dolore diceret no, electram percipitur sea an. 14 | 15 | Purto fabulas commune eu mel, ne habemus intellegat vix. Vel ea epicurei adipisci, nusquam expetenda te eos, ut mei mollis recusabo. Sea eligendi consulatu quaerendum et, mei wisi nominati id, eius tollit vim ei. Aliquip adipiscing mel ad, ex paulo habemus nec. Minim dictas sit eu, vitae perpetua neglegentur pro in. Mel duis voluptua eu, euismod recteque id cum. 16 | 17 | Et virtute verterem forensibus mel. In nam affert mucius philosophia. Eum invenire mnesarchum comprehensam et. Alii animal constituto cu nec, nibh postea splendide sed ne. Debet oporteat mnesarchum nec cu. 18 | 19 | Usu nostrum blandit ex, natum malis no vix. Vel ei nisl expetendis, ne falli definitionem mea. Mei prima ullamcorper reprehendunt ea, ea mel definiebas theophrastus. Iuvaret intellegebat an ius, ex vim dolores assueverit. Cu pertinax ullamcorper mea, causae concludaturque eu pro. 20 | 21 | Verterem delicatissimi pri et, in autem repudiare quo. Sea euismod laboramus ut, no his illum officiis eloquentiam, per ut doctus reprehendunt vituperatoribus. Errem labores scriptorem nec no, vix cu antiopam periculis. Te mea graece expetenda sententiae, feugiat scripserit ea eum, no sed odio wisi eruditi. 22 | 23 | Alii mentitum te quo, ea etiam elaboraret sit, no placerat referrentur pro. Vim oblique labores signiferumque te, pro eu qualisque prodesset. No aperiam graecis pri, ne erat possim ponderum per. Sed et affert aperiam imperdiet. Te porro nominati vel, at deserunt mnesarchum vel. Ullum periculis vis an. Cu sit amet homero semper. -------------------------------------------------------------------------------- /benches/lzzzz.rs: -------------------------------------------------------------------------------- 1 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 2 | use lzzzz::{lz4, lz4_hc, lz4f}; 3 | use std::{ 4 | i32, 5 | io::{Read, Write}, 6 | }; 7 | 8 | fn lz4_compress(level: i32, data: &[u8]) { 9 | let mut buf = [0u8; 4096]; 10 | lz4::compress(data, &mut buf, level).unwrap(); 11 | } 12 | 13 | fn lz4_decompress(orig_len: usize, data: &[u8]) { 14 | let mut buf = vec![0u8; orig_len]; 15 | lz4::decompress(data, &mut buf).unwrap(); 16 | } 17 | 18 | fn lz4_compress_streaming(n: usize, level: i32, data: &[u8]) { 19 | let mut buf = [0u8; 4096]; 20 | let mut comp = lz4::Compressor::new().unwrap(); 21 | for _ in 0..n { 22 | comp.next(data, &mut buf, level).unwrap(); 23 | } 24 | } 25 | 26 | fn lz4_benchmark(c: &mut Criterion) { 27 | let data = include_bytes!("lorem-ipsum.txt"); 28 | 29 | c.bench_function("lz4::compress (ACC_LEVEL_DEFAULT)", |b| { 30 | b.iter(|| lz4_compress(lz4::ACC_LEVEL_DEFAULT, black_box(data))) 31 | }); 32 | 33 | c.bench_function("lz4::compress (i32::MAX)", |b| { 34 | b.iter(|| lz4_compress(i32::MAX, black_box(data))) 35 | }); 36 | 37 | c.bench_function("lz4::Compressor (ACC_LEVEL_DEFAULT)", |b| { 38 | b.iter(|| lz4_compress_streaming(32, lz4::ACC_LEVEL_DEFAULT, black_box(data))) 39 | }); 40 | 41 | c.bench_function("lz4::Compressor (i32::MAX)", |b| { 42 | b.iter(|| lz4_compress_streaming(32, i32::MAX, black_box(data))) 43 | }); 44 | 45 | let mut compressed = Vec::new(); 46 | lz4::compress_to_vec(data, &mut compressed, lz4::ACC_LEVEL_DEFAULT).unwrap(); 47 | 48 | c.bench_function("lz4::decompress", |b| { 49 | b.iter(|| lz4_decompress(data.len(), black_box(&compressed))) 50 | }); 51 | } 52 | 53 | criterion_group!(lz4_benches, lz4_benchmark); 54 | 55 | fn lz4_hc_compress(level: i32, data: &[u8]) { 56 | let mut buf = [0u8; 4096]; 57 | lz4_hc::compress(data, &mut buf, level).unwrap(); 58 | } 59 | 60 | fn lz4_hc_compress_streaming(n: usize, level: i32, data: &[u8]) { 61 | let mut buf = [0u8; 4096]; 62 | let mut comp = lz4_hc::Compressor::new().unwrap(); 63 | comp.set_compression_level(level); 64 | for _ in 0..n { 65 | comp.next(data, &mut buf).unwrap(); 66 | } 67 | } 68 | 69 | fn lz4_hc_benchmark(c: &mut Criterion) { 70 | let data = include_bytes!("lorem-ipsum.txt"); 71 | 72 | c.bench_function("lz4_hc::compress (CLEVEL_DEFAULT)", |b| { 73 | b.iter(|| lz4_hc_compress(lz4_hc::CLEVEL_DEFAULT, black_box(data))) 74 | }); 75 | 76 | c.bench_function("lz4_hc::compress (CLEVEL_MIN)", |b| { 77 | b.iter(|| lz4_hc_compress(lz4_hc::CLEVEL_MIN, black_box(data))) 78 | }); 79 | 80 | c.bench_function("lz4_hc::compress (CLEVEL_MAX)", |b| { 81 | b.iter(|| lz4_hc_compress(lz4_hc::CLEVEL_MAX, black_box(data))) 82 | }); 83 | 84 | c.bench_function("lz4_hc::Compressor (CLEVEL_DEFAULT)", |b| { 85 | b.iter(|| lz4_hc_compress_streaming(32, lz4_hc::CLEVEL_DEFAULT, black_box(data))) 86 | }); 87 | 88 | c.bench_function("lz4_hc::Compressor (CLEVEL_MIN)", |b| { 89 | b.iter(|| lz4_hc_compress_streaming(32, lz4_hc::CLEVEL_MIN, black_box(data))) 90 | }); 91 | 92 | c.bench_function("lz4_hc::Compressor (CLEVEL_MAX)", |b| { 93 | b.iter(|| lz4_hc_compress_streaming(32, lz4_hc::CLEVEL_MAX, black_box(data))) 94 | }); 95 | } 96 | 97 | criterion_group!(lz4_hc_benches, lz4_hc_benchmark); 98 | 99 | fn lz4f_compress(prefs: &lz4f::Preferences, data: &[u8]) { 100 | let mut buf = [0u8; 4096]; 101 | lz4f::compress(data, &mut buf, prefs).unwrap(); 102 | } 103 | 104 | fn lz4f_decompress(data: &[u8]) { 105 | let mut buf = Vec::new(); 106 | lz4f::decompress_to_vec(data, &mut buf).unwrap(); 107 | } 108 | 109 | fn lz4f_write_compressor(n: usize, prefs: lz4f::Preferences, data: &[u8]) { 110 | let mut buf = Vec::new(); 111 | let mut w = lz4f::WriteCompressor::new(&mut buf, prefs).unwrap(); 112 | for _ in 0..n { 113 | w.write_all(data).unwrap(); 114 | } 115 | } 116 | 117 | fn lz4f_bufread_compressor(n: usize, prefs: lz4f::Preferences, data: &[u8]) { 118 | let mut buf = Vec::new(); 119 | let mut r = lz4f::BufReadCompressor::new(data, prefs).unwrap(); 120 | for _ in 0..n { 121 | r.read_to_end(&mut buf).unwrap(); 122 | } 123 | } 124 | 125 | fn lz4f_benchmark(c: &mut Criterion) { 126 | let data = include_bytes!("lorem-ipsum.txt"); 127 | 128 | c.bench_function("lz4f::compress (Default)", |b| { 129 | let prefs = lz4f::PreferencesBuilder::new().build(); 130 | b.iter(|| lz4f_compress(&prefs, black_box(data))) 131 | }); 132 | 133 | c.bench_function("lz4f::compress (compression_level: i32::MAX)", |b| { 134 | let prefs = lz4f::PreferencesBuilder::new() 135 | .compression_level(i32::MAX) 136 | .build(); 137 | b.iter(|| lz4f_compress(&prefs, black_box(data))) 138 | }); 139 | 140 | c.bench_function("lz4f::compress (compression_level: i32::MIN)", |b| { 141 | let prefs = lz4f::PreferencesBuilder::new() 142 | .compression_level(i32::MIN) 143 | .build(); 144 | b.iter(|| lz4f_compress(&prefs, black_box(data))) 145 | }); 146 | 147 | c.bench_function("lz4f::WriteCompressor (Default)", |b| { 148 | let prefs = lz4f::PreferencesBuilder::new().build(); 149 | b.iter(|| lz4f_write_compressor(32, prefs, black_box(data))) 150 | }); 151 | 152 | c.bench_function("lz4f::BufReadCompressor (Default)", |b| { 153 | let prefs = lz4f::PreferencesBuilder::new().build(); 154 | b.iter(|| lz4f_bufread_compressor(32, prefs, black_box(data))) 155 | }); 156 | 157 | let mut compressed = Vec::new(); 158 | lz4f::compress_to_vec(data, &mut compressed, &Default::default()).unwrap(); 159 | 160 | c.bench_function("lz4f::decompress", |b| { 161 | b.iter(|| lz4f_decompress(black_box(&compressed))) 162 | }); 163 | } 164 | 165 | criterion_group!(lz4f_benches, lz4f_benchmark); 166 | criterion_main!(lz4_benches, lz4_hc_benches, lz4f_benches); 167 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | fn main() -> Result<(), cc::Error> { 2 | let sources = &["lz4.c", "lz4hc.c", "lz4frame.c", "xxhash.c"][..]; 3 | let dir = std::path::Path::new("vendor/liblz4"); 4 | cc::Build::new() 5 | .files(sources.iter().map(|file| dir.join(file))) 6 | .try_compile("lz4") 7 | } 8 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | test: 2 | cargo test --release --all-features 3 | 4 | bench: 5 | cargo bench --all-features 6 | 7 | fmt: 8 | cargo +nightly fmt 9 | 10 | doc: 11 | cargo +nightly doc --all-features 12 | 13 | export RUSTDOCFLAGS := "--cfg docsrs" -------------------------------------------------------------------------------- /lzzzz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/picoHz/lzzzz/884d9057334f3b1984e38d8fbc5c740e7e543ddd/lzzzz.png -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | imports_granularity = "Crate" 2 | format_code_in_doc_comments = true 3 | normalize_doc_attributes = true 4 | normalize_comments = true 5 | format_strings = true 6 | edition = "2021" -------------------------------------------------------------------------------- /src/common/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::binding; 4 | use std::ffi::CStr; 5 | 6 | /// Returns the version number of liblz4. 7 | /// 8 | /// # Example 9 | /// 10 | /// ``` 11 | /// assert_eq!(lzzzz::version_number(), 11000); // 1.9.4 12 | /// ``` 13 | pub fn version_number() -> u32 { 14 | unsafe { binding::LZ4_versionNumber() as u32 } 15 | } 16 | 17 | /// Returns the version string of liblz4. 18 | /// 19 | /// # Example 20 | /// 21 | /// ``` 22 | /// assert_eq!(lzzzz::version_string(), "1.10.0"); 23 | /// ``` 24 | pub fn version_string() -> &'static str { 25 | unsafe { 26 | CStr::from_ptr(binding::LZ4_versionString()) 27 | .to_str() 28 | .unwrap() 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/common/binding.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::{c_char, c_int}; 2 | 3 | #[link(name = "lz4")] 4 | extern "C" { 5 | pub fn LZ4_versionNumber() -> c_int; 6 | pub fn LZ4_versionString() -> *const c_char; 7 | } 8 | -------------------------------------------------------------------------------- /src/common/error.rs: -------------------------------------------------------------------------------- 1 | use std::{convert, error, fmt, io, result}; 2 | 3 | /// A list specifying general categories of LZ4 error. 4 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 5 | #[non_exhaustive] 6 | pub enum ErrorKind { 7 | /// The state initialization failed for some reason. 8 | InitializationFailed, 9 | /// The compression failed for some reason. 10 | CompressionFailed, 11 | /// The decompression failed for some reason. 12 | DecompressionFailed, 13 | /// The frame header had an invalid value. 14 | FrameHeaderInvalid, 15 | /// The decompressor reached unexpected EOF. 16 | CompressedDataIncomplete, 17 | /// Dictionary data was not consistent during the streaming decompression. 18 | DictionaryChangedDuringDecompression, 19 | } 20 | 21 | impl fmt::Display for ErrorKind { 22 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> result::Result<(), fmt::Error> { 23 | ::fmt(self, f) 24 | } 25 | } 26 | 27 | /// The error type for LZ4 operations. 28 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 29 | pub struct Error { 30 | kind: ErrorKind, 31 | } 32 | 33 | impl Error { 34 | pub(crate) const fn new(kind: ErrorKind) -> Self { 35 | Self { kind } 36 | } 37 | 38 | /// Returns the corresponding `ErrorKind` for this error. 39 | pub const fn kind(self) -> ErrorKind { 40 | self.kind 41 | } 42 | } 43 | 44 | impl convert::From for io::Error { 45 | fn from(err: Error) -> Self { 46 | Self::new(io::ErrorKind::Other, err) 47 | } 48 | } 49 | 50 | impl fmt::Display for Error { 51 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> result::Result<(), fmt::Error> { 52 | ::fmt(&self.kind, f) 53 | } 54 | } 55 | 56 | impl error::Error for Error {} 57 | 58 | /// A specialized [`Result`] type for LZ4 operations. 59 | /// 60 | /// [`Result`]: https://doc.rust-lang.org/std/result/enum.Result.html 61 | pub type Result = result::Result; 62 | -------------------------------------------------------------------------------- /src/common/mod.rs: -------------------------------------------------------------------------------- 1 | mod api; 2 | mod binding; 3 | mod error; 4 | 5 | pub use api::{version_number, version_string}; 6 | pub use error::{Error, ErrorKind, Result}; 7 | 8 | pub(crate) const DEFAULT_BUF_SIZE: usize = 8 * 1024; 9 | pub(crate) const DICTIONARY_SIZE: usize = 64 * 1024; 10 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Full-featured liblz4 binding for Rust. 2 | 3 | #![deny(unsafe_code)] 4 | #![deny(clippy::all)] 5 | #![cfg_attr(docsrs, feature(doc_cfg))] 6 | 7 | mod common; 8 | 9 | pub mod lz4; 10 | pub mod lz4_hc; 11 | pub mod lz4f; 12 | 13 | pub use common::*; 14 | -------------------------------------------------------------------------------- /src/lz4/binding.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | mem, 3 | os::raw::{c_char, c_int, c_void}, 4 | }; 5 | 6 | const LZ4_MEMORY_USAGE: usize = 14; 7 | const LZ4_STREAMSIZE_U64: usize = (1 << (LZ4_MEMORY_USAGE - 3)) + 4; 8 | pub const LZ4_STREAMSIZE: usize = LZ4_STREAMSIZE_U64 * mem::size_of::(); 9 | 10 | #[repr(C)] 11 | pub struct LZ4Stream { 12 | _private: [u64; LZ4_STREAMSIZE_U64], 13 | } 14 | 15 | #[repr(C)] 16 | pub struct LZ4DecStream { 17 | _private: [u8; 0], 18 | } 19 | 20 | extern "C" { 21 | pub fn LZ4_compress_fast_extState( 22 | state: *mut c_void, 23 | src: *const c_char, 24 | dst: *mut c_char, 25 | src_size: c_int, 26 | dst_capacity: c_int, 27 | acceleration: c_int, 28 | ) -> c_int; 29 | pub fn LZ4_compress_fast_extState_fastReset( 30 | state: *mut c_void, 31 | src: *const c_char, 32 | dst: *mut c_char, 33 | src_size: c_int, 34 | dst_capacity: c_int, 35 | acceleration: c_int, 36 | ) -> c_int; 37 | pub fn LZ4_compress_destSize( 38 | src: *const c_char, 39 | dst: *mut c_char, 40 | src_size: *mut c_int, 41 | target_dst_size: c_int, 42 | ) -> c_int; 43 | pub fn LZ4_decompress_safe( 44 | src: *const c_char, 45 | dst: *mut c_char, 46 | compressed_size: c_int, 47 | dst_capacity: c_int, 48 | ) -> c_int; 49 | pub fn LZ4_decompress_safe_partial( 50 | src: *const c_char, 51 | dst: *mut c_char, 52 | src_size: c_int, 53 | target_output_size: c_int, 54 | dst_capacity: c_int, 55 | ) -> c_int; 56 | pub fn LZ4_decompress_safe_usingDict( 57 | src: *const c_char, 58 | dst: *mut c_char, 59 | compressed_size: c_int, 60 | dst_capacity: c_int, 61 | dict_start: *const c_char, 62 | dict_size: c_int, 63 | ) -> c_int; 64 | pub fn LZ4_decompress_safe_partial_usingDict( 65 | src: *const c_char, 66 | dst: *mut c_char, 67 | compressed_size: c_int, 68 | target_output_size: c_int, 69 | dst_capacity: c_int, 70 | dict_start: *const c_char, 71 | dict_size: c_int, 72 | ) -> c_int; 73 | pub fn LZ4_createStream() -> *mut LZ4Stream; 74 | pub fn LZ4_freeStream(ptr: *mut LZ4Stream) -> c_int; 75 | pub fn LZ4_initStream(buffer: *mut c_void, size: usize) -> *mut LZ4Stream; 76 | pub fn LZ4_loadDict(ptr: *mut LZ4Stream, dictionary: *const c_char, dict_size: c_int) -> c_int; 77 | pub fn LZ4_loadDictSlow(ptr: *mut LZ4Stream, dictionary: *const c_char, dict_size: c_int) -> c_int; 78 | pub fn LZ4_saveDict( 79 | ptr: *mut LZ4Stream, 80 | safe_buffer: *mut c_char, 81 | max_dict_size: c_int, 82 | ) -> c_int; 83 | pub fn LZ4_compress_fast_continue( 84 | ptr: *mut LZ4Stream, 85 | src: *const c_char, 86 | dst: *mut c_char, 87 | src_size: c_int, 88 | dst_capacity: c_int, 89 | acceleration: c_int, 90 | ) -> c_int; 91 | pub fn LZ4_createStreamDecode() -> *mut LZ4DecStream; 92 | pub fn LZ4_freeStreamDecode(stream: *mut LZ4DecStream) -> c_int; 93 | pub fn LZ4_setStreamDecode( 94 | ptr: *mut LZ4DecStream, 95 | dictionary: *const c_char, 96 | dict_size: c_int, 97 | ) -> c_int; 98 | pub fn LZ4_decompress_safe_continue( 99 | ptr: *mut LZ4DecStream, 100 | src: *const c_char, 101 | dst: *mut c_char, 102 | src_size: c_int, 103 | dst_capacity: c_int, 104 | ) -> c_int; 105 | pub fn LZ4_attach_dictionary( 106 | working_stream: *mut LZ4Stream, 107 | dictionary_stream: *const LZ4Stream, 108 | ); 109 | pub fn LZ4_resetStream_fast(streamPtr: *mut LZ4Stream); 110 | } 111 | -------------------------------------------------------------------------------- /src/lz4/block/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::super::binding; 4 | use crate::{Error, ErrorKind, Result}; 5 | 6 | use std::{ 7 | cell::RefCell, 8 | ops::Deref, 9 | os::raw::{c_char, c_int, c_void}, 10 | }; 11 | 12 | const LZ4_MAX_INPUT_SIZE: usize = 0x7E00_0000; 13 | 14 | pub const fn compress_bound(input_size: usize) -> usize { 15 | (input_size <= LZ4_MAX_INPUT_SIZE) as usize * (input_size + (input_size / 255) + 16) 16 | } 17 | 18 | pub const fn size_of_state() -> usize { 19 | binding::LZ4_STREAMSIZE 20 | } 21 | 22 | pub fn compress_fast_ext_state( 23 | state: &mut [u8], 24 | src: &[u8], 25 | dst: *mut u8, 26 | dst_len: usize, 27 | acceleration: i32, 28 | ) -> usize { 29 | unsafe { 30 | binding::LZ4_compress_fast_extState( 31 | state.as_mut_ptr() as *mut c_void, 32 | src.as_ptr() as *const c_char, 33 | dst as *mut c_char, 34 | src.len() as c_int, 35 | dst_len as c_int, 36 | acceleration as c_int, 37 | ) as usize 38 | } 39 | } 40 | 41 | pub fn compress_fast_ext_state_fast_reset( 42 | state: &mut [u8], 43 | src: &[u8], 44 | dst: *mut u8, 45 | dst_len: usize, 46 | acceleration: i32, 47 | ) -> usize { 48 | unsafe { 49 | binding::LZ4_compress_fast_extState_fastReset( 50 | state.as_mut_ptr() as *mut c_void, 51 | src.as_ptr() as *const c_char, 52 | dst as *mut c_char, 53 | src.len() as c_int, 54 | dst_len as c_int, 55 | acceleration as c_int, 56 | ) as usize 57 | } 58 | } 59 | 60 | pub fn compress_dest_size( 61 | src: &[u8], 62 | dst: &mut [u8], 63 | ) -> Result<(usize, usize)> { 64 | let mut src_size: c_int = src.len() as c_int; 65 | let result = unsafe { 66 | binding::LZ4_compress_destSize( 67 | src.as_ptr() as *const c_char, 68 | dst.as_mut_ptr() as *mut c_char, 69 | &mut src_size as *mut c_int, 70 | dst.len() as c_int, 71 | ) 72 | }; 73 | if result == 0 { 74 | Err(Error::new(ErrorKind::CompressionFailed)) 75 | } else { 76 | Ok((src_size as usize, result as usize)) 77 | } 78 | } 79 | 80 | pub fn decompress_safe(src: &[u8], dst: &mut [u8]) -> Result { 81 | let result = unsafe { 82 | binding::LZ4_decompress_safe( 83 | src.as_ptr() as *const c_char, 84 | dst.as_mut_ptr() as *mut c_char, 85 | src.len() as c_int, 86 | dst.len() as c_int, 87 | ) 88 | }; 89 | if result < 0 { 90 | Err(Error::new(ErrorKind::DecompressionFailed)) 91 | } else { 92 | Ok(result as usize) 93 | } 94 | } 95 | 96 | pub fn decompress_safe_partial(src: &[u8], dst: &mut [u8], original_size: usize) -> Result { 97 | let result = unsafe { 98 | binding::LZ4_decompress_safe_partial( 99 | src.as_ptr() as *const c_char, 100 | dst.as_mut_ptr() as *mut c_char, 101 | src.len() as c_int, 102 | original_size as c_int, 103 | dst.len() as c_int, 104 | ) 105 | }; 106 | if result < 0 { 107 | Err(Error::new(ErrorKind::DecompressionFailed)) 108 | } else { 109 | Ok(result as usize) 110 | } 111 | } 112 | 113 | pub fn decompress_safe_using_dict(src: &[u8], dst: &mut [u8], dict: &[u8]) -> Result { 114 | let result = unsafe { 115 | binding::LZ4_decompress_safe_usingDict( 116 | src.as_ptr() as *const c_char, 117 | dst.as_mut_ptr() as *mut c_char, 118 | src.len() as c_int, 119 | dst.len() as c_int, 120 | dict.as_ptr() as *const c_char, 121 | dict.len() as c_int, 122 | ) 123 | }; 124 | if result < 0 { 125 | Err(Error::new(ErrorKind::DecompressionFailed)) 126 | } else { 127 | Ok(result as usize) 128 | } 129 | } 130 | 131 | pub fn decompress_safe_partial_using_dict( 132 | src: &[u8], 133 | dst: &mut [u8], 134 | original_size: usize, 135 | dict: &[u8], 136 | ) -> Result { 137 | let result = unsafe { 138 | binding::LZ4_decompress_safe_partial_usingDict( 139 | src.as_ptr() as *const c_char, 140 | dst.as_mut_ptr() as *mut c_char, 141 | src.len() as c_int, 142 | original_size as c_int, 143 | dst.len() as c_int, 144 | dict.as_ptr() as *const c_char, 145 | dict.len() as c_int, 146 | ) 147 | }; 148 | if result < 0 { 149 | Err(Error::new(ErrorKind::DecompressionFailed)) 150 | } else { 151 | Ok(result as usize) 152 | } 153 | } 154 | 155 | #[derive(Clone)] 156 | pub struct ExtState(RefCell>); 157 | 158 | impl ExtState { 159 | fn new() -> Self { 160 | let size = size_of_state() + 1; 161 | Self(RefCell::new(vec![0; size].into_boxed_slice())) 162 | } 163 | 164 | pub fn with(f: F) -> R 165 | where 166 | F: FnOnce(&Self, bool) -> R, 167 | { 168 | EXT_STATE.with(|state| { 169 | let reset = { 170 | let mut state = state.borrow_mut(); 171 | let last = state.len() - 1; 172 | if state[last] == 0 { 173 | state[last] = 1; 174 | false 175 | } else { 176 | true 177 | } 178 | }; 179 | 180 | (f)(state, reset) 181 | }) 182 | } 183 | } 184 | 185 | impl Deref for ExtState { 186 | type Target = RefCell>; 187 | 188 | fn deref(&self) -> &Self::Target { 189 | &self.0 190 | } 191 | } 192 | 193 | thread_local!(static EXT_STATE: ExtState = ExtState::new()); 194 | -------------------------------------------------------------------------------- /src/lz4/block/mod.rs: -------------------------------------------------------------------------------- 1 | mod api; 2 | 3 | use crate::{Error, ErrorKind, Result}; 4 | use api::ExtState; 5 | use std::cmp; 6 | 7 | /// Calculates the maximum size of the compressed output. 8 | /// 9 | /// If `original_size` is too large to compress, this returns `0`. 10 | #[must_use] 11 | pub const fn max_compressed_size(original_size: usize) -> usize { 12 | api::compress_bound(original_size) 13 | } 14 | 15 | /// Performs LZ4 block compression. 16 | /// 17 | /// Ensure that the destination slice has enough capacity. 18 | /// If `dst.len()` is smaller than `lz4::max_compressed_size(src.len())`, 19 | /// this function may fail. 20 | /// 21 | /// Returns the number of bytes written into the destination buffer. 22 | /// 23 | /// # Example 24 | /// 25 | /// ``` 26 | /// use lzzzz::lz4; 27 | /// 28 | /// let data = b"The quick brown fox jumps over the lazy dog."; 29 | /// let mut buf = [0u8; 256]; 30 | /// 31 | /// // The slice should have enough capacity. 32 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 33 | /// 34 | /// let len = lz4::compress(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 35 | /// let compressed = &buf[..len]; 36 | /// 37 | /// # let mut buf = [0u8; 256]; 38 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 39 | /// # assert_eq!(&buf[..len], &data[..]); 40 | /// # Ok::<(), std::io::Error>(()) 41 | /// ``` 42 | pub fn compress(src: &[u8], dst: &mut [u8], acc: i32) -> Result { 43 | compress_to_ptr(src, dst.as_mut_ptr(), dst.len(), acc) 44 | } 45 | 46 | fn compress_to_ptr(src: &[u8], dst: *mut u8, dst_len: usize, acc: i32) -> Result { 47 | if src.is_empty() { 48 | return Ok(0); 49 | } 50 | 51 | let acc = cmp::min(acc, 33_554_431); 52 | 53 | let len = ExtState::with(|state, reset| { 54 | let mut state = state.borrow_mut(); 55 | if reset { 56 | api::compress_fast_ext_state_fast_reset(&mut state, src, dst, dst_len, acc) 57 | } else { 58 | api::compress_fast_ext_state(&mut state, src, dst, dst_len, acc) 59 | } 60 | }); 61 | if len > 0 { 62 | Ok(len) 63 | } else { 64 | Err(Error::new(ErrorKind::CompressionFailed)) 65 | } 66 | } 67 | 68 | /// Appends compressed data to `Vec`. 69 | /// 70 | /// Returns the number of bytes appended to the given `Vec`. 71 | /// 72 | /// # Example 73 | /// 74 | /// ``` 75 | /// use lzzzz::lz4; 76 | /// 77 | /// let data = b"The quick brown fox jumps over the lazy dog."; 78 | /// let mut buf = Vec::new(); 79 | /// 80 | /// lz4::compress_to_vec(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 81 | /// # let compressed = &buf; 82 | /// # let mut buf = [0u8; 256]; 83 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 84 | /// # assert_eq!(&buf[..len], &data[..]); 85 | /// # Ok::<(), std::io::Error>(()) 86 | /// ``` 87 | pub fn compress_to_vec(src: &[u8], dst: &mut Vec, acc: i32) -> Result { 88 | let orig_len = dst.len(); 89 | dst.reserve(max_compressed_size(src.len())); 90 | #[allow(unsafe_code)] 91 | unsafe { 92 | let result = compress_to_ptr( 93 | src, 94 | dst.as_mut_ptr().add(orig_len), 95 | dst.capacity() - orig_len, 96 | acc, 97 | ); 98 | dst.set_len(orig_len + result.as_ref().unwrap_or(&0)); 99 | result 100 | } 101 | } 102 | 103 | /// Compress data to fill `dst`. 104 | /// 105 | /// This function either compresses the entire `src` buffer into `dst` if it's 106 | /// large enough, or will fill `dst` with as much data as possible from `src`. 107 | /// 108 | /// Returns a pair `(read, wrote)` giving the number of bytes read from `src` 109 | /// and the number of bytes written to `dst`. 110 | /// 111 | /// # Example 112 | /// 113 | /// ``` 114 | /// use lzzzz::lz4; 115 | /// 116 | /// let data = b"The quick brown fox jumps over the lazy dog."; 117 | /// let mut buf = [0u8; 256]; 118 | /// 119 | /// // This slice should have enough capacity. 120 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 121 | /// 122 | /// let (read, wrote) = lz4::compress_fill(data, &mut buf)?; 123 | /// assert_eq!(read, data.len()); 124 | /// let compressed = &buf[..wrote]; 125 | /// 126 | /// # let mut buf = [0u8; 256]; 127 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 128 | /// # assert_eq!(&buf[..len], &data[..]); 129 | /// 130 | /// // This slice doesn't have enough capacity, but we can fill it. 131 | /// let mut smallbuf = [0u8; 32]; 132 | /// assert!(smallbuf.len() < lz4::max_compressed_size(data.len())); 133 | /// 134 | /// let (read, wrote) = lz4::compress_fill(data, &mut smallbuf)?; 135 | /// assert_eq!(wrote, smallbuf.len()); 136 | /// let remaining_data = &data[read..]; 137 | /// 138 | /// # let mut buf = [0u8; 256]; 139 | /// # let len = lz4::decompress(&smallbuf, &mut buf)?; 140 | /// # assert_eq!(&buf[..len], &data[..read]); 141 | /// # Ok::<(), std::io::Error>(()) 142 | /// ``` 143 | pub fn compress_fill(src: &[u8], dst: &mut [u8]) -> Result<(usize, usize)> { 144 | api::compress_dest_size(src, dst) 145 | } 146 | 147 | /// Decompresses an LZ4 block. 148 | /// 149 | /// The length of the destination slice must be equal to the original data length. 150 | /// 151 | /// Returns the number of bytes written into the destination buffer. 152 | /// 153 | /// # Example 154 | /// 155 | /// ``` 156 | /// use lzzzz::lz4; 157 | /// 158 | /// const ORIGINAL_SIZE: usize = 44; 159 | /// const COMPRESSED_DATA: &str = 160 | /// "8B1UaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGhlIGxhenkgZG9nLg=="; 161 | /// 162 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 163 | /// let mut buf = [0u8; ORIGINAL_SIZE]; 164 | /// 165 | /// lz4::decompress(&data[..], &mut buf[..])?; 166 | /// 167 | /// assert_eq!( 168 | /// &buf[..], 169 | /// &b"The quick brown fox jumps over the lazy dog."[..] 170 | /// ); 171 | /// # Ok::<(), std::io::Error>(()) 172 | /// ``` 173 | pub fn decompress(src: &[u8], dst: &mut [u8]) -> Result { 174 | api::decompress_safe(src, dst) 175 | } 176 | 177 | /// Decompresses an LZ4 block until the destination slice fills up. 178 | /// 179 | /// Returns the number of bytes written into the destination buffer. 180 | /// 181 | /// # Example 182 | /// 183 | /// ``` 184 | /// use lzzzz::lz4; 185 | /// 186 | /// const ORIGINAL_SIZE: usize = 44; 187 | /// const COMPRESSED_DATA: &str = 188 | /// "8B1UaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGhlIGxhenkgZG9nLg=="; 189 | /// 190 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 191 | /// let mut buf = [0u8; 24]; 192 | /// 193 | /// lz4::decompress_partial(&data[..], &mut buf[..], ORIGINAL_SIZE)?; 194 | /// 195 | /// assert_eq!(&buf[..], &b"The quick brown fox jump"[..]); 196 | /// # Ok::<(), std::io::Error>(()) 197 | /// ``` 198 | pub fn decompress_partial(src: &[u8], dst: &mut [u8], original_size: usize) -> Result { 199 | api::decompress_safe_partial(src, dst, original_size) 200 | } 201 | 202 | /// Decompresses an LZ4 block with a dictionary. 203 | /// 204 | /// Returns the number of bytes written into the destination buffer. 205 | /// 206 | /// # Example 207 | /// 208 | /// ``` 209 | /// use lzzzz::lz4; 210 | /// 211 | /// const ORIGINAL_SIZE: usize = 44; 212 | /// const COMPRESSED_DATA: &str = "DywAFFAgZG9nLg=="; 213 | /// const DICT_DATA: &[u8] = b"The quick brown fox jumps over the lazy cat."; 214 | /// 215 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 216 | /// let mut buf = [0u8; ORIGINAL_SIZE]; 217 | /// 218 | /// lz4::decompress_with_dict(&data[..], &mut buf[..], DICT_DATA)?; 219 | /// 220 | /// assert_eq!( 221 | /// &buf[..], 222 | /// &b"The quick brown fox jumps over the lazy dog."[..] 223 | /// ); 224 | /// # Ok::<(), std::io::Error>(()) 225 | /// ``` 226 | pub fn decompress_with_dict(src: &[u8], dst: &mut [u8], dict: &[u8]) -> Result { 227 | api::decompress_safe_using_dict(src, dst, dict) 228 | } 229 | 230 | /// Decompresses an LZ4 block with a dictionary until the destination slice fills up. 231 | /// 232 | /// Returns the number of bytes written into the destination buffer. 233 | /// 234 | /// # Example 235 | /// 236 | /// ``` 237 | /// use lzzzz::lz4; 238 | /// 239 | /// const ORIGINAL_SIZE: usize = 44; 240 | /// const COMPRESSED_DATA: &str = "DywAFFAgZG9nLg=="; 241 | /// const DICT_DATA: &[u8] = b"The quick brown fox jumps over the lazy cat."; 242 | /// 243 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 244 | /// let mut buf = [0u8; 24]; 245 | /// 246 | /// lz4::decompress_partial_with_dict(&data[..], &mut buf[..], ORIGINAL_SIZE, DICT_DATA)?; 247 | /// 248 | /// assert_eq!( 249 | /// &buf[..], 250 | /// &b"The quick brown fox jump"[..] 251 | /// ); 252 | /// # Ok::<(), std::io::Error>(()) 253 | /// ``` 254 | pub fn decompress_partial_with_dict( 255 | src: &[u8], 256 | dst: &mut [u8], 257 | original_size: usize, 258 | dict: &[u8], 259 | ) -> Result { 260 | api::decompress_safe_partial_using_dict(src, dst, original_size, dict) 261 | } 262 | -------------------------------------------------------------------------------- /src/lz4/mod.rs: -------------------------------------------------------------------------------- 1 | //! LZ4 compression and decompression. 2 | //! 3 | //! LZ4: Extremely fast compression algorithm. 4 | //! 5 | //! # Acceleration factor 6 | //! Some functions take the acceleration factor. 7 | //! 8 | //! Larger value increases the processing speed in exchange for the 9 | //! lesser compression ratio. 10 | //! 11 | //! ``` 12 | //! # use lzzzz::lz4; 13 | //! # let data = b"The quick brown fox jumps over the lazy dog."; 14 | //! # let mut buf = Vec::new(); 15 | //! // The default factor is 1 so both have the same meaning. 16 | //! lz4::compress_to_vec(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 17 | //! lz4::compress_to_vec(data, &mut buf, 1)?; 18 | //! 19 | //! // Factors lower than 1 are interpreted as 1 so these are also the same as above. 20 | //! lz4::compress_to_vec(data, &mut buf, 0)?; 21 | //! lz4::compress_to_vec(data, &mut buf, -100)?; 22 | //! 23 | //! // Faster but less effective compression. 24 | //! lz4::compress_to_vec(data, &mut buf, 1000)?; 25 | //! 26 | //! # Ok::<(), std::io::Error>(()) 27 | //! ``` 28 | 29 | mod binding; 30 | mod block; 31 | mod stream; 32 | 33 | pub use block::*; 34 | pub use stream::*; 35 | 36 | /// Predefined acceleration level (1). 37 | pub const ACC_LEVEL_DEFAULT: i32 = 1; 38 | -------------------------------------------------------------------------------- /src/lz4/stream/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::super::{ 4 | binding, 5 | binding::{LZ4DecStream, LZ4Stream}, 6 | }; 7 | use crate::{Error, ErrorKind, Result}; 8 | 9 | use std::{ 10 | mem::{size_of, MaybeUninit}, 11 | os::raw::{c_char, c_int, c_void}, 12 | ptr::NonNull, 13 | ptr::null_mut 14 | }; 15 | 16 | #[allow(clippy::large_enum_variant)] 17 | enum Stream { 18 | Stack(LZ4Stream), 19 | Heap(NonNull), 20 | } 21 | 22 | pub struct CompressionContext { 23 | stream: Stream, 24 | } 25 | 26 | unsafe impl Send for CompressionContext {} 27 | 28 | impl CompressionContext { 29 | pub fn new() -> Result { 30 | let mut stream = MaybeUninit::::uninit(); 31 | unsafe { 32 | let ptr = binding::LZ4_initStream( 33 | stream.as_mut_ptr() as *mut c_void, 34 | size_of::(), 35 | ); 36 | if !ptr.is_null() { 37 | return Ok(Self { 38 | stream: Stream::Stack(stream.assume_init()), 39 | }); 40 | } 41 | NonNull::new(binding::LZ4_createStream()) 42 | } 43 | .ok_or_else(|| Error::new(ErrorKind::InitializationFailed)) 44 | .map(|stream| Self { 45 | stream: Stream::Heap(stream), 46 | }) 47 | } 48 | 49 | fn get_ptr(&mut self) -> *mut LZ4Stream { 50 | match &mut self.stream { 51 | Stream::Stack(stream) => stream as *mut LZ4Stream, 52 | Stream::Heap(ptr) => ptr.as_ptr(), 53 | } 54 | } 55 | 56 | pub fn next(&mut self, src: &[u8], dst: *mut u8, dst_len: usize, acceleration: i32) -> usize { 57 | unsafe { 58 | binding::LZ4_compress_fast_continue( 59 | self.get_ptr(), 60 | src.as_ptr() as *const c_char, 61 | dst as *mut c_char, 62 | src.len() as c_int, 63 | dst_len as c_int, 64 | acceleration as c_int, 65 | ) as usize 66 | } 67 | } 68 | 69 | pub fn load_dict(&mut self, dict: &[u8]) { 70 | unsafe { 71 | binding::LZ4_loadDict( 72 | self.get_ptr(), 73 | dict.as_ptr() as *const c_char, 74 | dict.len() as c_int, 75 | ); 76 | } 77 | } 78 | 79 | pub fn load_dict_slow(&mut self, dict: &[u8]) { 80 | unsafe { 81 | binding::LZ4_loadDictSlow( 82 | self.get_ptr(), 83 | dict.as_ptr() as *const c_char, 84 | dict.len() as c_int, 85 | ); 86 | } 87 | } 88 | 89 | pub fn save_dict(&mut self, dict: &mut [u8]) { 90 | unsafe { 91 | binding::LZ4_saveDict( 92 | self.get_ptr(), 93 | dict.as_ptr() as *mut c_char, 94 | dict.len() as c_int, 95 | ); 96 | } 97 | } 98 | 99 | pub fn attach_dict(&mut self, dict_stream: Option<&mut CompressionContext>) { 100 | unsafe { 101 | if dict_stream.is_none() { 102 | // Note(sewer56): When detaching dictionary, we need to reset the stream state 103 | // This behaviour is consistent with what the LZ4 library itself does internally. 104 | binding::LZ4_resetStream_fast(self.get_ptr()); 105 | } 106 | 107 | let dict_ptr = dict_stream.map(|ctx| ctx.get_ptr()).unwrap_or(null_mut()); 108 | binding::LZ4_attach_dictionary(self.get_ptr(), dict_ptr); 109 | } 110 | } 111 | } 112 | 113 | impl Drop for CompressionContext { 114 | fn drop(&mut self) { 115 | if let Stream::Heap(mut ptr) = self.stream { 116 | unsafe { 117 | binding::LZ4_freeStream(ptr.as_mut()); 118 | } 119 | } 120 | } 121 | } 122 | 123 | pub struct DecompressionContext { 124 | stream: NonNull, 125 | } 126 | 127 | unsafe impl Send for DecompressionContext {} 128 | 129 | impl DecompressionContext { 130 | pub fn new() -> Result { 131 | unsafe { 132 | let ptr = NonNull::new(binding::LZ4_createStreamDecode()); 133 | ptr.ok_or_else(|| Error::new(ErrorKind::InitializationFailed)) 134 | .map(|stream| Self { stream }) 135 | } 136 | } 137 | 138 | pub fn reset(&mut self, dict: &[u8]) -> Result<()> { 139 | let result = unsafe { 140 | binding::LZ4_setStreamDecode( 141 | self.stream.as_ptr(), 142 | dict.as_ptr() as *const c_char, 143 | dict.len() as c_int, 144 | ) 145 | }; 146 | if result == 1 { 147 | Ok(()) 148 | } else { 149 | Err(Error::new(ErrorKind::InitializationFailed)) 150 | } 151 | } 152 | 153 | pub fn decompress(&mut self, src: &[u8], dst: *mut u8, dst_len: usize) -> Result { 154 | let result = unsafe { 155 | binding::LZ4_decompress_safe_continue( 156 | self.stream.as_ptr(), 157 | src.as_ptr() as *const c_char, 158 | dst as *mut c_char, 159 | src.len() as c_int, 160 | dst_len as c_int, 161 | ) 162 | }; 163 | if result < 0 { 164 | Err(Error::new(ErrorKind::DecompressionFailed)) 165 | } else { 166 | Ok(result as usize) 167 | } 168 | } 169 | } 170 | 171 | impl Drop for DecompressionContext { 172 | fn drop(&mut self) { 173 | unsafe { 174 | binding::LZ4_freeStreamDecode(self.stream.as_mut()); 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/lz4/stream/mod.rs: -------------------------------------------------------------------------------- 1 | mod api; 2 | 3 | use crate::{ 4 | common::{DEFAULT_BUF_SIZE, DICTIONARY_SIZE}, 5 | lz4, Error, ErrorKind, Result, 6 | }; 7 | use api::{CompressionContext, DecompressionContext}; 8 | use std::{borrow::Cow, cmp, collections::LinkedList, pin::Pin}; 9 | 10 | /// Streaming LZ4 compressor. 11 | /// 12 | /// # Example 13 | /// 14 | /// ``` 15 | /// use lzzzz::lz4; 16 | /// 17 | /// let data = b"The quick brown fox jumps over the lazy dog."; 18 | /// let mut buf = [0u8; 256]; 19 | /// 20 | /// // The slice should have enough capacity. 21 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 22 | /// 23 | /// let mut comp = lz4::Compressor::new()?; 24 | /// let len = comp.next(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 25 | /// let compressed = &buf[..len]; 26 | /// 27 | /// # let mut buf = [0u8; 256]; 28 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 29 | /// # assert_eq!(&buf[..len], &data[..]); 30 | /// # Ok::<(), std::io::Error>(()) 31 | /// ``` 32 | pub struct Compressor<'a> { 33 | ctx: CompressionContext, 34 | dict: Pin>, 35 | safe_buf: Vec, 36 | } 37 | 38 | impl<'a> Compressor<'a> { 39 | /// Creates a new `Compressor`. 40 | pub fn new() -> Result { 41 | Ok(Self { 42 | ctx: CompressionContext::new()?, 43 | dict: Pin::new(Cow::Borrowed(&[])), 44 | safe_buf: Vec::new(), 45 | }) 46 | } 47 | 48 | /// Creates a new `Compressor` with a dictionary. 49 | pub fn with_dict(dict: D) -> Result 50 | where 51 | D: Into>, 52 | { 53 | let mut comp = Self { 54 | dict: Pin::new(dict.into()), 55 | ..Self::new()? 56 | }; 57 | comp.ctx.load_dict(&comp.dict); 58 | Ok(comp) 59 | } 60 | 61 | /// Creates a new `Compressor` with a dictionary. 62 | /// This variant which consumes more initialization time to better reference the dictionary, 63 | /// resulting in slightly improved compression ratios at expense of time. 64 | pub fn with_dict_slow(dict: D) -> Result 65 | where 66 | D: Into>, 67 | { 68 | let mut comp = Self { 69 | dict: Pin::new(dict.into()), 70 | ..Self::new()? 71 | }; 72 | comp.ctx.load_dict_slow(&comp.dict); 73 | Ok(comp) 74 | } 75 | 76 | /// Performs LZ4 streaming compression. 77 | /// 78 | /// Returns the number of bytes written into the destination buffer. 79 | pub fn next(&mut self, src: &[u8], dst: &mut [u8], acc: i32) -> Result { 80 | self.next_to_ptr(src, dst.as_mut_ptr(), dst.len(), acc) 81 | } 82 | 83 | fn next_to_ptr(&mut self, src: &[u8], dst: *mut u8, dst_len: usize, acc: i32) -> Result { 84 | let is_empty = src.is_empty() && dst_len == 0; 85 | let dst_len = self.ctx.next(src, dst, dst_len, acc); 86 | 87 | self.save_dict(); 88 | 89 | if dst_len > 0 { 90 | Ok(dst_len) 91 | } else if is_empty { 92 | Ok(0) 93 | } else { 94 | Err(Error::new(ErrorKind::CompressionFailed)) 95 | } 96 | } 97 | 98 | /// Appends compressed data to `Vec`. 99 | /// 100 | /// Returns the number of bytes appended to the given `Vec`. 101 | pub fn next_to_vec(&mut self, src: &[u8], dst: &mut Vec, acc: i32) -> Result { 102 | let orig_len = dst.len(); 103 | dst.reserve(lz4::max_compressed_size(src.len())); 104 | #[allow(unsafe_code)] 105 | unsafe { 106 | let result = self.next_to_ptr( 107 | src, 108 | dst.as_mut_ptr().add(orig_len), 109 | dst.capacity() - orig_len, 110 | acc, 111 | ); 112 | dst.set_len(orig_len + result.as_ref().unwrap_or(&0)); 113 | result 114 | } 115 | } 116 | 117 | fn save_dict(&mut self) { 118 | self.safe_buf.resize(DICTIONARY_SIZE, 0); 119 | self.ctx.save_dict(&mut self.safe_buf); 120 | } 121 | 122 | /// Attaches a dictionary stream for efficient dictionary reuse. 123 | /// 124 | /// This allows efficient re-use of a static dictionary multiple times by referencing 125 | /// the dictionary stream in-place rather than copying it. 126 | /// 127 | /// # Arguments 128 | /// 129 | /// * `dict_stream` - The dictionary stream to attach, or None to unset any existing dictionary 130 | /// 131 | /// # Notes 132 | /// 133 | /// - The dictionary stream must have been prepared using `with_dict()` or `with_dict_slow()` 134 | /// - The dictionary will only remain attached through the first compression call 135 | /// - The dictionary stream (and its source buffer) must remain valid through the compression session 136 | /// 137 | /// # Example 138 | /// 139 | /// ``` 140 | /// use lzzzz::lz4; 141 | /// 142 | /// let dict_data = b"some dictionary data"; 143 | /// let data = b"data to compress"; 144 | /// 145 | /// // Create dictionary stream 146 | /// let mut dict_comp = lz4::Compressor::with_dict(dict_data)?; 147 | /// 148 | /// // Create working stream and attach dictionary 149 | /// let mut comp = lz4::Compressor::new()?; 150 | /// comp.attach_dict(Some(&mut dict_comp)); 151 | /// 152 | /// // Compress data using the attached dictionary 153 | /// let mut buf = [0u8; 256]; 154 | /// let len = comp.next(data, &mut buf, lz4::ACC_LEVEL_DEFAULT)?; 155 | /// # Ok::<(), std::io::Error>(()) 156 | /// ``` 157 | pub fn attach_dict(&mut self, dict_stream: Option<&mut Compressor<'a>>) { 158 | if let Some(dict) = dict_stream { 159 | self.ctx.attach_dict(Some(&mut dict.ctx)); 160 | } else { 161 | self.ctx.attach_dict(None); 162 | } 163 | } 164 | } 165 | 166 | /// Streaming LZ4 decompressor. 167 | /// 168 | /// # Example 169 | /// 170 | /// ``` 171 | /// use lzzzz::lz4; 172 | /// 173 | /// const ORIGINAL_SIZE: usize = 44; 174 | /// const COMPRESSED_DATA: &str = 175 | /// "8B1UaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGhlIGxhenkgZG9nLg=="; 176 | /// 177 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 178 | /// 179 | /// let mut decomp = lz4::Decompressor::new()?; 180 | /// let result = decomp.next(&data[..], ORIGINAL_SIZE)?; 181 | /// 182 | /// assert_eq!(result, &b"The quick brown fox jumps over the lazy dog."[..]); 183 | /// # Ok::<(), std::io::Error>(()) 184 | /// ``` 185 | pub struct Decompressor<'a> { 186 | ctx: DecompressionContext, 187 | cache: LinkedList>, 188 | cache_len: usize, 189 | last_len: usize, 190 | dict: Pin>, 191 | } 192 | 193 | impl<'a> Decompressor<'a> { 194 | /// Creates a new `Decompressor`. 195 | pub fn new() -> Result { 196 | Ok(Self { 197 | ctx: DecompressionContext::new()?, 198 | cache: LinkedList::new(), 199 | cache_len: 0, 200 | last_len: 0, 201 | dict: Pin::new(Cow::Borrowed(&[])), 202 | }) 203 | } 204 | 205 | /// Creates a new `Decompressor` with a dictionary. 206 | pub fn with_dict(dict: D) -> Result 207 | where 208 | D: Into>, 209 | { 210 | let mut decomp = Self { 211 | dict: Pin::new(dict.into()), 212 | ..Self::new()? 213 | }; 214 | decomp.ctx.reset(&decomp.dict)?; 215 | Ok(decomp) 216 | } 217 | 218 | /// Decompresses an LZ4 block. 219 | pub fn next(&mut self, src: &[u8], original_size: usize) -> Result<&[u8]> { 220 | if self 221 | .cache 222 | .back() 223 | .map(|v| v.capacity() - v.len()) 224 | .filter(|n| *n >= original_size) 225 | .is_none() 226 | { 227 | self.cache.push_back(Vec::with_capacity(cmp::max( 228 | original_size, 229 | DEFAULT_BUF_SIZE, 230 | ))); 231 | } 232 | 233 | let back = self.cache.back_mut().unwrap(); 234 | let orig_len = back.len(); 235 | 236 | #[allow(unsafe_code)] 237 | unsafe { 238 | let dst_len = self.ctx.decompress( 239 | src, 240 | back.as_mut_ptr().add(orig_len), 241 | back.capacity() - orig_len, 242 | )?; 243 | back.set_len(orig_len + dst_len); 244 | self.cache_len += dst_len; 245 | } 246 | self.last_len = original_size; 247 | 248 | while let Some(len) = self 249 | .cache 250 | .front() 251 | .map(Vec::len) 252 | .filter(|n| self.cache_len - n >= DICTIONARY_SIZE) 253 | { 254 | self.cache.pop_front(); 255 | self.cache_len -= len; 256 | } 257 | Ok(self.data()) 258 | } 259 | 260 | fn data(&self) -> &[u8] { 261 | if let Some(back) = self.cache.back() { 262 | let offset = back.len() - self.last_len; 263 | &back[offset..] 264 | } else { 265 | &[] 266 | } 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/lz4_hc/binding.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::{c_char, c_int, c_void}; 2 | 3 | const LZ4HC_HASH_LOG: usize = 15; 4 | const LZ4HC_HASHTABLESIZE: usize = 1 << LZ4HC_HASH_LOG; 5 | const LZ4HC_DICTIONARY_LOGSIZE: usize = 16; 6 | const LZ4HC_MAXD: usize = 1 << LZ4HC_DICTIONARY_LOGSIZE; 7 | pub const LZ4_STREAMHCSIZE: usize = 4 * LZ4HC_HASHTABLESIZE + 2 * LZ4HC_MAXD + 56; 8 | 9 | #[repr(C)] 10 | pub struct LZ4StreamHC { 11 | _private: [u8; 0], 12 | } 13 | 14 | extern "C" { 15 | pub fn LZ4_compress_HC_extStateHC( 16 | state: *mut c_void, 17 | src: *const c_char, 18 | dst: *mut c_char, 19 | src_size: c_int, 20 | dst_capacity: c_int, 21 | compression_level: c_int, 22 | ) -> c_int; 23 | pub fn LZ4_compress_HC_extStateHC_fastReset( 24 | state: *mut c_void, 25 | src: *const c_char, 26 | dst: *mut c_char, 27 | src_size: c_int, 28 | dst_capacity: c_int, 29 | compression_level: c_int, 30 | ) -> c_int; 31 | pub fn LZ4_compress_HC_destSize( 32 | state: *mut c_void, 33 | src: *const c_char, 34 | dst: *mut c_char, 35 | src_size_ptr: *mut c_int, 36 | target_dst_dize: c_int, 37 | compression_level: c_int, 38 | ) -> c_int; 39 | 40 | pub fn LZ4_createStreamHC() -> *mut LZ4StreamHC; 41 | pub fn LZ4_freeStreamHC(ptr: *mut LZ4StreamHC) -> c_int; 42 | pub fn LZ4_loadDictHC( 43 | ptr: *mut LZ4StreamHC, 44 | dictionary: *const c_char, 45 | dict_size: c_int, 46 | ) -> c_int; 47 | pub fn LZ4_saveDictHC( 48 | ptr: *mut LZ4StreamHC, 49 | safe_buffer: *mut c_char, 50 | max_dict_size: c_int, 51 | ) -> c_int; 52 | pub fn LZ4_compress_HC_continue( 53 | ptr: *mut LZ4StreamHC, 54 | src: *const c_char, 55 | dst: *mut c_char, 56 | src_size: c_int, 57 | dst_capacity: c_int, 58 | ) -> c_int; 59 | pub fn LZ4_compress_HC_continue_destSize( 60 | ptr: *mut LZ4StreamHC, 61 | src: *const c_char, 62 | dst: *mut c_char, 63 | src_size_ptr: *mut c_int, 64 | target_dst_size: c_int, 65 | ) -> c_int; 66 | pub fn LZ4_setCompressionLevel(ptr: *mut LZ4StreamHC, compression_level: c_int); 67 | pub fn LZ4_favorDecompressionSpeed(ptr: *mut LZ4StreamHC, favor: c_int); 68 | pub fn LZ4_attach_HC_dictionary( 69 | working_stream: *mut LZ4StreamHC, 70 | dictionary_stream: *const LZ4StreamHC 71 | ); 72 | 73 | pub fn LZ4_resetStreamHC_fast( 74 | streamPtr: *mut LZ4StreamHC, 75 | compressionLevel: c_int 76 | ); 77 | } 78 | -------------------------------------------------------------------------------- /src/lz4_hc/block/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::super::binding; 4 | 5 | use std::{ 6 | cell::RefCell, 7 | ops::Deref, 8 | os::raw::{c_char, c_int, c_void}, 9 | }; 10 | 11 | pub const fn size_of_state() -> usize { 12 | binding::LZ4_STREAMHCSIZE 13 | } 14 | 15 | pub fn compress_ext_state( 16 | state: &mut [u8], 17 | src: &[u8], 18 | dst: *mut u8, 19 | dst_len: usize, 20 | compression_level: i32, 21 | ) -> usize { 22 | unsafe { 23 | binding::LZ4_compress_HC_extStateHC( 24 | state.as_mut_ptr() as *mut c_void, 25 | src.as_ptr() as *const c_char, 26 | dst as *mut c_char, 27 | src.len() as c_int, 28 | dst_len as c_int, 29 | compression_level as c_int, 30 | ) as usize 31 | } 32 | } 33 | 34 | pub fn compress_ext_state_fast_reset( 35 | state: &mut [u8], 36 | src: &[u8], 37 | dst: *mut u8, 38 | dst_len: usize, 39 | compression_level: i32, 40 | ) -> usize { 41 | unsafe { 42 | binding::LZ4_compress_HC_extStateHC_fastReset( 43 | state.as_mut_ptr() as *mut c_void, 44 | src.as_ptr() as *const c_char, 45 | dst as *mut c_char, 46 | src.len() as c_int, 47 | dst_len as c_int, 48 | compression_level as c_int, 49 | ) as usize 50 | } 51 | } 52 | 53 | pub fn compress_dest_size( 54 | state: &mut [u8], 55 | src: &[u8], 56 | dst: &mut [u8], 57 | compression_level: i32, 58 | ) -> (usize, usize) { 59 | let mut src_len = src.len() as i32; 60 | let dst_len = unsafe { 61 | binding::LZ4_compress_HC_destSize( 62 | state.as_mut_ptr() as *mut c_void, 63 | src.as_ptr() as *const c_char, 64 | dst.as_mut_ptr() as *mut c_char, 65 | &mut src_len as *mut c_int, 66 | dst.len() as c_int, 67 | compression_level as c_int, 68 | ) as usize 69 | }; 70 | (src_len as usize, dst_len) 71 | } 72 | 73 | #[derive(Clone)] 74 | pub struct ExtState(RefCell>); 75 | 76 | impl ExtState { 77 | fn new() -> Self { 78 | let size = size_of_state() + 1; 79 | Self(RefCell::new(vec![0; size].into_boxed_slice())) 80 | } 81 | 82 | pub fn with(f: F) -> R 83 | where 84 | F: FnOnce(&Self, bool) -> R, 85 | { 86 | EXT_STATE.with(|state| { 87 | let reset = { 88 | let mut state = state.borrow_mut(); 89 | let last = state.len() - 1; 90 | if state[last] == 0 { 91 | state[last] = 1; 92 | false 93 | } else { 94 | true 95 | } 96 | }; 97 | 98 | (f)(state, reset) 99 | }) 100 | } 101 | } 102 | 103 | impl Deref for ExtState { 104 | type Target = RefCell>; 105 | 106 | fn deref(&self) -> &Self::Target { 107 | &self.0 108 | } 109 | } 110 | 111 | thread_local!(static EXT_STATE: ExtState = ExtState::new()); 112 | -------------------------------------------------------------------------------- /src/lz4_hc/block/mod.rs: -------------------------------------------------------------------------------- 1 | mod api; 2 | 3 | use crate::{lz4, Error, ErrorKind, Result}; 4 | use api::ExtState; 5 | use std::{cmp, io::Cursor}; 6 | 7 | /// Performs LZ4_HC block compression. 8 | /// 9 | /// Ensure that the destination slice has enough capacity. 10 | /// If `dst.len()` is smaller than `lz4::max_compressed_size(src.len())`, 11 | /// this function may fail. 12 | /// 13 | /// Returns the number of bytes written into the destination buffer. 14 | /// 15 | /// # Example 16 | /// 17 | /// ``` 18 | /// use lzzzz::{lz4, lz4_hc}; 19 | /// 20 | /// let data = b"The quick brown fox jumps over the lazy dog."; 21 | /// let mut buf = [0u8; 256]; 22 | /// 23 | /// // The slice should have enough capacity. 24 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 25 | /// 26 | /// let len = lz4_hc::compress(data, &mut buf, lz4_hc::CLEVEL_DEFAULT)?; 27 | /// let compressed = &buf[..len]; 28 | /// 29 | /// # let mut buf = [0u8; 256]; 30 | /// # let len = lz4::decompress( 31 | /// # compressed, 32 | /// # &mut buf[..data.len()], 33 | /// # )?; 34 | /// # assert_eq!(&buf[..len], &data[..]); 35 | /// # Ok::<(), std::io::Error>(()) 36 | /// ``` 37 | pub fn compress(src: &[u8], dst: &mut [u8], level: i32) -> Result { 38 | compress_to_ptr(src, dst.as_mut_ptr(), dst.len(), level) 39 | } 40 | 41 | fn compress_to_ptr(src: &[u8], dst: *mut u8, dst_len: usize, level: i32) -> Result { 42 | if src.is_empty() { 43 | return Ok(0); 44 | } 45 | let len = ExtState::with(|state, reset| { 46 | if reset { 47 | api::compress_ext_state_fast_reset(&mut state.borrow_mut(), src, dst, dst_len, level) 48 | } else { 49 | api::compress_ext_state(&mut state.borrow_mut(), src, dst, dst_len, level) 50 | } 51 | }); 52 | if len > 0 { 53 | Ok(len) 54 | } else { 55 | Err(Error::new(ErrorKind::CompressionFailed)) 56 | } 57 | } 58 | 59 | /// Compress data to fill `dst`. 60 | /// 61 | /// This function either compresses the entire `src` buffer into `dst` if it's 62 | /// large enough, or will fill `dst` with as much data as possible from `src`. 63 | /// 64 | /// Returns a pair `(read, wrote)` giving the number of bytes read from `src` 65 | /// and the number of bytes written to `dst`. 66 | /// 67 | /// # Example 68 | /// 69 | /// ``` 70 | /// use lzzzz::{lz4, lz4_hc}; 71 | /// 72 | /// let data = b"The quick brown fox jumps over the lazy dog."; 73 | /// let mut buf = [0u8; 256]; 74 | /// 75 | /// // This slice should have enough capacity. 76 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 77 | /// 78 | /// let (read, wrote) = lz4_hc::compress_fill(data, &mut buf, lz4_hc::CLEVEL_DEFAULT)?; 79 | /// assert_eq!(read, data.len()); 80 | /// let compressed = &buf[..wrote]; 81 | /// 82 | /// # let mut buf = [0u8; 256]; 83 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 84 | /// # assert_eq!(&buf[..len], &data[..]); 85 | /// 86 | /// // This slice doesn't have enough capacity, but we can fill it. 87 | /// let mut smallbuf = [0u8; 32]; 88 | /// assert!(smallbuf.len() < lz4::max_compressed_size(data.len())); 89 | /// 90 | /// let (read, wrote) = lz4_hc::compress_fill(data, &mut smallbuf, lz4_hc::CLEVEL_DEFAULT)?; 91 | /// assert_eq!(wrote, smallbuf.len()); 92 | /// let remaining_data = &data[read..]; 93 | /// 94 | /// # let mut buf = [0u8; 256]; 95 | /// # let len = lz4::decompress(&smallbuf, &mut buf)?; 96 | /// # assert_eq!(&buf[..len], &data[..read]); 97 | /// # Ok::<(), std::io::Error>(()) 98 | /// ``` 99 | pub fn compress_fill(src: &[u8], dst: &mut [u8], level: i32) -> Result<(usize, usize)> { 100 | if src.is_empty() { 101 | return Ok((0, 0)); 102 | } 103 | let (read, wrote) = ExtState::with(|state, _reset| { 104 | api::compress_dest_size(&mut state.borrow_mut(), src, dst, level) 105 | }); 106 | if wrote > 0 { 107 | Ok((read, wrote)) 108 | } else { 109 | Err(Error::new(ErrorKind::CompressionFailed)) 110 | } 111 | } 112 | 113 | /// Compresses data until the destination slice fills up. 114 | /// 115 | /// Returns the number of bytes written into the destination buffer. 116 | /// 117 | /// # Example 118 | /// 119 | /// ``` 120 | /// use lzzzz::{lz4, lz4_hc}; 121 | /// use std::io::Cursor; 122 | /// 123 | /// let data = b"The quick brown fox jumps over the lazy dog."; 124 | /// let mut buf = [0u8; 16]; 125 | /// 126 | /// let mut src = Cursor::new(&data[..]); 127 | /// let len = lz4_hc::compress_partial(&mut src, &mut buf, lz4_hc::CLEVEL_DEFAULT)?; 128 | /// let compressed = &buf[..len]; 129 | /// 130 | /// # let mut buf = [0u8; 256]; 131 | /// # let len = lz4::decompress( 132 | /// # compressed, 133 | /// # &mut buf[..data.len()], 134 | /// # )?; 135 | /// # assert_eq!(&buf[..len], &data[..src.position() as usize]); 136 | /// # Ok::<(), std::io::Error>(()) 137 | /// ``` 138 | #[deprecated(since = "1.1.0", note = "Use compress_fill instead.")] 139 | pub fn compress_partial(src: &mut Cursor, dst: &mut [u8], level: i32) -> Result 140 | where 141 | T: AsRef<[u8]>, 142 | { 143 | let src_ref = src.get_ref().as_ref(); 144 | let pos = cmp::min(src_ref.len(), src.position() as usize); 145 | let src_ref = &src_ref[pos..]; 146 | if src_ref.is_empty() || dst.is_empty() { 147 | return Ok(0); 148 | } 149 | let (src_len, dst_len) = ExtState::with(|state, _| { 150 | api::compress_dest_size(&mut state.borrow_mut(), src_ref, dst, level) 151 | }); 152 | src.set_position(src.position() + src_len as u64); 153 | Ok(dst_len) 154 | } 155 | 156 | /// Appends compressed data to `Vec`. 157 | /// 158 | /// Returns the number of bytes appended to the given `Vec`. 159 | /// 160 | /// # Example 161 | /// 162 | /// ``` 163 | /// use lzzzz::{lz4, lz4_hc}; 164 | /// 165 | /// let data = b"The quick brown fox jumps over the lazy dog."; 166 | /// let mut buf = Vec::new(); 167 | /// 168 | /// lz4_hc::compress_to_vec(data, &mut buf, lz4_hc::CLEVEL_DEFAULT)?; 169 | /// 170 | /// # let compressed = &buf; 171 | /// # let mut buf = [0u8; 256]; 172 | /// # let len = lz4::decompress( 173 | /// # compressed, 174 | /// # &mut buf[..data.len()], 175 | /// # )?; 176 | /// # assert_eq!(&buf[..len], &data[..]); 177 | /// # Ok::<(), std::io::Error>(()) 178 | /// ``` 179 | pub fn compress_to_vec(src: &[u8], dst: &mut Vec, level: i32) -> Result { 180 | let orig_len = dst.len(); 181 | dst.reserve(lz4::max_compressed_size(src.len())); 182 | #[allow(unsafe_code)] 183 | unsafe { 184 | let result = compress_to_ptr( 185 | src, 186 | dst.as_mut_ptr().add(orig_len), 187 | dst.capacity() - orig_len, 188 | level, 189 | ); 190 | dst.set_len(orig_len + result.as_ref().unwrap_or(&0)); 191 | result 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/lz4_hc/mod.rs: -------------------------------------------------------------------------------- 1 | //! LZ4_HC compression. 2 | //! 3 | //! LZ4_HC: High compression variant of LZ4. 4 | //! 5 | //! # Decompression 6 | //! The `lz4_hc` module doesn't provide decompression functionality. 7 | //! Use the [`lz4`] module instead. 8 | //! 9 | //! [`lz4`]: ../lz4/index.html 10 | 11 | mod binding; 12 | mod block; 13 | mod stream; 14 | 15 | pub use block::*; 16 | pub use stream::*; 17 | 18 | /// Predefined compression level (3). 19 | pub const CLEVEL_MIN: i32 = 3; 20 | 21 | /// Predefined compression level (9). 22 | pub const CLEVEL_DEFAULT: i32 = 9; 23 | 24 | /// Predefined compression level (10). 25 | pub const CLEVEL_OPT_MIN: i32 = 10; 26 | 27 | /// Predefined compression level (12). 28 | pub const CLEVEL_MAX: i32 = 12; 29 | 30 | /// Decompression speed mode flag. 31 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 32 | pub enum FavorDecSpeed { 33 | Disabled, 34 | Enabled, 35 | } 36 | 37 | impl Default for FavorDecSpeed { 38 | fn default() -> Self { 39 | Self::Disabled 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/lz4_hc/stream/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::super::{binding, binding::LZ4StreamHC}; 4 | use crate::{Error, ErrorKind, Result}; 5 | 6 | use std::{ 7 | os::raw::{c_char, c_int}, 8 | ptr::NonNull, 9 | ptr::null_mut 10 | }; 11 | 12 | pub struct CompressionContext { 13 | stream: NonNull, 14 | } 15 | 16 | unsafe impl Send for CompressionContext {} 17 | 18 | impl CompressionContext { 19 | pub fn new() -> Result { 20 | let ptr = unsafe { NonNull::new(binding::LZ4_createStreamHC()) }; 21 | ptr.ok_or_else(|| Error::new(ErrorKind::InitializationFailed)) 22 | .map(|stream| Self { stream }) 23 | } 24 | 25 | pub fn set_compression_level(&mut self, compression_level: i32) { 26 | unsafe { 27 | binding::LZ4_setCompressionLevel(self.stream.as_ptr(), compression_level as c_int) 28 | } 29 | } 30 | 31 | pub fn set_favor_dec_speed(&mut self, flag: bool) { 32 | unsafe { binding::LZ4_favorDecompressionSpeed(self.stream.as_ptr(), i32::from(flag)) } 33 | } 34 | 35 | pub fn load_dict(&mut self, dict: &[u8]) { 36 | unsafe { 37 | binding::LZ4_loadDictHC( 38 | self.stream.as_ptr(), 39 | dict.as_ptr() as *const c_char, 40 | dict.len() as c_int, 41 | ); 42 | } 43 | } 44 | 45 | pub fn save_dict(&mut self, dict: &mut [u8]) { 46 | unsafe { 47 | binding::LZ4_saveDictHC( 48 | self.stream.as_ptr(), 49 | dict.as_ptr() as *mut c_char, 50 | dict.len() as c_int, 51 | ); 52 | } 53 | } 54 | 55 | pub fn next(&mut self, src: &[u8], dst: *mut u8, dst_len: usize) -> Result { 56 | if src.is_empty() { 57 | return Ok(0); 58 | } 59 | let dst_len = unsafe { 60 | binding::LZ4_compress_HC_continue( 61 | self.stream.as_ptr(), 62 | src.as_ptr() as *const c_char, 63 | dst as *mut c_char, 64 | src.len() as c_int, 65 | dst_len as c_int, 66 | ) as usize 67 | }; 68 | if dst_len > 0 { 69 | Ok(dst_len) 70 | } else { 71 | Err(Error::new(ErrorKind::CompressionFailed)) 72 | } 73 | } 74 | 75 | pub fn next_partial(&mut self, src: &[u8], dst: &mut [u8]) -> Result<(usize, usize)> { 76 | if src.is_empty() || dst.is_empty() { 77 | return Ok((0, 0)); 78 | } 79 | let mut src_len = src.len() as c_int; 80 | let dst_len = unsafe { 81 | binding::LZ4_compress_HC_continue_destSize( 82 | self.stream.as_ptr(), 83 | src.as_ptr() as *const c_char, 84 | dst.as_mut_ptr() as *mut c_char, 85 | &mut src_len as *mut c_int, 86 | dst.len() as c_int, 87 | ) as usize 88 | }; 89 | if dst_len > 0 { 90 | Ok((src_len as usize, dst_len)) 91 | } else { 92 | Err(Error::new(ErrorKind::CompressionFailed)) 93 | } 94 | } 95 | 96 | pub fn attach_dict(&mut self, dict_stream: Option<&CompressionContext>, compression_level: i32) { 97 | unsafe { 98 | if dict_stream.is_none() { 99 | // Note(sewer56): When detaching dictionary, we need to reset the stream state 100 | // This behaviour is consistent with what the LZ4 library itself does internally. 101 | // The LZ4HC API does not have a way to retrieve compression level, so we must pass it manually, 102 | // since the HC API differs here. 103 | binding::LZ4_resetStreamHC_fast(self.stream.as_ptr(), compression_level); 104 | } 105 | let dict_ptr = dict_stream.map(|ctx| ctx.stream.as_ptr()).unwrap_or(null_mut()); 106 | binding::LZ4_attach_HC_dictionary(self.stream.as_ptr(), dict_ptr); 107 | } 108 | } 109 | } 110 | 111 | impl Drop for CompressionContext { 112 | fn drop(&mut self) { 113 | unsafe { 114 | binding::LZ4_freeStreamHC(self.stream.as_mut()); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/lz4_hc/stream/mod.rs: -------------------------------------------------------------------------------- 1 | mod api; 2 | 3 | use crate::{common::DICTIONARY_SIZE, lz4, lz4_hc::FavorDecSpeed, Result}; 4 | use api::CompressionContext; 5 | use std::{borrow::Cow, cmp, io::Cursor, pin::Pin}; 6 | 7 | /// Streaming LZ4_HC compressor. 8 | /// 9 | /// # Example 10 | /// 11 | /// ``` 12 | /// use lzzzz::{lz4, lz4_hc}; 13 | /// 14 | /// let data = b"The quick brown fox jumps over the lazy dog."; 15 | /// let mut buf = [0u8; 256]; 16 | /// 17 | /// // The slice should have enough capacity. 18 | /// assert!(buf.len() >= lz4::max_compressed_size(data.len())); 19 | /// 20 | /// let mut comp = lz4_hc::Compressor::new()?; 21 | /// let len = comp.next(data, &mut buf)?; 22 | /// let compressed = &buf[..len]; 23 | /// 24 | /// # let mut buf = [0u8; 256]; 25 | /// # let len = lz4::decompress(compressed, &mut buf[..data.len()])?; 26 | /// # assert_eq!(&buf[..len], &data[..]); 27 | /// # Ok::<(), std::io::Error>(()) 28 | /// ``` 29 | pub struct Compressor<'a> { 30 | ctx: CompressionContext, 31 | dict: Pin>, 32 | safe_buf: Vec, 33 | } 34 | 35 | impl<'a> Compressor<'a> { 36 | /// Creates a new `Compressor`. 37 | pub fn new() -> Result { 38 | Ok(Self { 39 | ctx: CompressionContext::new()?, 40 | dict: Pin::new(Cow::Borrowed(&[])), 41 | safe_buf: Vec::new(), 42 | }) 43 | } 44 | 45 | /// Creates a new `Compressor` with a dictionary. 46 | pub fn with_dict(dict: D, compression_level: i32) -> Result 47 | where 48 | D: Into>, 49 | { 50 | // Note(sewer56). 51 | // The LZ4 documentation states the following: 52 | // - In order for LZ4_loadDictHC() to create the correct data structure, 53 | // it is essential to set the compression level _before_ loading the dictionary. 54 | // Therefore this API requires a `compression_level`. 55 | 56 | let mut comp = Self { 57 | dict: Pin::new(dict.into()), 58 | ..Self::new()? 59 | }; 60 | 61 | comp.ctx.set_compression_level(compression_level); 62 | comp.ctx.load_dict(&comp.dict); 63 | Ok(comp) 64 | } 65 | 66 | /// Sets the compression level. 67 | pub fn set_compression_level(&mut self, level: i32) { 68 | self.ctx.set_compression_level(level); 69 | } 70 | 71 | /// Sets the decompression speed mode flag. 72 | pub fn set_favor_dec_speed(&mut self, dec_speed: FavorDecSpeed) { 73 | self.ctx 74 | .set_favor_dec_speed(dec_speed == FavorDecSpeed::Enabled); 75 | } 76 | 77 | /// Performs LZ4_HC streaming compression. 78 | /// 79 | /// Returns the number of bytes written into the destination buffer. 80 | pub fn next(&mut self, src: &[u8], dst: &mut [u8]) -> Result { 81 | self.next_to_ptr(src, dst.as_mut_ptr(), dst.len()) 82 | } 83 | 84 | fn next_to_ptr(&mut self, src: &[u8], dst: *mut u8, dst_len: usize) -> Result { 85 | let result = self.ctx.next(src, dst, dst_len)?; 86 | self.save_dict(); 87 | Ok(result) 88 | } 89 | 90 | /// Performs LZ4_HC streaming compression to fill `dst`. 91 | /// 92 | /// This function either compresses the entire `src` buffer into `dst` if it's 93 | /// large enough, or will fill `dst` with as much data as possible from `src`. 94 | /// 95 | /// Returns a pair `(read, wrote)` giving the number of bytes read from `src` 96 | /// and the number of bytes written to `dst`. 97 | /// 98 | /// # Example 99 | /// 100 | /// ``` 101 | /// use lzzzz::{lz4, lz4_hc}; 102 | /// 103 | /// let data = b"The quick brown fox jumps over the lazy dog."; 104 | /// 105 | /// let mut smallbuf = [0u8; 32]; 106 | /// assert!(smallbuf.len() < lz4::max_compressed_size(data.len())); 107 | /// 108 | /// let mut comp = lz4_hc::Compressor::new()?; 109 | /// let (read, wrote) = comp.next_fill(data, &mut smallbuf)?; 110 | /// let remaining_data = &data[read..]; 111 | /// 112 | /// # let mut buf = [0u8; 256]; 113 | /// # let len = lz4::decompress(&smallbuf, &mut buf)?; 114 | /// # assert_eq!(&buf[..len], &data[..read]); 115 | /// # Ok::<(), std::io::Error>(()) 116 | /// ``` 117 | pub fn next_fill(&mut self, src: &[u8], dst: &mut [u8]) -> Result<(usize, usize)> { 118 | let (src_len, dst_len) = self.ctx.next_partial(src, dst)?; 119 | self.save_dict(); 120 | Ok((src_len, dst_len)) 121 | } 122 | 123 | /// Compresses data until the destination slice fills up. 124 | /// 125 | /// Returns the number of bytes written into the destination buffer. 126 | #[deprecated(since = "1.1.0", note = "Use next_fill instead.")] 127 | pub fn next_partial(&mut self, src: &mut Cursor, dst: &mut [u8]) -> Result 128 | where 129 | T: AsRef<[u8]>, 130 | { 131 | let src_ref = src.get_ref().as_ref(); 132 | let pos = cmp::min(src_ref.len(), src.position() as usize); 133 | let src_ref = &src_ref[pos..]; 134 | let (src_len, dst_len) = self.ctx.next_partial(src_ref, dst)?; 135 | src.set_position(src.position() + src_len as u64); 136 | self.save_dict(); 137 | Ok(dst_len) 138 | } 139 | 140 | /// Appends a compressed frame to `Vec`. 141 | /// 142 | /// Returns the number of bytes appended to the given `Vec`. 143 | pub fn next_to_vec(&mut self, src: &[u8], dst: &mut Vec) -> Result { 144 | let orig_len = dst.len(); 145 | dst.reserve(lz4::max_compressed_size(src.len())); 146 | #[allow(unsafe_code)] 147 | unsafe { 148 | let result = self.next_to_ptr( 149 | src, 150 | dst.as_mut_ptr().add(orig_len), 151 | dst.capacity() - orig_len, 152 | ); 153 | dst.set_len(orig_len + result.as_ref().unwrap_or(&0)); 154 | result 155 | } 156 | } 157 | 158 | fn save_dict(&mut self) { 159 | self.safe_buf.resize(DICTIONARY_SIZE, 0); 160 | self.ctx.save_dict(&mut self.safe_buf); 161 | } 162 | 163 | /// Attaches a dictionary stream for efficient dictionary reuse. 164 | /// 165 | /// This allows efficient re-use of a static dictionary multiple times by referencing 166 | /// the dictionary stream in-place rather than copying it. 167 | /// 168 | /// # Arguments 169 | /// 170 | /// * `dict_stream` - The dictionary stream to attach, or None to unset any existing dictionary 171 | /// * `compression_level` - The compression level to use (CLEVEL_MIN to CLEVEL_MAX) 172 | /// 173 | /// # Notes 174 | /// 175 | /// - The dictionary stream must have been prepared using `with_dict()` 176 | /// - The dictionary will only remain attached through the first compression call 177 | /// - The dictionary stream (and its source buffer) must remain valid through the compression session 178 | /// 179 | /// # Example 180 | /// 181 | /// ``` 182 | /// use lzzzz::lz4_hc; 183 | /// 184 | /// let dict_data = b"dictionary data"; 185 | /// let data = b"data to compress"; 186 | /// 187 | /// // Create dictionary stream 188 | /// let dict_comp = lz4_hc::Compressor::with_dict(dict_data, lz4_hc::CLEVEL_DEFAULT)?; 189 | /// 190 | /// // Create working stream and attach dictionary 191 | /// let mut comp = lz4_hc::Compressor::new()?; 192 | /// comp.attach_dict(Some(&dict_comp), lz4_hc::CLEVEL_DEFAULT); 193 | /// 194 | /// // Compress data using the attached dictionary 195 | /// let mut buf = [0u8; 256]; 196 | /// let len = comp.next(data, &mut buf)?; 197 | /// # Ok::<(), std::io::Error>(()) 198 | /// ``` 199 | pub fn attach_dict(&mut self, dict_stream: Option<&Compressor<'a>>, compression_level: i32) { 200 | if let Some(dict) = dict_stream { 201 | self.ctx.attach_dict(Some(&dict.ctx), compression_level); 202 | } else { 203 | self.ctx.attach_dict(None, compression_level); 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/lz4f/api.rs: -------------------------------------------------------------------------------- 1 | #![allow(unsafe_code)] 2 | 3 | use super::{ 4 | binding, 5 | binding::{ 6 | LZ4FCompressionCtx, LZ4FCompressionDict, LZ4FCompressionOptions, LZ4FDecompressionCtx, 7 | LZ4FDecompressionOptions, 8 | }, 9 | Dictionary, 10 | }; 11 | use crate::lz4f::{Error, ErrorKind, FrameInfo, Preferences, Result}; 12 | 13 | use std::{mem::MaybeUninit, os::raw::c_void, ptr::NonNull}; 14 | 15 | pub const LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH: usize = 5; 16 | pub const LZ4F_HEADER_SIZE_MAX: usize = 19; 17 | 18 | pub struct CompressionContext { 19 | ctx: NonNull, 20 | dict: Option, 21 | } 22 | 23 | unsafe impl Send for CompressionContext {} 24 | 25 | impl CompressionContext { 26 | pub fn new(dict: Option) -> Result { 27 | let ctx = MaybeUninit::<*mut LZ4FCompressionCtx>::uninit(); 28 | unsafe { 29 | let code = binding::LZ4F_createCompressionContext( 30 | ctx.as_ptr() as *mut *mut binding::LZ4FCompressionCtx, 31 | binding::LZ4F_getVersion(), 32 | ); 33 | result_from_code(code).and_then(|_| { 34 | Ok(Self { 35 | ctx: NonNull::new(ctx.assume_init()) 36 | .ok_or_else(|| crate::Error::new(crate::ErrorKind::InitializationFailed))?, 37 | dict, 38 | }) 39 | }) 40 | } 41 | } 42 | 43 | pub fn begin(&mut self, dst: *mut u8, dst_len: usize, prefs: &Preferences) -> Result { 44 | let code = unsafe { 45 | if let Some(dict) = &self.dict { 46 | binding::LZ4F_compressBegin_usingCDict( 47 | self.ctx.as_ptr(), 48 | dst as *mut c_void, 49 | dst_len, 50 | dict.handle().0.as_ptr(), 51 | prefs as *const Preferences, 52 | ) 53 | } else { 54 | binding::LZ4F_compressBegin(self.ctx.as_ptr(), dst as *mut c_void, dst_len, prefs) 55 | } 56 | }; 57 | result_from_code(code).map(|_| code) 58 | } 59 | 60 | pub fn update( 61 | &mut self, 62 | dst: *mut u8, 63 | dst_len: usize, 64 | src: &[u8], 65 | stable_src: bool, 66 | ) -> Result { 67 | let opt = LZ4FCompressionOptions::stable(stable_src); 68 | let code = unsafe { 69 | binding::LZ4F_compressUpdate( 70 | self.ctx.as_ptr(), 71 | dst as *mut c_void, 72 | dst_len, 73 | src.as_ptr() as *const c_void, 74 | src.len(), 75 | &opt as *const LZ4FCompressionOptions, 76 | ) 77 | }; 78 | result_from_code(code).map(|_| code) 79 | } 80 | 81 | pub fn flush(&mut self, dst: *mut u8, dst_len: usize, stable_src: bool) -> Result { 82 | let opt = LZ4FCompressionOptions::stable(stable_src); 83 | let code = unsafe { 84 | binding::LZ4F_flush( 85 | self.ctx.as_ptr(), 86 | dst as *mut c_void, 87 | dst_len, 88 | &opt as *const LZ4FCompressionOptions, 89 | ) 90 | }; 91 | result_from_code(code).map(|_| code) 92 | } 93 | 94 | pub fn end(&mut self, dst: *mut u8, dst_len: usize, stable_src: bool) -> Result { 95 | let opt = LZ4FCompressionOptions::stable(stable_src); 96 | let code = unsafe { 97 | binding::LZ4F_compressEnd( 98 | self.ctx.as_ptr(), 99 | dst as *mut c_void, 100 | dst_len, 101 | &opt as *const LZ4FCompressionOptions, 102 | ) 103 | }; 104 | result_from_code(code).map(|_| code) 105 | } 106 | 107 | pub fn compress_bound(src_size: usize, prefs: &Preferences) -> usize { 108 | unsafe { binding::LZ4F_compressBound(src_size, prefs as *const Preferences) } 109 | } 110 | } 111 | 112 | impl Drop for CompressionContext { 113 | fn drop(&mut self) { 114 | unsafe { 115 | binding::LZ4F_freeCompressionContext(self.ctx.as_ptr()); 116 | } 117 | } 118 | } 119 | 120 | pub struct DecompressionContext { 121 | ctx: NonNull, 122 | } 123 | 124 | unsafe impl Send for DecompressionContext {} 125 | 126 | impl DecompressionContext { 127 | pub fn new() -> Result { 128 | let ctx = MaybeUninit::<*mut LZ4FDecompressionCtx>::uninit(); 129 | unsafe { 130 | let code = binding::LZ4F_createDecompressionContext( 131 | ctx.as_ptr() as *mut *mut binding::LZ4FDecompressionCtx, 132 | binding::LZ4F_getVersion(), 133 | ); 134 | result_from_code(code).and_then(|_| { 135 | Ok(Self { 136 | ctx: NonNull::new(ctx.assume_init()) 137 | .ok_or_else(|| crate::Error::new(crate::ErrorKind::InitializationFailed))?, 138 | }) 139 | }) 140 | } 141 | } 142 | 143 | pub fn get_frame_info(&self, src: &[u8]) -> Result<(FrameInfo, usize)> { 144 | let mut info = MaybeUninit::::uninit(); 145 | let mut src_len = src.len(); 146 | let code = unsafe { 147 | binding::LZ4F_getFrameInfo( 148 | self.ctx.as_ptr(), 149 | info.as_mut_ptr(), 150 | src.as_ptr() as *const c_void, 151 | &mut src_len as *mut usize, 152 | ) 153 | }; 154 | result_from_code(code).map(|_| (unsafe { info.assume_init() }, src_len)) 155 | } 156 | 157 | pub fn decompress_dict( 158 | &mut self, 159 | src: &[u8], 160 | dst: &mut [u8], 161 | dict: &[u8], 162 | stable_dst: bool, 163 | ) -> Result<(usize, usize, usize)> { 164 | let mut dst_len = dst.len(); 165 | let mut src_len = src.len(); 166 | let opt = LZ4FDecompressionOptions::stable(stable_dst); 167 | let code = unsafe { 168 | binding::LZ4F_decompress_usingDict( 169 | self.ctx.as_ptr(), 170 | dst.as_mut_ptr() as *mut c_void, 171 | &mut dst_len as *mut usize, 172 | src.as_ptr() as *const c_void, 173 | &mut src_len as *mut usize, 174 | dict.as_ptr() as *const c_void, 175 | dict.len(), 176 | &opt as *const LZ4FDecompressionOptions, 177 | ) 178 | }; 179 | result_from_code(code).map(|_| (src_len, dst_len, code)) 180 | } 181 | 182 | pub fn reset(&mut self) { 183 | unsafe { 184 | binding::LZ4F_resetDecompressionContext(self.ctx.as_ptr()); 185 | } 186 | } 187 | } 188 | 189 | impl Drop for DecompressionContext { 190 | fn drop(&mut self) { 191 | unsafe { 192 | binding::LZ4F_freeDecompressionContext(self.ctx.as_ptr()); 193 | } 194 | } 195 | } 196 | 197 | pub fn compress_frame_bound(src_size: usize, prefs: &Preferences) -> usize { 198 | unsafe { binding::LZ4F_compressFrameBound(src_size, prefs as *const Preferences) } 199 | } 200 | 201 | pub fn header_size(src: &[u8]) -> usize { 202 | unsafe { binding::LZ4F_headerSize(src.as_ptr() as *const c_void, src.len()) } 203 | } 204 | 205 | pub fn compress(src: &[u8], dst: *mut u8, dst_len: usize, prefs: &Preferences) -> Result { 206 | let code = unsafe { 207 | binding::LZ4F_compressFrame( 208 | dst as *mut c_void, 209 | dst_len, 210 | src.as_ptr() as *const c_void, 211 | src.len(), 212 | prefs as *const Preferences, 213 | ) 214 | }; 215 | result_from_code(code).map(|_| code) 216 | } 217 | 218 | fn result_from_code(code: usize) -> Result<()> { 219 | Err(Error::new(match code.wrapping_neg() { 220 | 1 => ErrorKind::Generic, 221 | 2 => ErrorKind::MaxBlockSizeInvalid, 222 | 3 => ErrorKind::BlockModeInvalid, 223 | 4 => ErrorKind::ContentChecksumFlagInvalid, 224 | 5 => ErrorKind::CompressionLevelInvalid, 225 | 6 => ErrorKind::HeaderVersionWrong, 226 | 7 => ErrorKind::BlockChecksumInvalid, 227 | 8 => ErrorKind::ReservedFlagSet, 228 | 9 => ErrorKind::AllocationFailed, 229 | 10 => ErrorKind::SrcSizeTooLarge, 230 | 11 => ErrorKind::DstMaxSizeTooSmall, 231 | 12 => ErrorKind::FrameHeaderIncomplete, 232 | 13 => ErrorKind::FrameTypeUnknown, 233 | 14 => ErrorKind::FrameSizeWrong, 234 | 15 => ErrorKind::SrcPtrWrong, 235 | 16 => ErrorKind::DecompressionFailed, 236 | 17 => ErrorKind::HeaderChecksumInvalid, 237 | 18 => ErrorKind::ContentChecksumInvalid, 238 | 19 => ErrorKind::FrameDecodingAlreadyStarted, 239 | _ => return Ok(()), 240 | })) 241 | } 242 | 243 | pub struct DictionaryHandle(NonNull); 244 | 245 | unsafe impl Send for DictionaryHandle {} 246 | unsafe impl Sync for DictionaryHandle {} 247 | 248 | impl DictionaryHandle { 249 | pub fn new(data: &[u8]) -> Result { 250 | let dict = unsafe { binding::LZ4F_createCDict(data.as_ptr() as *const c_void, data.len()) }; 251 | NonNull::new(dict) 252 | .ok_or_else(|| crate::Error::new(crate::ErrorKind::InitializationFailed).into()) 253 | .map(Self) 254 | } 255 | } 256 | 257 | impl Drop for DictionaryHandle { 258 | fn drop(&mut self) { 259 | unsafe { 260 | binding::LZ4F_freeCDict(self.0.as_ptr()); 261 | } 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /src/lz4f/binding.rs: -------------------------------------------------------------------------------- 1 | use super::{FrameInfo, Preferences}; 2 | use std::os::raw::{c_uint, c_void}; 3 | 4 | #[allow(non_camel_case_types)] 5 | type size_t = usize; 6 | 7 | #[repr(C)] 8 | pub struct LZ4FCompressionCtx { 9 | _private: [u8; 0], 10 | } 11 | 12 | #[repr(C)] 13 | pub struct LZ4FDecompressionCtx { 14 | _private: [u8; 0], 15 | } 16 | 17 | #[repr(C)] 18 | pub struct LZ4FCompressionDict { 19 | _private: [u8; 0], 20 | } 21 | 22 | #[derive(Debug, Default, Copy, Clone)] 23 | #[repr(C)] 24 | pub struct LZ4FCompressionOptions { 25 | pub stable_src: c_uint, 26 | pub _reserved: [c_uint; 3], 27 | } 28 | 29 | impl LZ4FCompressionOptions { 30 | pub fn stable(stable: bool) -> Self { 31 | Self { 32 | stable_src: u32::from(stable), 33 | ..Default::default() 34 | } 35 | } 36 | } 37 | 38 | #[derive(Debug, Default, Copy, Clone)] 39 | #[repr(C)] 40 | pub struct LZ4FDecompressionOptions { 41 | pub stable_dst: c_uint, 42 | pub _reserved: [c_uint; 3], 43 | } 44 | 45 | impl LZ4FDecompressionOptions { 46 | pub fn stable(stable: bool) -> Self { 47 | Self { 48 | stable_dst: u32::from(stable), 49 | ..Default::default() 50 | } 51 | } 52 | } 53 | 54 | extern "C" { 55 | pub fn LZ4F_getVersion() -> c_uint; 56 | pub fn LZ4F_compressBound(src_size: size_t, prefs: *const Preferences) -> size_t; 57 | pub fn LZ4F_compressFrameBound(src_size: size_t, prefs: *const Preferences) -> size_t; 58 | pub fn LZ4F_compressFrame( 59 | dst_buffer: *mut c_void, 60 | dst_capacity: size_t, 61 | src_buffer: *const c_void, 62 | src_size: size_t, 63 | prefs: *const Preferences, 64 | ) -> size_t; 65 | pub fn LZ4F_decompress_usingDict( 66 | ctx: *mut LZ4FDecompressionCtx, 67 | dst_buffer: *mut c_void, 68 | dst_size_ptr: *mut size_t, 69 | src_buffer: *const c_void, 70 | src_size_ptr: *mut size_t, 71 | dict: *const c_void, 72 | dict_size: size_t, 73 | opt: *const LZ4FDecompressionOptions, 74 | ) -> size_t; 75 | pub fn LZ4F_createCDict( 76 | dict_buffer: *const c_void, 77 | dict_size: size_t, 78 | ) -> *mut LZ4FCompressionDict; 79 | pub fn LZ4F_freeCDict(dict: *mut LZ4FCompressionDict); 80 | 81 | pub fn LZ4F_createCompressionContext( 82 | ctx: *mut *mut LZ4FCompressionCtx, 83 | version: c_uint, 84 | ) -> size_t; 85 | pub fn LZ4F_freeCompressionContext(ctx: *mut LZ4FCompressionCtx); 86 | pub fn LZ4F_compressBegin( 87 | ctx: *mut LZ4FCompressionCtx, 88 | dst_buffer: *mut c_void, 89 | dst_capacity: size_t, 90 | prefs: *const Preferences, 91 | ) -> size_t; 92 | pub fn LZ4F_compressBegin_usingCDict( 93 | ctx: *mut LZ4FCompressionCtx, 94 | dst_buffer: *mut c_void, 95 | dst_capacity: size_t, 96 | dist: *const LZ4FCompressionDict, 97 | prefs: *const Preferences, 98 | ) -> size_t; 99 | pub fn LZ4F_compressUpdate( 100 | ctx: *mut LZ4FCompressionCtx, 101 | dst_buffer: *mut c_void, 102 | dst_capacity: size_t, 103 | src_buffer: *const c_void, 104 | src_size: size_t, 105 | opt: *const LZ4FCompressionOptions, 106 | ) -> size_t; 107 | pub fn LZ4F_flush( 108 | ctx: *mut LZ4FCompressionCtx, 109 | dst_buffer: *mut c_void, 110 | dst_capacity: size_t, 111 | opt: *const LZ4FCompressionOptions, 112 | ) -> size_t; 113 | pub fn LZ4F_compressEnd( 114 | ctx: *mut LZ4FCompressionCtx, 115 | dst_buffer: *mut c_void, 116 | dst_capacity: size_t, 117 | opt: *const LZ4FCompressionOptions, 118 | ) -> size_t; 119 | pub fn LZ4F_createDecompressionContext( 120 | ctx: *mut *mut LZ4FDecompressionCtx, 121 | version: c_uint, 122 | ) -> size_t; 123 | pub fn LZ4F_freeDecompressionContext(ctx: *mut LZ4FDecompressionCtx) -> size_t; 124 | pub fn LZ4F_resetDecompressionContext(ctx: *mut LZ4FDecompressionCtx); 125 | pub fn LZ4F_headerSize(src: *const c_void, src_size: size_t) -> size_t; 126 | pub fn LZ4F_getFrameInfo( 127 | ctx: *mut LZ4FDecompressionCtx, 128 | frame_info_ptr: *mut FrameInfo, 129 | src_buffer: *const c_void, 130 | src_size_ptr: *mut size_t, 131 | ) -> size_t; 132 | } 133 | -------------------------------------------------------------------------------- /src/lz4f/dictionary.rs: -------------------------------------------------------------------------------- 1 | use super::{api::DictionaryHandle, Result}; 2 | use std::sync::Arc; 3 | 4 | /// A pre-compiled dictionary for the efficient compression. 5 | #[derive(Clone)] 6 | pub struct Dictionary(Arc); 7 | 8 | impl Dictionary { 9 | /// Builds a new `Dictionary`. 10 | pub fn new(data: &[u8]) -> Result { 11 | Ok(Self(Arc::new(DictionaryHandle::new(data)?))) 12 | } 13 | 14 | pub(crate) fn handle(&self) -> &DictionaryHandle { 15 | &self.0 16 | } 17 | } 18 | 19 | #[cfg(test)] 20 | mod tests { 21 | use super::Dictionary; 22 | 23 | #[test] 24 | fn create_dictionary() { 25 | assert!(Dictionary::new(&[]).is_ok()); 26 | assert!(Dictionary::new(&b"quick brown fox jumps over the lazy dog"[..]).is_ok()); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/lz4f/error.rs: -------------------------------------------------------------------------------- 1 | use std::{convert, fmt, io}; 2 | 3 | /// A list specifying general categories of LZ4F error. 4 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 5 | #[non_exhaustive] 6 | #[allow(missing_docs)] 7 | pub enum ErrorKind { 8 | Generic, 9 | MaxBlockSizeInvalid, 10 | BlockModeInvalid, 11 | ContentChecksumFlagInvalid, 12 | CompressionLevelInvalid, 13 | HeaderVersionWrong, 14 | BlockChecksumInvalid, 15 | ReservedFlagSet, 16 | AllocationFailed, 17 | SrcSizeTooLarge, 18 | DstMaxSizeTooSmall, 19 | FrameHeaderIncomplete, 20 | FrameTypeUnknown, 21 | FrameSizeWrong, 22 | SrcPtrWrong, 23 | DecompressionFailed, 24 | HeaderChecksumInvalid, 25 | ContentChecksumInvalid, 26 | FrameDecodingAlreadyStarted, 27 | } 28 | 29 | impl fmt::Display for ErrorKind { 30 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> { 31 | ::fmt(self, f) 32 | } 33 | } 34 | 35 | /// The error type for LZ4F operations. 36 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 37 | pub enum Error { 38 | Lz4f(ErrorKind), 39 | Common(crate::ErrorKind), 40 | } 41 | 42 | impl Error { 43 | pub(super) const fn new(kind: ErrorKind) -> Self { 44 | Self::Lz4f(kind) 45 | } 46 | } 47 | 48 | impl convert::From for io::Error { 49 | fn from(err: Error) -> Self { 50 | Self::new(io::ErrorKind::Other, err) 51 | } 52 | } 53 | 54 | impl convert::From for Error { 55 | fn from(err: crate::Error) -> Self { 56 | Self::Common(err.kind()) 57 | } 58 | } 59 | 60 | impl fmt::Display for Error { 61 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> { 62 | match self { 63 | Self::Lz4f(kind) => ::fmt(kind, f), 64 | Self::Common(kind) => ::fmt(kind, f), 65 | } 66 | } 67 | } 68 | 69 | impl std::error::Error for Error {} 70 | 71 | /// A specialized [`Result`] type for LZ4F operations. 72 | /// 73 | /// [`Result`]: https://doc.rust-lang.org/std/result/enum.Result.html 74 | pub type Result = std::result::Result; 75 | -------------------------------------------------------------------------------- /src/lz4f/frame.rs: -------------------------------------------------------------------------------- 1 | //! LZ4 Frame Compressor/Decompressor 2 | 3 | use super::{api, Result}; 4 | use crate::{common::DEFAULT_BUF_SIZE, lz4f::Preferences, Error, ErrorKind}; 5 | use std::{cell::RefCell, ops::Deref}; 6 | 7 | /// Calculates the maximum size of the compressed output. 8 | /// 9 | /// If `original_size` is too large to compress, this returns `0`. 10 | /// 11 | /// Returned values are reliable only for [`compress`] and [`compress_to_vec`]. 12 | /// Streaming compressors may produce larger compressed frames. 13 | /// 14 | /// [`compress`]: fn.compress.html 15 | /// [`compress_to_vec`]: fn.compress_to_vec.html 16 | #[must_use] 17 | pub fn max_compressed_size(original_size: usize, prefs: &Preferences) -> usize { 18 | api::compress_frame_bound(original_size, prefs) 19 | } 20 | 21 | /// Performs LZ4F compression. 22 | /// 23 | /// Ensure that the destination slice has enough capacity. 24 | /// If `dst.len()` is smaller than `lz4f::max_compressed_size(src.len())`, 25 | /// this function may fail. 26 | /// 27 | /// Returns the number of bytes written into the destination buffer. 28 | /// 29 | /// # Example 30 | /// 31 | /// Compress data with the default compression mode: 32 | /// ``` 33 | /// use lzzzz::lz4f; 34 | /// 35 | /// let prefs = lz4f::Preferences::default(); 36 | /// let data = b"The quick brown fox jumps over the lazy dog."; 37 | /// let mut buf = [0u8; 2048]; 38 | /// 39 | /// // The slice should have enough capacity. 40 | /// assert!(buf.len() >= lz4f::max_compressed_size(data.len(), &prefs)); 41 | /// 42 | /// let len = lz4f::compress(data, &mut buf, &prefs)?; 43 | /// let compressed = &buf[..len]; 44 | /// # let mut buf = Vec::new(); 45 | /// # lz4f::decompress_to_vec(compressed, &mut buf)?; 46 | /// # assert_eq!(buf.as_slice(), &data[..]); 47 | /// # Ok::<(), std::io::Error>(()) 48 | /// ``` 49 | pub fn compress(src: &[u8], dst: &mut [u8], prefs: &Preferences) -> Result { 50 | compress_to_ptr(src, dst.as_mut_ptr(), dst.len(), prefs) 51 | } 52 | 53 | fn compress_to_ptr(src: &[u8], dst: *mut u8, dst_len: usize, prefs: &Preferences) -> Result { 54 | let mut prefs = *prefs; 55 | if prefs.frame_info().content_size() > 0 { 56 | prefs.set_content_size(src.len()); 57 | } 58 | api::compress(src, dst, dst_len, &prefs) 59 | } 60 | 61 | /// Appends a compressed frame to `Vec`. 62 | /// 63 | /// Returns the number of bytes appended to the given `Vec`. 64 | /// 65 | /// # Example 66 | /// 67 | /// Compress data with the default compression mode: 68 | /// ``` 69 | /// use lzzzz::lz4f; 70 | /// 71 | /// let prefs = lz4f::Preferences::default(); 72 | /// let data = b"The quick brown fox jumps over the lazy dog."; 73 | /// let mut buf = Vec::new(); 74 | /// 75 | /// let len = lz4f::compress_to_vec(data, &mut buf, &prefs)?; 76 | /// let compressed = &buf; 77 | /// # let mut buf = Vec::new(); 78 | /// # lz4f::decompress_to_vec(compressed, &mut buf)?; 79 | /// # assert_eq!(buf.as_slice(), &data[..]); 80 | /// # Ok::<(), std::io::Error>(()) 81 | /// ``` 82 | pub fn compress_to_vec(src: &[u8], dst: &mut Vec, prefs: &Preferences) -> Result { 83 | let orig_len = dst.len(); 84 | dst.reserve(max_compressed_size(src.len(), prefs)); 85 | #[allow(unsafe_code)] 86 | unsafe { 87 | let result = compress_to_ptr( 88 | src, 89 | dst.as_mut_ptr().add(orig_len), 90 | dst.capacity() - orig_len, 91 | prefs, 92 | ); 93 | dst.set_len(orig_len + result.as_ref().unwrap_or(&0)); 94 | result 95 | } 96 | } 97 | 98 | /// Decompresses an LZ4 frame. 99 | /// 100 | /// Returns the number of bytes appended to the given `Vec`. 101 | /// 102 | /// # Example 103 | /// 104 | /// ``` 105 | /// use lzzzz::lz4f; 106 | /// 107 | /// const COMPRESSED_DATA: &str = 108 | /// "BCJNGGBAgiwAAIBUaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGhlIGxhenkgZG9nLgAAAAA="; 109 | /// 110 | /// let data = base64::decode(COMPRESSED_DATA).unwrap(); 111 | /// let mut buf = Vec::new(); 112 | /// 113 | /// lz4f::decompress_to_vec(&data[..], &mut buf)?; 114 | /// 115 | /// assert_eq!( 116 | /// &buf[..], 117 | /// &b"The quick brown fox jumps over the lazy dog."[..] 118 | /// ); 119 | /// # Ok::<(), std::io::Error>(()) 120 | /// ``` 121 | pub fn decompress_to_vec(src: &[u8], dst: &mut Vec) -> Result { 122 | let header_len = dst.len(); 123 | let mut src_offset = 0; 124 | let mut dst_offset = header_len; 125 | DecompressionCtx::with(|ctx| { 126 | let mut ctx = ctx.borrow_mut(); 127 | ctx.reset(); 128 | loop { 129 | dst.resize_with(dst.len() + DEFAULT_BUF_SIZE, Default::default); 130 | match ctx.decompress_dict(&src[src_offset..], &mut dst[dst_offset..], &[], false) { 131 | Ok((src_len, dst_len, expected)) => { 132 | src_offset += src_len; 133 | dst_offset += dst_len; 134 | if expected == 0 { 135 | dst.resize_with(dst_offset, Default::default); 136 | return Ok(dst_offset - header_len); 137 | } else if src_offset >= src.len() { 138 | dst.resize_with(header_len, Default::default); 139 | return Err(Error::new(ErrorKind::CompressedDataIncomplete).into()); 140 | } 141 | } 142 | Err(err) => { 143 | dst.resize_with(header_len, Default::default); 144 | return Err(err); 145 | } 146 | } 147 | } 148 | }) 149 | } 150 | 151 | struct DecompressionCtx(RefCell); 152 | 153 | impl DecompressionCtx { 154 | fn new() -> Self { 155 | Self(RefCell::new(api::DecompressionContext::new().unwrap())) 156 | } 157 | 158 | fn with(f: F) -> R 159 | where 160 | F: FnOnce(&RefCell) -> R, 161 | { 162 | DECOMPRESSION_CTX.with(|state| (f)(state)) 163 | } 164 | } 165 | 166 | impl Deref for DecompressionCtx { 167 | type Target = RefCell; 168 | 169 | fn deref(&self) -> &Self::Target { 170 | &self.0 171 | } 172 | } 173 | 174 | thread_local!(static DECOMPRESSION_CTX: DecompressionCtx = DecompressionCtx::new()); 175 | -------------------------------------------------------------------------------- /src/lz4f/frame_info.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::{c_uint, c_ulonglong}; 2 | 3 | /// Block size flag. 4 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 5 | #[non_exhaustive] 6 | #[repr(C)] 7 | pub enum BlockSize { 8 | Default = 0, 9 | Max64KB = 4, 10 | Max256KB = 5, 11 | Max1MB = 6, 12 | Max4MB = 7, 13 | } 14 | 15 | impl Default for BlockSize { 16 | fn default() -> Self { 17 | Self::Default 18 | } 19 | } 20 | 21 | /// Block mode flag. 22 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 23 | #[repr(C)] 24 | pub enum BlockMode { 25 | Linked, 26 | Independent, 27 | } 28 | 29 | impl Default for BlockMode { 30 | fn default() -> Self { 31 | Self::Linked 32 | } 33 | } 34 | 35 | /// Content checksum flag. 36 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 37 | #[repr(C)] 38 | pub enum ContentChecksum { 39 | Disabled, 40 | Enabled, 41 | } 42 | 43 | impl Default for ContentChecksum { 44 | fn default() -> Self { 45 | Self::Disabled 46 | } 47 | } 48 | 49 | /// Frame type flag. 50 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 51 | #[repr(C)] 52 | pub enum FrameType { 53 | Frame, 54 | SkippableFrame, 55 | } 56 | 57 | impl Default for FrameType { 58 | fn default() -> Self { 59 | Self::Frame 60 | } 61 | } 62 | 63 | /// Block checksum flag. 64 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 65 | #[repr(C)] 66 | pub enum BlockChecksum { 67 | Disabled, 68 | Enabled, 69 | } 70 | 71 | impl Default for BlockChecksum { 72 | fn default() -> Self { 73 | Self::Disabled 74 | } 75 | } 76 | 77 | /// LZ4 Frame parameters. 78 | #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash)] 79 | #[repr(C)] 80 | pub struct FrameInfo { 81 | block_size: BlockSize, 82 | block_mode: BlockMode, 83 | content_checksum: ContentChecksum, 84 | frame_type: FrameType, 85 | content_size: c_ulonglong, 86 | dict_id: c_uint, 87 | block_checksum: BlockChecksum, 88 | } 89 | 90 | impl FrameInfo { 91 | /// Returns the block size. 92 | pub const fn block_size(&self) -> BlockSize { 93 | self.block_size 94 | } 95 | 96 | /// Returns the block mode. 97 | pub const fn block_mode(&self) -> BlockMode { 98 | self.block_mode 99 | } 100 | 101 | /// Returns the content checksum. 102 | pub const fn content_checksum(&self) -> ContentChecksum { 103 | self.content_checksum 104 | } 105 | 106 | /// Returns the frame type. 107 | pub const fn frame_type(&self) -> FrameType { 108 | self.frame_type 109 | } 110 | 111 | /// Returns the content size. 112 | pub const fn content_size(&self) -> usize { 113 | self.content_size as usize 114 | } 115 | 116 | /// Returns the dictionary id. 117 | pub const fn dict_id(&self) -> u32 { 118 | self.dict_id 119 | } 120 | 121 | /// Returns the block checksum. 122 | pub const fn block_checksum(&self) -> BlockChecksum { 123 | self.block_checksum 124 | } 125 | 126 | pub(super) fn set_block_size(&mut self, block_size: BlockSize) { 127 | self.block_size = block_size; 128 | } 129 | 130 | pub(super) fn set_block_mode(&mut self, block_mode: BlockMode) { 131 | self.block_mode = block_mode; 132 | } 133 | 134 | pub(super) fn set_content_checksum(&mut self, checksum: ContentChecksum) { 135 | self.content_checksum = checksum; 136 | } 137 | 138 | pub(super) fn set_content_size(&mut self, size: usize) { 139 | self.content_size = size as c_ulonglong; 140 | } 141 | 142 | pub(super) fn set_dict_id(&mut self, dict_id: u32) { 143 | self.dict_id = dict_id as c_uint; 144 | } 145 | 146 | pub(super) fn set_block_checksum(&mut self, checksum: BlockChecksum) { 147 | self.block_checksum = checksum; 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/lz4f/mod.rs: -------------------------------------------------------------------------------- 1 | //! LZ4F compression and decompression. 2 | //! 3 | //! LZ4F: LZ4 Frame Format. 4 | mod api; 5 | mod binding; 6 | mod dictionary; 7 | mod error; 8 | mod frame; 9 | mod frame_info; 10 | mod preferences; 11 | mod stream; 12 | 13 | pub use dictionary::*; 14 | pub use error::*; 15 | pub use frame::*; 16 | pub use frame_info::*; 17 | pub use preferences::*; 18 | pub use stream::{comp::*, decomp::*}; 19 | -------------------------------------------------------------------------------- /src/lz4f/preferences.rs: -------------------------------------------------------------------------------- 1 | use super::frame_info::{BlockChecksum, BlockMode, BlockSize, ContentChecksum, FrameInfo}; 2 | use std::os::raw::{c_int, c_uint}; 3 | 4 | /// Predefined compression level (0). 5 | pub const CLEVEL_DEFAULT: i32 = 0; 6 | 7 | /// Predefined compression level (10). 8 | pub const CLEVEL_HIGH: i32 = 10; 9 | 10 | /// Predefined compression level (12). 11 | pub const CLEVEL_MAX: i32 = 12; 12 | 13 | /// Auto flush mode flag. 14 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 15 | #[repr(C)] 16 | pub enum AutoFlush { 17 | Disabled, 18 | Enabled, 19 | } 20 | 21 | impl Default for AutoFlush { 22 | fn default() -> Self { 23 | Self::Disabled 24 | } 25 | } 26 | 27 | /// Decompression speed mode flag. 28 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] 29 | #[repr(C)] 30 | pub enum FavorDecSpeed { 31 | Disabled, 32 | Enabled, 33 | } 34 | 35 | impl Default for FavorDecSpeed { 36 | fn default() -> Self { 37 | Self::Disabled 38 | } 39 | } 40 | 41 | /// Compression preferences. 42 | #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash)] 43 | #[repr(C)] 44 | pub struct Preferences { 45 | frame_info: FrameInfo, 46 | compression_level: c_int, 47 | auto_flush: AutoFlush, 48 | favor_dec_speed: FavorDecSpeed, 49 | _reserved: [c_uint; 3], 50 | } 51 | 52 | impl Preferences { 53 | /// Returns the frame info. 54 | pub const fn frame_info(&self) -> FrameInfo { 55 | self.frame_info 56 | } 57 | 58 | /// Returns the compression level. 59 | pub const fn compression_level(&self) -> i32 { 60 | self.compression_level 61 | } 62 | 63 | /// Returns the auto flush mode flag. 64 | pub const fn auto_flush(&self) -> AutoFlush { 65 | self.auto_flush 66 | } 67 | 68 | /// Returns the decompression speed mode flag. 69 | pub const fn favor_dec_speed(&self) -> FavorDecSpeed { 70 | self.favor_dec_speed 71 | } 72 | 73 | pub(super) fn set_block_size(&mut self, block_size: BlockSize) { 74 | self.frame_info.set_block_size(block_size); 75 | } 76 | 77 | pub(super) fn set_block_mode(&mut self, block_mode: BlockMode) { 78 | self.frame_info.set_block_mode(block_mode); 79 | } 80 | 81 | pub(super) fn set_content_checksum(&mut self, checksum: ContentChecksum) { 82 | self.frame_info.set_content_checksum(checksum); 83 | } 84 | 85 | pub(super) fn set_content_size(&mut self, size: usize) { 86 | self.frame_info.set_content_size(size); 87 | } 88 | 89 | pub(super) fn set_dict_id(&mut self, dict_id: u32) { 90 | self.frame_info.set_dict_id(dict_id); 91 | } 92 | 93 | pub(super) fn set_block_checksum(&mut self, checksum: BlockChecksum) { 94 | self.frame_info.set_block_checksum(checksum); 95 | } 96 | 97 | pub(super) fn set_compression_level(&mut self, level: i32) { 98 | self.compression_level = level as c_int; 99 | } 100 | 101 | pub(super) fn set_favor_dec_speed(&mut self, dec_speed: FavorDecSpeed) { 102 | self.favor_dec_speed = dec_speed; 103 | } 104 | 105 | pub(super) fn set_auto_flush(&mut self, auto_flush: AutoFlush) { 106 | self.auto_flush = auto_flush; 107 | } 108 | } 109 | 110 | /// Builds a custom `Preferences`. 111 | /// 112 | /// # Example 113 | /// 114 | /// ``` 115 | /// use lzzzz::lz4f::{BlockSize, PreferencesBuilder, CLEVEL_MAX}; 116 | /// 117 | /// let pref = PreferencesBuilder::new() 118 | /// .block_size(BlockSize::Max1MB) 119 | /// .compression_level(CLEVEL_MAX) 120 | /// .build(); 121 | /// ``` 122 | #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)] 123 | pub struct PreferencesBuilder { 124 | prefs: Preferences, 125 | } 126 | 127 | impl PreferencesBuilder { 128 | /// Creates a new `PreferencesBuilder`. 129 | pub fn new() -> Self { 130 | Default::default() 131 | } 132 | 133 | /// Sets the block size. 134 | pub fn block_size(&mut self, block_size: BlockSize) -> &mut Self { 135 | self.prefs.set_block_size(block_size); 136 | self 137 | } 138 | 139 | /// Sets the block mode. 140 | pub fn block_mode(&mut self, block_mode: BlockMode) -> &mut Self { 141 | self.prefs.set_block_mode(block_mode); 142 | self 143 | } 144 | 145 | /// Sets the content checksum. 146 | pub fn content_checksum(&mut self, checksum: ContentChecksum) -> &mut Self { 147 | self.prefs.set_content_checksum(checksum); 148 | self 149 | } 150 | 151 | /// Sets the content size. 152 | /// 153 | /// A value greater than 0 enables the content size field in the frame header and 154 | /// automatically replaced with an actual content size. 155 | pub fn content_size(&mut self, size: usize) -> &mut Self { 156 | self.prefs.set_content_size(size); 157 | self 158 | } 159 | 160 | /// Sets the dictionary id. 161 | pub fn dict_id(&mut self, dict_id: u32) -> &mut Self { 162 | self.prefs.set_dict_id(dict_id); 163 | self 164 | } 165 | 166 | /// Sets the block checksum. 167 | pub fn block_checksum(&mut self, checksum: BlockChecksum) -> &mut Self { 168 | self.prefs.set_block_checksum(checksum); 169 | self 170 | } 171 | 172 | /// Sets the compression level. 173 | pub fn compression_level(&mut self, level: i32) -> &mut Self { 174 | self.prefs.set_compression_level(level); 175 | self 176 | } 177 | 178 | /// Sets the decompression speed mode flag. 179 | pub fn favor_dec_speed(&mut self, dec_speed: FavorDecSpeed) -> &mut Self { 180 | self.prefs.set_favor_dec_speed(dec_speed); 181 | self 182 | } 183 | 184 | /// Sets the auto flush mode flag. 185 | pub fn auto_flush(&mut self, auto_flush: AutoFlush) -> &mut Self { 186 | self.prefs.set_auto_flush(auto_flush); 187 | self 188 | } 189 | 190 | /// Builds a `Preferences` with this configuration. 191 | pub const fn build(&self) -> Preferences { 192 | self.prefs 193 | } 194 | } 195 | 196 | impl From for PreferencesBuilder { 197 | fn from(prefs: Preferences) -> Self { 198 | Self { prefs } 199 | } 200 | } 201 | 202 | #[cfg(test)] 203 | mod tests { 204 | use crate::lz4f::{ 205 | BlockChecksum, BlockMode, BlockSize, ContentChecksum, FavorDecSpeed, Preferences, 206 | PreferencesBuilder, CLEVEL_DEFAULT, CLEVEL_HIGH, CLEVEL_MAX, 207 | }; 208 | use std::{i32, u32}; 209 | 210 | #[test] 211 | fn preferences_builder() { 212 | assert_eq!(PreferencesBuilder::new().build(), Preferences::default()); 213 | assert_eq!( 214 | PreferencesBuilder::new() 215 | .favor_dec_speed(FavorDecSpeed::Enabled) 216 | .build() 217 | .favor_dec_speed, 218 | FavorDecSpeed::Enabled 219 | ); 220 | assert_eq!( 221 | PreferencesBuilder::new() 222 | .block_size(BlockSize::Max64KB) 223 | .build() 224 | .frame_info 225 | .block_size(), 226 | BlockSize::Max64KB 227 | ); 228 | assert_eq!( 229 | PreferencesBuilder::new() 230 | .block_size(BlockSize::Max256KB) 231 | .build() 232 | .frame_info 233 | .block_size(), 234 | BlockSize::Max256KB 235 | ); 236 | assert_eq!( 237 | PreferencesBuilder::new() 238 | .block_size(BlockSize::Max1MB) 239 | .build() 240 | .frame_info 241 | .block_size(), 242 | BlockSize::Max1MB 243 | ); 244 | assert_eq!( 245 | PreferencesBuilder::new() 246 | .block_size(BlockSize::Max4MB) 247 | .build() 248 | .frame_info 249 | .block_size(), 250 | BlockSize::Max4MB 251 | ); 252 | assert_eq!( 253 | PreferencesBuilder::new() 254 | .content_checksum(ContentChecksum::Enabled) 255 | .build() 256 | .frame_info 257 | .content_checksum(), 258 | ContentChecksum::Enabled 259 | ); 260 | assert_eq!( 261 | PreferencesBuilder::new() 262 | .block_mode(BlockMode::Independent) 263 | .build() 264 | .frame_info 265 | .block_mode(), 266 | BlockMode::Independent 267 | ); 268 | assert_eq!( 269 | PreferencesBuilder::new() 270 | .compression_level(i32::MAX) 271 | .build() 272 | .compression_level, 273 | i32::MAX 274 | ); 275 | assert_eq!( 276 | PreferencesBuilder::new() 277 | .compression_level(CLEVEL_DEFAULT) 278 | .build() 279 | .compression_level, 280 | CLEVEL_DEFAULT 281 | ); 282 | assert_eq!( 283 | PreferencesBuilder::new() 284 | .compression_level(CLEVEL_HIGH) 285 | .build() 286 | .compression_level, 287 | CLEVEL_HIGH 288 | ); 289 | assert_eq!( 290 | PreferencesBuilder::new() 291 | .compression_level(CLEVEL_MAX) 292 | .build() 293 | .compression_level, 294 | CLEVEL_MAX 295 | ); 296 | assert_eq!( 297 | PreferencesBuilder::new() 298 | .compression_level(i32::MIN) 299 | .build() 300 | .compression_level, 301 | i32::MIN 302 | ); 303 | assert_eq!( 304 | PreferencesBuilder::new() 305 | .block_checksum(BlockChecksum::Enabled) 306 | .build() 307 | .frame_info 308 | .block_checksum(), 309 | BlockChecksum::Enabled 310 | ); 311 | assert_eq!( 312 | PreferencesBuilder::new() 313 | .dict_id(u32::MAX) 314 | .build() 315 | .frame_info 316 | .dict_id(), 317 | u32::MAX 318 | ); 319 | assert_eq!( 320 | PreferencesBuilder::new() 321 | .dict_id(u32::MIN) 322 | .build() 323 | .frame_info 324 | .dict_id(), 325 | u32::MIN 326 | ); 327 | } 328 | } 329 | -------------------------------------------------------------------------------- /src/lz4f/stream/comp/bufread.rs: -------------------------------------------------------------------------------- 1 | use super::{Compressor, Dictionary, Preferences}; 2 | use crate::lz4f::Result; 3 | use std::{ 4 | fmt, 5 | io::{BufRead, Read}, 6 | }; 7 | 8 | /// The [`BufRead`]-based streaming compressor. 9 | /// 10 | /// # Example 11 | /// 12 | /// ``` 13 | /// # use std::env; 14 | /// # use std::path::Path; 15 | /// # use lzzzz::{Error, Result}; 16 | /// # use assert_fs::prelude::*; 17 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 18 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 19 | /// # 20 | /// # tmp_dir.child("foo.txt").write_str("Hello").unwrap(); 21 | /// # 22 | /// use lzzzz::lz4f::BufReadCompressor; 23 | /// use std::{ 24 | /// fs::File, 25 | /// io::{prelude::*, BufReader}, 26 | /// }; 27 | /// 28 | /// let mut f = File::open("foo.txt")?; 29 | /// let mut b = BufReader::new(f); 30 | /// let mut r = BufReadCompressor::new(&mut b, Default::default())?; 31 | /// 32 | /// let mut buf = Vec::new(); 33 | /// r.read_to_end(&mut buf)?; 34 | /// # Ok::<(), std::io::Error>(()) 35 | /// ``` 36 | /// 37 | /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html 38 | pub struct BufReadCompressor { 39 | pub(super) inner: R, 40 | pub(super) comp: Compressor, 41 | consumed: usize, 42 | } 43 | 44 | impl BufReadCompressor { 45 | /// Creates a new `BufReadCompressor`. 46 | pub fn new(reader: R, prefs: Preferences) -> Result { 47 | Ok(Self { 48 | inner: reader, 49 | comp: Compressor::new(prefs, None)?, 50 | consumed: 0, 51 | }) 52 | } 53 | 54 | /// Creates a new `BufReadCompressor` with a dictionary. 55 | pub fn with_dict(reader: R, prefs: Preferences, dict: Dictionary) -> Result { 56 | Ok(Self { 57 | inner: reader, 58 | comp: Compressor::new(prefs, Some(dict))?, 59 | consumed: 0, 60 | }) 61 | } 62 | 63 | /// Returns ownership of the reader. 64 | pub fn into_inner(self) -> R { 65 | self.inner 66 | } 67 | 68 | /// Returns a mutable reference to the reader. 69 | pub fn get_mut(&mut self) -> &mut R { 70 | &mut self.inner 71 | } 72 | 73 | /// Returns a shared reference to the reader. 74 | pub fn get_ref(&self) -> &R { 75 | &self.inner 76 | } 77 | } 78 | 79 | impl fmt::Debug for BufReadCompressor 80 | where 81 | R: BufRead + fmt::Debug, 82 | { 83 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 84 | fmt.debug_struct("BufReadCompressor") 85 | .field("reader", &self.inner) 86 | .field("prefs", &self.comp.prefs()) 87 | .finish() 88 | } 89 | } 90 | 91 | impl Read for BufReadCompressor { 92 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 93 | let consumed = { 94 | let inner_buf = self.inner.fill_buf()?; 95 | if inner_buf.is_empty() { 96 | self.comp.end(false)?; 97 | if self.comp.buf().is_empty() { 98 | return Ok(0); 99 | } 100 | 0 101 | } else { 102 | self.comp.update(inner_buf, false)?; 103 | inner_buf.len() 104 | } 105 | }; 106 | self.inner.consume(consumed); 107 | 108 | let len = std::cmp::min(buf.len(), self.comp.buf().len() - self.consumed); 109 | buf[..len].copy_from_slice(&self.comp.buf()[self.consumed..][..len]); 110 | self.consumed += len; 111 | if self.consumed >= self.comp.buf().len() { 112 | self.comp.clear_buf(); 113 | self.consumed = 0; 114 | } 115 | Ok(len) 116 | } 117 | } 118 | 119 | impl BufRead for BufReadCompressor { 120 | fn fill_buf(&mut self) -> std::io::Result<&[u8]> { 121 | let _ = self.read(&mut [])?; 122 | Ok(&self.comp.buf()[self.consumed..]) 123 | } 124 | 125 | fn consume(&mut self, amt: usize) { 126 | self.consumed += amt; 127 | if self.consumed >= self.comp.buf().len() { 128 | self.comp.clear_buf(); 129 | self.consumed = 0; 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/lz4f/stream/comp/mod.rs: -------------------------------------------------------------------------------- 1 | //! Streaming LZ4F compressors. 2 | mod bufread; 3 | mod read; 4 | mod write; 5 | 6 | use crate::lz4f::Result; 7 | 8 | pub use bufread::*; 9 | pub use read::*; 10 | pub use write::*; 11 | 12 | use crate::lz4f::{ 13 | api::{CompressionContext, LZ4F_HEADER_SIZE_MAX}, 14 | Dictionary, Preferences, 15 | }; 16 | 17 | pub(crate) struct Compressor { 18 | ctx: CompressionContext, 19 | prefs: Preferences, 20 | state: State, 21 | buffer: Vec, 22 | } 23 | 24 | impl Compressor { 25 | pub fn new(prefs: Preferences, dict: Option) -> Result { 26 | Ok(Self { 27 | ctx: CompressionContext::new(dict)?, 28 | prefs, 29 | state: State::Created, 30 | buffer: Vec::with_capacity(LZ4F_HEADER_SIZE_MAX), 31 | }) 32 | } 33 | 34 | pub fn prefs(&self) -> &Preferences { 35 | &self.prefs 36 | } 37 | 38 | fn begin(&mut self) -> Result<()> { 39 | if let State::Created = self.state { 40 | assert!(self.buffer.is_empty()); 41 | self.state = State::Active; 42 | let len = self.ctx.begin( 43 | self.buffer.as_mut_ptr(), 44 | self.buffer.capacity(), 45 | &self.prefs, 46 | )?; 47 | #[allow(unsafe_code)] 48 | unsafe { 49 | self.buffer.set_len(len); 50 | } 51 | } 52 | Ok(()) 53 | } 54 | 55 | pub fn update(&mut self, src: &[u8], stable_src: bool) -> Result<()> { 56 | self.begin()?; 57 | let ext_len = CompressionContext::compress_bound(src.len(), &self.prefs); 58 | self.buffer.reserve(ext_len); 59 | let offset = self.buffer.len(); 60 | #[allow(unsafe_code)] 61 | unsafe { 62 | let len = self.ctx.update( 63 | self.buffer.as_mut_ptr().add(offset), 64 | self.buffer.capacity() - offset, 65 | src, 66 | stable_src, 67 | )?; 68 | self.buffer.set_len(offset + len); 69 | if len == 0 { 70 | self.flush(stable_src) 71 | } else { 72 | Ok(()) 73 | } 74 | } 75 | } 76 | 77 | pub fn flush(&mut self, stable_src: bool) -> Result<()> { 78 | self.begin()?; 79 | let ext_len = CompressionContext::compress_bound(0, &self.prefs); 80 | self.buffer.reserve(ext_len); 81 | let offset = self.buffer.len(); 82 | #[allow(unsafe_code)] 83 | unsafe { 84 | let len = self.ctx.flush( 85 | self.buffer.as_mut_ptr().add(offset), 86 | self.buffer.capacity() - offset, 87 | stable_src, 88 | )?; 89 | self.buffer.set_len(offset + len); 90 | } 91 | Ok(()) 92 | } 93 | 94 | pub fn end(&mut self, stable_src: bool) -> Result<()> { 95 | self.begin()?; 96 | if let State::Active = self.state { 97 | self.state = State::Finished; 98 | let ext_len = CompressionContext::compress_bound(0, &self.prefs); 99 | self.buffer.reserve(ext_len); 100 | let offset = self.buffer.len(); 101 | #[allow(unsafe_code)] 102 | unsafe { 103 | let len = self.ctx.end( 104 | self.buffer.as_mut_ptr().add(offset), 105 | self.buffer.capacity() - offset, 106 | stable_src, 107 | )?; 108 | self.buffer.set_len(offset + len); 109 | } 110 | } 111 | Ok(()) 112 | } 113 | 114 | pub fn buf(&self) -> &[u8] { 115 | &self.buffer 116 | } 117 | 118 | pub fn clear_buf(&mut self) { 119 | self.buffer.clear(); 120 | } 121 | } 122 | 123 | pub(crate) enum State { 124 | Created, 125 | Active, 126 | Finished, 127 | } 128 | -------------------------------------------------------------------------------- /src/lz4f/stream/comp/read.rs: -------------------------------------------------------------------------------- 1 | use super::{BufReadCompressor, Dictionary, Preferences}; 2 | use crate::lz4f::Result; 3 | use std::{ 4 | fmt, 5 | io::{BufReader, Read}, 6 | }; 7 | 8 | /// The [`Read`]-based streaming compressor. 9 | /// 10 | /// # Example 11 | /// 12 | /// ``` 13 | /// # use std::env; 14 | /// # use std::path::Path; 15 | /// # use lzzzz::{Error, Result}; 16 | /// # use assert_fs::prelude::*; 17 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 18 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 19 | /// # 20 | /// # tmp_dir.child("foo.txt").write_str("Hello").unwrap(); 21 | /// # 22 | /// use lzzzz::lz4f::ReadCompressor; 23 | /// use std::{fs::File, io::prelude::*}; 24 | /// 25 | /// let mut f = File::open("foo.txt")?; 26 | /// let mut r = ReadCompressor::new(&mut f, Default::default())?; 27 | /// 28 | /// let mut buf = Vec::new(); 29 | /// r.read_to_end(&mut buf)?; 30 | /// # Ok::<(), std::io::Error>(()) 31 | /// ``` 32 | /// 33 | /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html 34 | pub struct ReadCompressor { 35 | inner: BufReadCompressor>, 36 | } 37 | 38 | impl ReadCompressor { 39 | /// Creates a new `ReadCompressor`. 40 | pub fn new(reader: R, prefs: Preferences) -> Result { 41 | Ok(Self { 42 | inner: BufReadCompressor::new(BufReader::new(reader), prefs)?, 43 | }) 44 | } 45 | 46 | /// Creates a new `ReadCompressor` with a dictionary. 47 | pub fn with_dict(reader: R, prefs: Preferences, dict: Dictionary) -> Result { 48 | Ok(Self { 49 | inner: BufReadCompressor::with_dict(BufReader::new(reader), prefs, dict)?, 50 | }) 51 | } 52 | 53 | /// Returns ownership of the reader. 54 | pub fn into_inner(self) -> R { 55 | self.inner.into_inner().into_inner() 56 | } 57 | 58 | /// Returns a mutable reference to the reader. 59 | pub fn get_mut(&mut self) -> &mut R { 60 | self.inner.get_mut().get_mut() 61 | } 62 | 63 | /// Returns a shared reference to the reader. 64 | pub fn get_ref(&self) -> &R { 65 | self.inner.get_ref().get_ref() 66 | } 67 | } 68 | 69 | impl fmt::Debug for ReadCompressor 70 | where 71 | R: Read + fmt::Debug, 72 | { 73 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 74 | fmt.debug_struct("ReadCompressor") 75 | .field("reader", &self.inner.inner.get_ref()) 76 | .field("prefs", &self.inner.comp.prefs()) 77 | .finish() 78 | } 79 | } 80 | 81 | impl Read for ReadCompressor { 82 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 83 | self.inner.read(buf) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/lz4f/stream/comp/write.rs: -------------------------------------------------------------------------------- 1 | use super::{Compressor, Dictionary, Preferences}; 2 | use crate::lz4f::Result; 3 | use std::{fmt, io::Write}; 4 | 5 | /// The [`Write`]-based streaming compressor. 6 | /// 7 | /// # Example 8 | /// 9 | /// ``` 10 | /// # use std::env; 11 | /// # use std::path::Path; 12 | /// # use lzzzz::{Error, Result}; 13 | /// # use assert_fs::prelude::*; 14 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 15 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 16 | /// use lzzzz::lz4f::WriteCompressor; 17 | /// use std::{fs::File, io::prelude::*}; 18 | /// 19 | /// let mut f = File::create("foo.lz4")?; 20 | /// let mut w = WriteCompressor::new(&mut f, Default::default())?; 21 | /// 22 | /// w.write_all(b"Hello world!")?; 23 | /// # Ok::<(), std::io::Error>(()) 24 | /// ``` 25 | /// 26 | /// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html 27 | pub struct WriteCompressor { 28 | inner: Option, 29 | comp: Compressor, 30 | } 31 | 32 | impl WriteCompressor { 33 | /// Creates a new `WriteCompressor`. 34 | pub fn new(writer: W, prefs: Preferences) -> Result { 35 | Ok(Self { 36 | inner: Some(writer), 37 | comp: Compressor::new(prefs, None)?, 38 | }) 39 | } 40 | 41 | /// Creates a new `WriteCompressor` with a dictionary. 42 | pub fn with_dict(writer: W, prefs: Preferences, dict: Dictionary) -> Result { 43 | Ok(Self { 44 | inner: Some(writer), 45 | comp: Compressor::new(prefs, Some(dict))?, 46 | }) 47 | } 48 | 49 | /// Returns a mutable reference to the writer. 50 | pub fn get_mut(&mut self) -> &mut W { 51 | self.inner.as_mut().unwrap() 52 | } 53 | 54 | /// Returns a shared reference to the writer. 55 | pub fn get_ref(&self) -> &W { 56 | self.inner.as_ref().unwrap() 57 | } 58 | 59 | /// Returns the ownership of the writer, finishing the stream in the process. 60 | pub fn into_inner(mut self) -> W { 61 | let _ = self.end(); 62 | self.inner.take().unwrap() 63 | } 64 | 65 | fn end(&mut self) -> std::io::Result<()> { 66 | if let Some(device) = &mut self.inner { 67 | self.comp.end(false)?; 68 | device.write_all(self.comp.buf())?; 69 | self.comp.clear_buf(); 70 | device.flush()?; 71 | } 72 | 73 | Ok(()) 74 | } 75 | } 76 | 77 | impl fmt::Debug for WriteCompressor 78 | where 79 | W: Write + fmt::Debug, 80 | { 81 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 82 | fmt.debug_struct("WriteCompressor") 83 | .field("writer", &self.inner) 84 | .field("prefs", &self.comp.prefs()) 85 | .finish() 86 | } 87 | } 88 | 89 | impl Write for WriteCompressor { 90 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 91 | self.comp.update(buf, false)?; 92 | self.inner.as_mut().unwrap().write_all(self.comp.buf())?; 93 | self.comp.clear_buf(); 94 | Ok(buf.len()) 95 | } 96 | 97 | fn flush(&mut self) -> std::io::Result<()> { 98 | self.comp.flush(false)?; 99 | self.inner.as_mut().unwrap().write_all(self.comp.buf())?; 100 | self.comp.clear_buf(); 101 | self.inner.as_mut().unwrap().flush() 102 | } 103 | } 104 | 105 | impl Drop for WriteCompressor { 106 | fn drop(&mut self) { 107 | let _ = self.end(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/lz4f/stream/decomp/bufread.rs: -------------------------------------------------------------------------------- 1 | use super::Decompressor; 2 | use crate::lz4f::{FrameInfo, Result}; 3 | use std::{ 4 | borrow::Cow, 5 | fmt, 6 | io::{BufRead, Read}, 7 | }; 8 | 9 | /// The [`BufRead`]-based streaming decompressor. 10 | /// 11 | /// # Example 12 | /// 13 | /// ``` 14 | /// # use std::env; 15 | /// # use std::path::Path; 16 | /// # use lzzzz::{Error, Result}; 17 | /// # use assert_fs::prelude::*; 18 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 19 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 20 | /// # 21 | /// # let mut buf = Vec::new(); 22 | /// # lzzzz::lz4f::compress_to_vec(b"Hello world!", &mut buf, &Default::default())?; 23 | /// # tmp_dir.child("foo.lz4").write_binary(&buf).unwrap(); 24 | /// # 25 | /// use lzzzz::lz4f::BufReadDecompressor; 26 | /// use std::{ 27 | /// fs::File, 28 | /// io::{prelude::*, BufReader}, 29 | /// }; 30 | /// 31 | /// let mut f = File::open("foo.lz4")?; 32 | /// let mut b = BufReader::new(f); 33 | /// let mut r = BufReadDecompressor::new(&mut b)?; 34 | /// 35 | /// let mut buf = Vec::new(); 36 | /// r.read_to_end(&mut buf)?; 37 | /// # Ok::<(), std::io::Error>(()) 38 | /// ``` 39 | /// 40 | /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html 41 | pub struct BufReadDecompressor<'a, R: BufRead> { 42 | pub(super) inner: R, 43 | decomp: Decompressor<'a>, 44 | consumed: usize, 45 | } 46 | 47 | impl<'a, R: BufRead> BufReadDecompressor<'a, R> { 48 | /// Creates a new `BufReadDecompressor`. 49 | pub fn new(reader: R) -> Result { 50 | Ok(Self { 51 | inner: reader, 52 | decomp: Decompressor::new()?, 53 | consumed: 0, 54 | }) 55 | } 56 | 57 | /// Sets the dictionary. 58 | pub fn set_dict(&mut self, dict: D) 59 | where 60 | D: Into>, 61 | { 62 | self.decomp.set_dict(dict); 63 | } 64 | 65 | /// Reads the frame header and returns `FrameInfo`. 66 | /// 67 | /// Calling this function before any `Read` or `BufRead` operations 68 | /// does not consume the frame body. 69 | pub fn read_frame_info(&mut self) -> std::io::Result { 70 | loop { 71 | if let Some(frame) = self.decomp.frame_info() { 72 | return Ok(frame); 73 | } 74 | self.decomp.decode_header_only(true); 75 | let _ = self.read(&mut [])?; 76 | self.decomp.decode_header_only(false); 77 | } 78 | } 79 | 80 | /// Returns ownership of the reader. 81 | pub fn into_inner(self) -> R { 82 | self.inner 83 | } 84 | 85 | /// Returns a mutable reference to the reader. 86 | pub fn get_mut(&mut self) -> &mut R { 87 | &mut self.inner 88 | } 89 | 90 | /// Returns a shared reference to the reader. 91 | pub fn get_ref(&self) -> &R { 92 | &self.inner 93 | } 94 | } 95 | 96 | impl fmt::Debug for BufReadDecompressor<'_, R> 97 | where 98 | R: BufRead + fmt::Debug, 99 | { 100 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 101 | fmt.debug_struct("BufReadDecompressor") 102 | .field("reader", &self.inner) 103 | .finish() 104 | } 105 | } 106 | 107 | impl Read for BufReadDecompressor<'_, R> { 108 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 109 | loop { 110 | let inner_buf = self.inner.fill_buf()?; 111 | let consumed = self.decomp.decompress(inner_buf)?; 112 | self.inner.consume(consumed); 113 | if consumed == 0 { 114 | break; 115 | } 116 | } 117 | 118 | let len = std::cmp::min(buf.len(), self.decomp.buf().len() - self.consumed); 119 | buf[..len].copy_from_slice(&self.decomp.buf()[self.consumed..][..len]); 120 | self.consumed += len; 121 | if self.consumed >= self.decomp.buf().len() { 122 | self.decomp.clear_buf(); 123 | self.consumed = 0; 124 | } 125 | Ok(len) 126 | } 127 | } 128 | 129 | impl BufRead for BufReadDecompressor<'_, R> { 130 | fn fill_buf(&mut self) -> std::io::Result<&[u8]> { 131 | let _ = self.read(&mut [])?; 132 | Ok(&self.decomp.buf()[self.consumed..]) 133 | } 134 | 135 | fn consume(&mut self, amt: usize) { 136 | self.consumed += amt; 137 | if self.consumed >= self.decomp.buf().len() { 138 | self.decomp.clear_buf(); 139 | self.consumed = 0; 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/lz4f/stream/decomp/mod.rs: -------------------------------------------------------------------------------- 1 | //! Streaming LZ4F decompressors. 2 | mod bufread; 3 | mod read; 4 | mod write; 5 | 6 | pub use bufread::*; 7 | pub use read::*; 8 | pub use write::*; 9 | 10 | use crate::{ 11 | common::DEFAULT_BUF_SIZE, 12 | lz4f::{ 13 | api::{ 14 | header_size, DecompressionContext, LZ4F_HEADER_SIZE_MAX, 15 | LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, 16 | }, 17 | FrameInfo, Result, 18 | }, 19 | Error, ErrorKind, 20 | }; 21 | use std::{borrow::Cow, cmp, pin::Pin, ptr}; 22 | 23 | #[derive(Clone, Copy, PartialEq)] 24 | struct DictPtr(*const u8, usize); 25 | 26 | #[allow(unsafe_code)] 27 | unsafe impl Send for DictPtr {} 28 | 29 | enum State { 30 | Header { 31 | header: [u8; LZ4F_HEADER_SIZE_MAX], 32 | header_len: usize, 33 | }, 34 | Body { 35 | frame_info: FrameInfo, 36 | comp_dict: Option, 37 | }, 38 | } 39 | 40 | pub(crate) struct Decompressor<'a> { 41 | ctx: DecompressionContext, 42 | state: State, 43 | buffer: Vec, 44 | dict: Pin>, 45 | header_only: bool, 46 | } 47 | 48 | impl<'a> Decompressor<'a> { 49 | pub fn new() -> Result { 50 | Ok(Self { 51 | ctx: DecompressionContext::new()?, 52 | state: State::Header { 53 | header: [0; LZ4F_HEADER_SIZE_MAX], 54 | header_len: 0, 55 | }, 56 | buffer: Vec::new(), 57 | dict: Pin::new(Cow::Borrowed(&[])), 58 | header_only: false, 59 | }) 60 | } 61 | 62 | pub fn set_dict(&mut self, dict: D) 63 | where 64 | D: Into>, 65 | { 66 | self.dict = Pin::new(dict.into()); 67 | } 68 | 69 | pub fn frame_info(&self) -> Option { 70 | if let State::Body { frame_info, .. } = self.state { 71 | Some(frame_info) 72 | } else { 73 | None 74 | } 75 | } 76 | 77 | pub fn decode_header_only(&mut self, flag: bool) { 78 | self.header_only = flag; 79 | } 80 | 81 | pub fn decompress(&mut self, src: &[u8]) -> Result { 82 | let mut header_consumed = 0; 83 | if let State::Header { 84 | ref mut header, 85 | ref mut header_len, 86 | } = &mut self.state 87 | { 88 | if *header_len < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH { 89 | let len = cmp::min(LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH - *header_len, src.len()); 90 | header[*header_len..*header_len + len].copy_from_slice(&src[..len]); 91 | *header_len += len; 92 | header_consumed += len; 93 | } 94 | if *header_len >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH { 95 | let exact_header_len = header_size(&header[..*header_len]); 96 | if exact_header_len > LZ4F_HEADER_SIZE_MAX { 97 | return Err(Error::new(ErrorKind::FrameHeaderInvalid).into()); 98 | } 99 | let src = &src[header_consumed..]; 100 | if *header_len < exact_header_len { 101 | let len = cmp::min(exact_header_len - *header_len, src.len()); 102 | header[*header_len..*header_len + len].copy_from_slice(&src[..len]); 103 | *header_len += len; 104 | header_consumed += len; 105 | } 106 | if *header_len >= exact_header_len { 107 | let (frame, rep) = self.ctx.get_frame_info(&header[..*header_len])?; 108 | header_consumed = cmp::min(header_consumed, rep); 109 | 110 | self.state = State::Body { 111 | frame_info: frame, 112 | comp_dict: None, 113 | } 114 | } 115 | } 116 | } 117 | 118 | if let State::Header { header, header_len } = self.state { 119 | if src.is_empty() { 120 | self.ctx.get_frame_info(&header[..header_len])?; 121 | } 122 | } 123 | 124 | if self.header_only { 125 | return Ok(header_consumed); 126 | } 127 | 128 | let src = &src[header_consumed..]; 129 | let dict_ptr = self.dict_ptr(); 130 | if let State::Body { 131 | ref mut comp_dict, .. 132 | } = &mut self.state 133 | { 134 | if dict_ptr != *comp_dict.get_or_insert(dict_ptr) { 135 | return Err(Error::new(ErrorKind::DictionaryChangedDuringDecompression).into()); 136 | } 137 | 138 | let len = self.buffer.len(); 139 | if len < DEFAULT_BUF_SIZE { 140 | self.buffer.resize_with(DEFAULT_BUF_SIZE, Default::default) 141 | } 142 | let (src_len, dst_len, _) = 143 | self.ctx 144 | .decompress_dict(src, &mut self.buffer[len..], &self.dict, false)?; 145 | self.buffer.resize_with(len + dst_len, Default::default); 146 | Ok(src_len + header_consumed) 147 | } else { 148 | Ok(header_consumed) 149 | } 150 | } 151 | 152 | fn dict_ptr(&self) -> DictPtr { 153 | let dict = &self.dict; 154 | if dict.is_empty() { 155 | DictPtr(ptr::null(), 0) 156 | } else { 157 | DictPtr(dict.as_ptr(), dict.len()) 158 | } 159 | } 160 | 161 | pub fn buf(&self) -> &[u8] { 162 | &self.buffer 163 | } 164 | 165 | pub fn clear_buf(&mut self) { 166 | self.buffer.clear(); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/lz4f/stream/decomp/read.rs: -------------------------------------------------------------------------------- 1 | use super::BufReadDecompressor; 2 | use crate::lz4f::{FrameInfo, Result}; 3 | use std::{ 4 | borrow::Cow, 5 | fmt, 6 | io::{BufReader, Read}, 7 | }; 8 | 9 | /// The [`Read`]-based streaming decompressor. 10 | /// 11 | /// # Example 12 | /// 13 | /// ``` 14 | /// # use std::env; 15 | /// # use std::path::Path; 16 | /// # use lzzzz::{Error, Result}; 17 | /// # use assert_fs::prelude::*; 18 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 19 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 20 | /// # 21 | /// # let mut buf = Vec::new(); 22 | /// # lzzzz::lz4f::compress_to_vec(b"Hello world!", &mut buf, &Default::default())?; 23 | /// # tmp_dir.child("foo.lz4").write_binary(&buf).unwrap(); 24 | /// # 25 | /// use lzzzz::lz4f::ReadDecompressor; 26 | /// use std::{fs::File, io::prelude::*}; 27 | /// 28 | /// let mut f = File::open("foo.lz4")?; 29 | /// let mut r = ReadDecompressor::new(&mut f)?; 30 | /// 31 | /// let mut buf = Vec::new(); 32 | /// r.read_to_end(&mut buf)?; 33 | /// # Ok::<(), std::io::Error>(()) 34 | /// ``` 35 | /// 36 | /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html 37 | pub struct ReadDecompressor<'a, R: Read> { 38 | inner: BufReadDecompressor<'a, BufReader>, 39 | } 40 | 41 | impl fmt::Debug for ReadDecompressor<'_, R> 42 | where 43 | R: Read + fmt::Debug, 44 | { 45 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 46 | fmt.debug_struct("ReadDecompressor") 47 | .field("reader", &self.inner.inner.get_ref()) 48 | .finish() 49 | } 50 | } 51 | 52 | impl<'a, R: Read> ReadDecompressor<'a, R> { 53 | /// Creates a new `ReadDecompressor`. 54 | pub fn new(reader: R) -> Result { 55 | Ok(Self { 56 | inner: BufReadDecompressor::new(BufReader::new(reader))?, 57 | }) 58 | } 59 | 60 | /// Sets the dictionary. 61 | pub fn set_dict(&mut self, dict: D) 62 | where 63 | D: Into>, 64 | { 65 | self.inner.set_dict(dict); 66 | } 67 | 68 | /// Reads the frame header and returns `FrameInfo`. 69 | /// 70 | /// Calling this function before any `Read` operations 71 | /// does not consume the frame body. 72 | pub fn read_frame_info(&mut self) -> std::io::Result { 73 | self.inner.read_frame_info() 74 | } 75 | 76 | /// Returns ownership of the reader. 77 | pub fn into_inner(self) -> R { 78 | self.inner.into_inner().into_inner() 79 | } 80 | 81 | /// Returns a mutable reference to the reader. 82 | pub fn get_mut(&mut self) -> &mut R { 83 | self.inner.get_mut().get_mut() 84 | } 85 | 86 | /// Returns a shared reference to the reader. 87 | pub fn get_ref(&self) -> &R { 88 | self.inner.get_ref().get_ref() 89 | } 90 | } 91 | 92 | impl Read for ReadDecompressor<'_, R> { 93 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 94 | self.inner.read(buf) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/lz4f/stream/decomp/write.rs: -------------------------------------------------------------------------------- 1 | use crate::lz4f::{Decompressor, FrameInfo, Result}; 2 | use std::{borrow::Cow, fmt, io::Write}; 3 | 4 | /// The [`Write`]-based streaming decompressor. 5 | /// 6 | /// # Example 7 | /// 8 | /// ``` 9 | /// # use std::env; 10 | /// # use std::path::Path; 11 | /// # use lzzzz::{Error, Result}; 12 | /// # use assert_fs::prelude::*; 13 | /// # let tmp_dir = assert_fs::TempDir::new().unwrap().into_persistent(); 14 | /// # env::set_current_dir(tmp_dir.path()).unwrap(); 15 | /// use lzzzz::lz4f::{compress_to_vec, WriteDecompressor}; 16 | /// use std::{fs::File, io::prelude::*}; 17 | /// 18 | /// let mut f = File::create("foo.txt")?; 19 | /// let mut w = WriteDecompressor::new(&mut f)?; 20 | /// 21 | /// let mut buf = Vec::new(); 22 | /// compress_to_vec(b"Hello world!", &mut buf, &Default::default())?; 23 | /// 24 | /// w.write_all(&buf)?; 25 | /// # Ok::<(), std::io::Error>(()) 26 | /// ``` 27 | /// 28 | /// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html 29 | pub struct WriteDecompressor<'a, W: Write> { 30 | inner: W, 31 | decomp: Decompressor<'a>, 32 | } 33 | 34 | impl<'a, W: Write> WriteDecompressor<'a, W> { 35 | /// Creates a new `WriteDecompressor`. 36 | pub fn new(writer: W) -> Result { 37 | Ok(Self { 38 | inner: writer, 39 | decomp: Decompressor::new()?, 40 | }) 41 | } 42 | 43 | /// Sets the dictionary. 44 | pub fn set_dict(&mut self, dict: D) 45 | where 46 | D: Into>, 47 | { 48 | self.decomp.set_dict(dict); 49 | } 50 | 51 | /// Returns `FrameInfo` if the frame header is already decoded. 52 | /// Otherwise, returns `None`. 53 | pub fn frame_info(&self) -> Option { 54 | self.decomp.frame_info() 55 | } 56 | 57 | /// Sets the 'header-only' mode. 58 | /// 59 | /// When the 'header-only' mode is enabled, the decompressor doesn't 60 | /// consume the frame body and `write()` always returns `Ok(0)` 61 | /// if the frame header is already decoded. 62 | pub fn decode_header_only(&mut self, flag: bool) { 63 | self.decomp.decode_header_only(flag); 64 | } 65 | 66 | /// Returns a mutable reference to the writer. 67 | pub fn get_mut(&mut self) -> &mut W { 68 | &mut self.inner 69 | } 70 | 71 | /// Returns a shared reference to the writer. 72 | pub fn get_ref(&self) -> &W { 73 | &self.inner 74 | } 75 | 76 | /// Returns ownership of the writer. 77 | pub fn into_inner(self) -> W { 78 | self.inner 79 | } 80 | } 81 | 82 | impl fmt::Debug for WriteDecompressor<'_, W> 83 | where 84 | W: Write + fmt::Debug, 85 | { 86 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 87 | fmt.debug_struct("WriteDecompressor") 88 | .field("writer", &self.inner) 89 | .finish() 90 | } 91 | } 92 | 93 | impl Write for WriteDecompressor<'_, W> { 94 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 95 | let report = self.decomp.decompress(buf)?; 96 | self.inner.write_all(self.decomp.buf())?; 97 | self.decomp.clear_buf(); 98 | Ok(report) 99 | } 100 | 101 | fn flush(&mut self) -> std::io::Result<()> { 102 | self.inner.flush() 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/lz4f/stream/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod comp; 2 | pub mod decomp; 3 | -------------------------------------------------------------------------------- /tests/common/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use bytes::Bytes; 4 | use lazy_static::lazy_static; 5 | use lzzzz::{lz4, lz4_hc, lz4f::*}; 6 | use rand::{distributions::Standard, rngs::SmallRng, Rng, SeedableRng}; 7 | use std::{i32, u32}; 8 | 9 | lazy_static! { 10 | static ref DATA_SET: Vec = { 11 | (0..20) 12 | .map(|n| { 13 | let rng = SmallRng::seed_from_u64(n as u64); 14 | rng.sample_iter(Standard).take(16 << n).collect() 15 | }) 16 | .collect() 17 | }; 18 | } 19 | 20 | fn generate_data() -> impl Iterator { 21 | DATA_SET.clone().into_iter() 22 | } 23 | 24 | fn preferences_set() -> impl Iterator { 25 | vec![ 26 | PreferencesBuilder::new().build(), 27 | PreferencesBuilder::new() 28 | .block_size(BlockSize::Max64KB) 29 | .build(), 30 | PreferencesBuilder::new() 31 | .block_size(BlockSize::Max256KB) 32 | .build(), 33 | PreferencesBuilder::new() 34 | .block_size(BlockSize::Max1MB) 35 | .build(), 36 | PreferencesBuilder::new() 37 | .block_size(BlockSize::Max4MB) 38 | .build(), 39 | PreferencesBuilder::new() 40 | .block_mode(BlockMode::Independent) 41 | .build(), 42 | PreferencesBuilder::new() 43 | .content_checksum(ContentChecksum::Enabled) 44 | .build(), 45 | PreferencesBuilder::new().dict_id(u32::MAX).build(), 46 | PreferencesBuilder::new() 47 | .block_checksum(BlockChecksum::Enabled) 48 | .build(), 49 | PreferencesBuilder::new() 50 | .compression_level(CLEVEL_HIGH) 51 | .build(), 52 | PreferencesBuilder::new() 53 | .compression_level(CLEVEL_MAX) 54 | .build(), 55 | PreferencesBuilder::new() 56 | .compression_level(i32::MAX) 57 | .build(), 58 | PreferencesBuilder::new() 59 | .compression_level(i32::MIN) 60 | .build(), 61 | PreferencesBuilder::new() 62 | .favor_dec_speed(FavorDecSpeed::Enabled) 63 | .build(), 64 | PreferencesBuilder::new() 65 | .auto_flush(AutoFlush::Enabled) 66 | .build(), 67 | ] 68 | .into_iter() 69 | } 70 | 71 | pub fn lz4f_test_set() -> impl Iterator { 72 | generate_data() 73 | .flat_map(|data| preferences_set().map(move |prefs| (data.clone(), prefs))) 74 | } 75 | 76 | fn compression_acc_set() -> impl Iterator { 77 | vec![lz4::ACC_LEVEL_DEFAULT, 0, i32::MIN, i32::MAX].into_iter() 78 | } 79 | 80 | pub fn lz4_test_set() -> impl Iterator { 81 | generate_data() 82 | .flat_map(|data| compression_acc_set().map(move |acc| (data.clone(), acc))) 83 | } 84 | 85 | pub fn lz4_stream_test_set() -> impl Iterator, i32)> { 86 | compression_acc_set().map(|acc| (generate_data().collect(), acc)) 87 | } 88 | 89 | fn compression_level_set() -> impl Iterator { 90 | vec![ 91 | lz4_hc::CLEVEL_DEFAULT, 92 | lz4_hc::CLEVEL_MIN, 93 | lz4_hc::CLEVEL_OPT_MIN, 94 | lz4_hc::CLEVEL_MAX, 95 | i32::MIN, 96 | i32::MAX, 97 | ] 98 | .into_iter() 99 | } 100 | 101 | pub fn lz4_hc_test_set() -> impl Iterator { 102 | generate_data() 103 | .flat_map(|data| compression_level_set().map(move |level| (data.clone(), level))) 104 | } 105 | 106 | pub fn lz4_hc_stream_test_set() -> impl Iterator, i32)> { 107 | compression_level_set().map(|level| (generate_data().collect(), level)) 108 | } 109 | -------------------------------------------------------------------------------- /tests/lz4.rs: -------------------------------------------------------------------------------- 1 | use lzzzz::lz4; 2 | use rayon::{iter::ParallelBridge, prelude::*}; 3 | use std::cmp; 4 | 5 | mod common; 6 | use common::lz4_test_set; 7 | 8 | mod compress { 9 | use super::*; 10 | 11 | #[test] 12 | fn default() { 13 | lz4_test_set().par_bridge().for_each(|(src, mode)| { 14 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 15 | let mut decomp_buf = vec![0; src.len()]; 16 | let len = lz4::compress(&src, &mut comp_buf, mode).unwrap(); 17 | lz4::decompress(&comp_buf[..len], &mut decomp_buf).unwrap(); 18 | assert_eq!(decomp_buf, src); 19 | }); 20 | } 21 | 22 | #[test] 23 | fn fill() { 24 | lz4_test_set() 25 | .flat_map(|(src, mode)| (0..20).map(move |n| (src.clone(), mode, 16 << n))) 26 | .par_bridge() 27 | .for_each(|(src, _mode, len)| { 28 | let mut comp_buf = vec![0; len]; 29 | let mut decomp_buf = Vec::new(); 30 | let (read, wrote) = lz4::compress_fill(&src, &mut comp_buf).unwrap(); 31 | decomp_buf.resize(read, 0); 32 | let len = lz4::decompress(&comp_buf[..wrote], &mut decomp_buf).unwrap(); 33 | assert_eq!(len, read); 34 | assert!(src.starts_with(&decomp_buf)); 35 | }); 36 | } 37 | } 38 | 39 | mod compress_to_vec { 40 | use super::*; 41 | 42 | #[test] 43 | fn default() { 44 | lz4_test_set().par_bridge().for_each(|(src, mode)| { 45 | let header = &b"HEADER"[..]; 46 | let mut comp_buf = Vec::from(header); 47 | let mut decomp_buf = vec![0; src.len()]; 48 | lz4::compress_to_vec(&src, &mut comp_buf, mode).unwrap(); 49 | assert!(comp_buf.starts_with(header)); 50 | lz4::decompress(&comp_buf[header.len()..], &mut decomp_buf).unwrap(); 51 | assert_eq!(decomp_buf, src); 52 | }); 53 | } 54 | } 55 | 56 | mod decompress { 57 | use super::*; 58 | 59 | #[test] 60 | fn default() { 61 | lz4_test_set().par_bridge().for_each(|(src, mode)| { 62 | let mut comp_buf = Vec::new(); 63 | let mut decomp_buf = vec![0; src.len()]; 64 | lz4::compress_to_vec(&src, &mut comp_buf, mode).unwrap(); 65 | lz4::decompress(&comp_buf, &mut decomp_buf).unwrap(); 66 | assert_eq!(src, &decomp_buf); 67 | }); 68 | } 69 | 70 | #[test] 71 | fn partial() { 72 | lz4_test_set() 73 | .flat_map(|(src, mode)| (0..20).map(move |n| (src.clone(), mode, 16 << n))) 74 | .par_bridge() 75 | .for_each(|(src, mode, len)| { 76 | let mut comp_buf = Vec::new(); 77 | let mut decomp_buf = vec![0; cmp::min(src.len(), len)]; 78 | lz4::compress_to_vec(&src, &mut comp_buf, mode).unwrap(); 79 | lz4::decompress_partial(&comp_buf, &mut decomp_buf, src.len()).unwrap(); 80 | assert!(src.starts_with(&decomp_buf)); 81 | }); 82 | } 83 | 84 | #[test] 85 | fn with_dict_and_dict_slow() { 86 | lz4_test_set().par_bridge().for_each(|(src, mode)| { 87 | for with_dict in [lz4::Compressor::with_dict, lz4::Compressor::with_dict_slow] { 88 | let mut comp_buf = Vec::new(); 89 | let mut decomp_buf = vec![0; src.len()]; 90 | let mut comp = with_dict(src.as_ref()).unwrap(); 91 | comp.next_to_vec(&src, &mut comp_buf, mode).unwrap(); 92 | lz4::decompress_with_dict(&comp_buf, &mut decomp_buf, &src).unwrap(); 93 | assert_eq!(src, &decomp_buf); 94 | } 95 | }); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /tests/lz4_hc.rs: -------------------------------------------------------------------------------- 1 | use lzzzz::{lz4, lz4_hc}; 2 | use rayon::{iter::ParallelBridge, prelude::*}; 3 | 4 | mod common; 5 | use common::lz4_hc_test_set; 6 | 7 | mod compress { 8 | use super::*; 9 | 10 | #[test] 11 | fn default() { 12 | lz4_hc_test_set().par_bridge().for_each(|(src, level)| { 13 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 14 | let mut decomp_buf = vec![0; src.len()]; 15 | let len = lz4_hc::compress(&src, &mut comp_buf, level).unwrap(); 16 | lz4::decompress(&comp_buf[..len], &mut decomp_buf).unwrap(); 17 | assert_eq!(decomp_buf, src); 18 | }); 19 | } 20 | 21 | #[test] 22 | fn fill() { 23 | lz4_hc_test_set() 24 | .flat_map(|(src, level)| (0..20).map(move |n| (src.clone(), level, 16 << n))) 25 | .par_bridge() 26 | .for_each(|(src, level, len)| { 27 | let mut comp_buf = vec![0; len]; 28 | let mut decomp_buf = Vec::new(); 29 | let (read, wrote) = lz4_hc::compress_fill(&src, &mut comp_buf, level).unwrap(); 30 | decomp_buf.resize(read, 0); 31 | let len = lz4::decompress(&comp_buf[..wrote], &mut decomp_buf).unwrap(); 32 | assert_eq!(len, read); 33 | assert!(src.starts_with(&decomp_buf)); 34 | }); 35 | } 36 | } 37 | 38 | mod compress_fill { 39 | use super::*; 40 | 41 | #[test] 42 | fn default() { 43 | lz4_hc_test_set() 44 | .flat_map(|(src, level)| (0..20).map(move |n| (src.clone(), level, 16 << n))) 45 | .par_bridge() 46 | .for_each(|(src, level, len)| { 47 | let mut comp_buf = vec![0; len]; 48 | let mut decomp_buf = Vec::new(); 49 | let pos = src.len() / 2; 50 | let (src_len, dst_len) = 51 | lz4_hc::compress_fill(&src[pos..], &mut comp_buf, level).unwrap(); 52 | decomp_buf.resize(src_len, 0); 53 | lz4::decompress(&comp_buf[..dst_len], &mut decomp_buf).unwrap(); 54 | assert!(src[pos..].starts_with(&decomp_buf)); 55 | }); 56 | } 57 | } 58 | 59 | mod compress_to_vec { 60 | use super::*; 61 | 62 | #[test] 63 | fn default() { 64 | lz4_hc_test_set().par_bridge().for_each(|(src, level)| { 65 | let header = &b"HEADER"[..]; 66 | let mut comp_buf = Vec::from(header); 67 | let mut decomp_buf = vec![0; src.len()]; 68 | lz4_hc::compress_to_vec(&src, &mut comp_buf, level).unwrap(); 69 | assert!(comp_buf.starts_with(header)); 70 | lz4::decompress(&comp_buf[header.len()..], &mut decomp_buf).unwrap(); 71 | assert_eq!(decomp_buf, src); 72 | }); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /tests/lz4_hc_stream.rs: -------------------------------------------------------------------------------- 1 | use lzzzz::{lz4, lz4_hc}; 2 | use rand::{distributions::Standard, rngs::SmallRng, Rng, SeedableRng}; 3 | use rayon::{iter::ParallelBridge, prelude::*}; 4 | use static_assertions::assert_impl_all; 5 | 6 | mod common; 7 | use common::lz4_hc_stream_test_set; 8 | 9 | assert_impl_all!(lz4_hc::Compressor: Send); 10 | 11 | mod compressor { 12 | use super::*; 13 | 14 | #[test] 15 | fn default() { 16 | lz4_hc_stream_test_set() 17 | .par_bridge() 18 | .for_each(|(src_set, level)| { 19 | let mut comp = lz4_hc::Compressor::new().unwrap(); 20 | let mut decomp = lz4::Decompressor::new().unwrap(); 21 | comp.set_compression_level(level); 22 | for src in src_set { 23 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 24 | let len = comp.next(&src, &mut comp_buf).unwrap(); 25 | assert_eq!(decomp.next(&comp_buf[..len], src.len()).unwrap(), &src); 26 | } 27 | }); 28 | } 29 | 30 | #[test] 31 | fn dictionary() { 32 | lz4_hc_stream_test_set() 33 | .par_bridge() 34 | .for_each(|(src_set, level)| { 35 | let dict = SmallRng::seed_from_u64(0) 36 | .sample_iter(Standard) 37 | .take(64 * 1024) 38 | .collect::>(); 39 | let mut comp = lz4_hc::Compressor::with_dict(&dict, lz4_hc::CLEVEL_DEFAULT).unwrap(); 40 | let mut decomp = lz4::Decompressor::with_dict(&dict).unwrap(); 41 | comp.set_compression_level(level); 42 | for src in src_set { 43 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 44 | let len = comp.next(&src, &mut comp_buf).unwrap(); 45 | assert_eq!(decomp.next(&comp_buf[..len], src.len()).unwrap(), &src); 46 | } 47 | }); 48 | } 49 | 50 | #[test] 51 | fn dynamic_adaptation() { 52 | lz4_hc_stream_test_set() 53 | .par_bridge() 54 | .for_each(|(src_set, _)| { 55 | let mut comp = lz4_hc::Compressor::new().unwrap(); 56 | let mut decomp = lz4::Decompressor::new().unwrap(); 57 | let mut rng = SmallRng::seed_from_u64(0); 58 | for src in src_set { 59 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 60 | 61 | comp.set_compression_level(rng.gen()); 62 | comp.set_favor_dec_speed(if rng.gen_bool(0.5) { 63 | lz4_hc::FavorDecSpeed::Enabled 64 | } else { 65 | lz4_hc::FavorDecSpeed::Disabled 66 | }); 67 | 68 | let len = comp.next(&src, &mut comp_buf).unwrap(); 69 | assert_eq!(decomp.next(&comp_buf[..len], src.len()).unwrap(), &src); 70 | } 71 | }); 72 | } 73 | 74 | #[test] 75 | fn attach_dictionary() { 76 | // Basic test data 77 | let data = b"The quick brown fox jumps over the lazy dog"; 78 | let level = lz4_hc::CLEVEL_DEFAULT; 79 | 80 | // Create dictionary stream and set its compression level 81 | let mut dict_comp = lz4_hc::Compressor::with_dict(data, level).unwrap(); 82 | 83 | // Create working stream 84 | let mut comp = lz4_hc::Compressor::new().unwrap(); 85 | comp.set_compression_level(level); // Set level before attachment 86 | 87 | // Compress with attached dictionary 88 | comp.attach_dict(Some(&mut dict_comp), level); 89 | let mut output_attached_dict = Vec::new(); 90 | comp.next_to_vec(data, &mut output_attached_dict).unwrap(); 91 | 92 | // Compress with no dictionary 93 | comp.attach_dict(None, level); 94 | let mut output_no_dict = Vec::new(); 95 | comp.next_to_vec(data, &mut output_no_dict).unwrap(); 96 | 97 | // Results should match 98 | assert_ne!(output_attached_dict, output_no_dict, "Data with no dict should be different"); 99 | 100 | // Code below is disabled because it (unexpectedly) does not work. 101 | // Seems to be an upstream lz4 issue. 102 | 103 | // Create regular dictionary compressor with same level 104 | // let mut comp_regular_dict = lz4_hc::Compressor::with_dict(data, level).unwrap(); 105 | // let mut output_regular_dict = Vec::new(); 106 | // comp_regular_dict.next_to_vec(data, &mut output_regular_dict).unwrap(); 107 | // assert_eq!(output_attached_dict, output_regular_dict, "Compressed data should match"); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /tests/lz4_stream.rs: -------------------------------------------------------------------------------- 1 | use lzzzz::lz4; 2 | use rand::{distributions::Standard, rngs::SmallRng, Rng, SeedableRng}; 3 | use rayon::{iter::ParallelBridge, prelude::*}; 4 | use static_assertions::assert_impl_all; 5 | 6 | mod common; 7 | use common::lz4_stream_test_set; 8 | 9 | assert_impl_all!(lz4::Compressor: Send); 10 | assert_impl_all!(lz4::Decompressor: Send); 11 | 12 | mod compressor { 13 | use super::*; 14 | 15 | #[test] 16 | fn default() { 17 | lz4_stream_test_set() 18 | .par_bridge() 19 | .for_each(|(src_set, mode)| { 20 | let mut comp = lz4::Compressor::new().unwrap(); 21 | let mut decomp = lz4::Decompressor::new().unwrap(); 22 | for src in src_set { 23 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 24 | let len = comp.next(&src, &mut comp_buf, mode).unwrap(); 25 | assert_eq!(decomp.next(&comp_buf[..len], src.len()).unwrap(), &src); 26 | } 27 | }); 28 | } 29 | 30 | #[test] 31 | fn dictionary() { 32 | run_dictionary_test(|dict| lz4::Compressor::with_dict(dict)) 33 | } 34 | 35 | #[test] 36 | fn dictionary_slow() { 37 | run_dictionary_test(|dict| lz4::Compressor::with_dict_slow(dict)) 38 | } 39 | 40 | /// Helper function to run dictionary compression tests with either normal or slow mode 41 | fn run_dictionary_test(compressor_factory: F) 42 | where 43 | F: Fn(&[u8]) -> Result + Sync, 44 | { 45 | lz4_stream_test_set() 46 | .par_bridge() 47 | .for_each(|(src_set, mode)| { 48 | let dict = SmallRng::seed_from_u64(0) 49 | .sample_iter(Standard) 50 | .take(64 * 1024) 51 | .collect::>(); 52 | 53 | let mut comp = compressor_factory(&dict).unwrap(); 54 | let mut decomp = lz4::Decompressor::with_dict(&dict).unwrap(); 55 | 56 | for src in src_set { 57 | let mut comp_buf = vec![0; lz4::max_compressed_size(src.len())]; 58 | let len = comp.next(&src, &mut comp_buf, mode).unwrap(); 59 | assert_eq!(decomp.next(&comp_buf[..len], src.len()).unwrap(), &src); 60 | } 61 | }); 62 | } 63 | 64 | #[test] 65 | fn attach_dictionary() { 66 | // Basic test data 67 | let data = b"The quick brown fox jumps over the lazy dog"; 68 | 69 | // Create dictionary stream 70 | let mut dict_comp = lz4::Compressor::with_dict(data).unwrap(); 71 | 72 | // Create working stream and attach dictionary 73 | let mut comp = lz4::Compressor::new().unwrap(); 74 | 75 | // Compress with attached dictionary 76 | comp.attach_dict(Some(&mut dict_comp)); 77 | let mut output_attached_dict = Vec::new(); 78 | comp.next_to_vec(data, &mut output_attached_dict, lz4::ACC_LEVEL_DEFAULT).unwrap(); 79 | 80 | // Compress with no dictionary 81 | comp.attach_dict(None); 82 | let mut output_no_dict = Vec::new(); 83 | comp.next_to_vec(data, &mut output_no_dict, lz4::ACC_LEVEL_DEFAULT).unwrap(); 84 | 85 | // Verify the data compresses to same result as regular dictionary compression 86 | let mut comp_regular_dict = lz4::Compressor::with_dict(data).unwrap(); 87 | let mut output_regular_dict = Vec::new(); 88 | comp_regular_dict.next_to_vec(data, &mut output_regular_dict, lz4::ACC_LEVEL_DEFAULT).unwrap(); 89 | 90 | // Results should match 91 | assert_eq!(output_attached_dict, output_regular_dict, "Compressed data should match"); 92 | assert_ne!(output_attached_dict, output_no_dict, "Data with no dict should be different"); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /tests/lz4f.rs: -------------------------------------------------------------------------------- 1 | use lzzzz::{lz4f, lz4f::*}; 2 | use rayon::{iter::ParallelBridge, prelude::*}; 3 | 4 | mod common; 5 | use common::lz4f_test_set; 6 | 7 | mod compress_to_vec { 8 | use super::*; 9 | 10 | #[test] 11 | fn default() { 12 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 13 | let header = &b"HEADER"[..]; 14 | let mut comp_buf = Vec::from(header); 15 | let mut decomp_buf = Vec::from(header); 16 | 17 | assert_eq!( 18 | lz4f::compress_to_vec(&src, &mut comp_buf, &prefs).unwrap(), 19 | comp_buf.len() - header.len() 20 | ); 21 | assert_eq!( 22 | lz4f::decompress_to_vec(&comp_buf[header.len()..], &mut decomp_buf).unwrap(), 23 | decomp_buf.len() - header.len() 24 | ); 25 | assert_eq!(&decomp_buf[header.len()..], &src[..]); 26 | }); 27 | } 28 | } 29 | 30 | mod compress { 31 | use super::*; 32 | 33 | #[test] 34 | fn default() { 35 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 36 | let mut comp_buf = vec![0; lz4f::max_compressed_size(src.len(), &prefs)]; 37 | let mut decomp_buf = Vec::new(); 38 | 39 | let len = lz4f::compress(&src, &mut comp_buf, &prefs).unwrap(); 40 | comp_buf.resize_with(len, Default::default); 41 | assert_eq!( 42 | lz4f::decompress_to_vec(&comp_buf, &mut decomp_buf).unwrap(), 43 | decomp_buf.len() 44 | ); 45 | assert_eq!(decomp_buf, src); 46 | }); 47 | } 48 | 49 | #[test] 50 | fn too_small_dst() { 51 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 52 | let mut comp_buf = Vec::new(); 53 | assert_eq!( 54 | lz4f::compress(&src, &mut comp_buf, &prefs), 55 | Err(Error::Lz4f(ErrorKind::DstMaxSizeTooSmall)) 56 | ); 57 | }); 58 | } 59 | 60 | #[test] 61 | fn content_size() { 62 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 63 | let prefs = PreferencesBuilder::from(prefs).content_size(1).build(); 64 | let mut comp_buf = vec![0; lz4f::max_compressed_size(src.len(), &prefs)]; 65 | let mut decomp_buf = Vec::new(); 66 | 67 | let len = lz4f::compress(&src, &mut comp_buf, &prefs).unwrap(); 68 | comp_buf.resize_with(len, Default::default); 69 | assert_eq!( 70 | lz4f::decompress_to_vec(&comp_buf, &mut decomp_buf).unwrap(), 71 | decomp_buf.len() 72 | ); 73 | assert_eq!(decomp_buf, src); 74 | 75 | let mut comp_buf = comp_buf.as_slice(); 76 | let mut r = ReadDecompressor::new(&mut comp_buf).unwrap(); 77 | assert_eq!(r.read_frame_info().unwrap().content_size(), src.len()); 78 | }); 79 | } 80 | } 81 | 82 | mod decompress_to_vec { 83 | use super::*; 84 | 85 | #[test] 86 | fn invalid_header() { 87 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 88 | let header = &b"HEADER"[..]; 89 | let mut comp_buf = Vec::from(header); 90 | let mut decomp_buf = Vec::from(header); 91 | assert_eq!( 92 | lz4f::compress_to_vec(&src, &mut comp_buf, &prefs).unwrap(), 93 | comp_buf.len() - header.len() 94 | ); 95 | assert_eq!( 96 | lz4f::decompress_to_vec(&comp_buf[1..], &mut decomp_buf), 97 | Err(Error::Lz4f(ErrorKind::FrameTypeUnknown)) 98 | ); 99 | assert_eq!(decomp_buf, header); 100 | }); 101 | } 102 | 103 | #[test] 104 | fn incomplete_src() { 105 | lz4f_test_set().par_bridge().for_each(|(src, prefs)| { 106 | let header = &b"HEADER"[..]; 107 | let mut comp_buf = Vec::new(); 108 | let mut decomp_buf = Vec::from(header); 109 | assert_eq!( 110 | lz4f::compress_to_vec(&src, &mut comp_buf, &prefs).unwrap(), 111 | comp_buf.len() 112 | ); 113 | assert_eq!( 114 | lz4f::decompress_to_vec(&comp_buf[..comp_buf.len() - 1], &mut decomp_buf), 115 | Err(Error::Common(lzzzz::ErrorKind::CompressedDataIncomplete)) 116 | ); 117 | assert_eq!(decomp_buf, header); 118 | }); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /vendor/liblz4/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | liblz4.pc 3 | -------------------------------------------------------------------------------- /vendor/liblz4/LICENSE: -------------------------------------------------------------------------------- 1 | LZ4 Library 2 | Copyright (c) 2011-2020, Yann Collet 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | list of conditions and the following disclaimer in the documentation and/or 13 | other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /vendor/liblz4/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # LZ4 library - Makefile 3 | # Copyright (C) Yann Collet 2011-2023 4 | # All rights reserved. 5 | # 6 | # This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets 7 | # 8 | # BSD license 9 | # Redistribution and use in source and binary forms, with or without modification, 10 | # are permitted provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright notice, this 13 | # list of conditions and the following disclaimer. 14 | # 15 | # * Redistributions in binary form must reproduce the above copyright notice, this 16 | # list of conditions and the following disclaimer in the documentation and/or 17 | # other materials provided with the distribution. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 23 | # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 26 | # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | # 30 | # You can contact the author at : 31 | # - LZ4 source repository : https://github.com/lz4/lz4 32 | # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c 33 | # ################################################################ 34 | SED ?= sed 35 | 36 | # Version numbers 37 | LIBVER_MAJOR_SCRIPT:=`$(SED) -n '/define[[:blank:]][[:blank:]]*LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` 38 | LIBVER_MINOR_SCRIPT:=`$(SED) -n '/define[[:blank:]][[:blank:]]*LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` 39 | LIBVER_PATCH_SCRIPT:=`$(SED) -n '/define[[:blank:]][[:blank:]]*LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./lz4.h` 40 | LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) 41 | LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) 42 | LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) 43 | LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) 44 | LIBVER := $(shell echo $(LIBVER_SCRIPT)) 45 | 46 | BUILD_SHARED:=yes 47 | BUILD_STATIC:=yes 48 | 49 | CPPFLAGS+= -DXXH_NAMESPACE=LZ4_ 50 | USERCFLAGS:= -O3 $(CFLAGS) 51 | DEBUGFLAGS:= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 52 | -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \ 53 | -Wundef -Wpointer-arith -Wstrict-aliasing=1 54 | CFLAGS = $(DEBUGFLAGS) $(USERCFLAGS) 55 | ALLFLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 56 | 57 | SRCFILES := $(sort $(wildcard *.c)) 58 | 59 | include ../Makefile.inc 60 | 61 | # OS X linker doesn't support -soname, and use different extension 62 | # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html 63 | ifeq ($(TARGET_OS), Darwin) 64 | SHARED_EXT = dylib 65 | SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) 66 | SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) 67 | SONAME_FLAGS = -install_name $(libdir)/liblz4.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) -dynamiclib 68 | else 69 | ifeq ($(WINBASED),yes) 70 | SHARED_EXT = dll 71 | SHARED_EXT_VER = $(SHARED_EXT) 72 | else # posix non-mac 73 | SONAME_FLAGS = -Wl,-soname=liblz4.$(SHARED_EXT).$(LIBVER_MAJOR) 74 | SHARED_EXT = so 75 | SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) 76 | SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) 77 | endif 78 | endif 79 | 80 | .PHONY: default 81 | default: lib-release 82 | 83 | # silent mode by default; verbose can be triggered by V=1 or VERBOSE=1 84 | $(V)$(VERBOSE).SILENT: 85 | 86 | .PHONY:lib-release 87 | lib-release: DEBUGFLAGS := 88 | lib-release: lib liblz4.pc 89 | 90 | .PHONY: lib 91 | lib: liblz4.a liblz4 92 | 93 | .PHONY: all 94 | all: lib liblz4.pc 95 | 96 | .PHONY: all32 97 | all32: CFLAGS+=-m32 98 | all32: all 99 | 100 | CLEAN += liblz4.a 101 | liblz4.a: $(SRCFILES) 102 | ifeq ($(BUILD_STATIC),yes) # can be disabled on command line 103 | @echo compiling static library 104 | $(COMPILE.c) $^ 105 | $(AR) rcs $@ *.o 106 | endif 107 | 108 | ifeq ($(WINBASED),yes) 109 | 110 | CLEAN += $(liblz4-dll.rc) 111 | liblz4-dll.rc: liblz4-dll.rc.in 112 | @echo creating library resource 113 | $(SED) -e 's|@LIBLZ4@|$(LIBLZ4_NAME)|' \ 114 | -e 's|@LIBVER_MAJOR@|$(LIBVER_MAJOR)|g' \ 115 | -e 's|@LIBVER_MINOR@|$(LIBVER_MINOR)|g' \ 116 | -e 's|@LIBVER_PATCH@|$(LIBVER_PATCH)|g' \ 117 | $< >$@ 118 | 119 | liblz4-dll.o: liblz4-dll.rc 120 | $(WINDRES) -i liblz4-dll.rc -o liblz4-dll.o 121 | 122 | CLEAN += $(LIBLZ4_EXP) 123 | $(LIBLZ4): $(SRCFILES) liblz4-dll.o 124 | ifeq ($(BUILD_SHARED),yes) 125 | @echo compiling dynamic library $(LIBVER) 126 | $(CC) $(ALLFLAGS) -DLZ4_DLL_EXPORT=1 -shared $^ -o $@ -Wl,--out-implib,$(LIBLZ4_EXP) 127 | endif 128 | 129 | else # not windows 130 | 131 | $(LIBLZ4): $(SRCFILES) 132 | ifeq ($(BUILD_SHARED),yes) 133 | @echo compiling dynamic library $(LIBVER) 134 | $(CC) $(ALLFLAGS) -shared $^ -fPIC -fvisibility=hidden $(SONAME_FLAGS) -o $@ 135 | @echo creating versioned links 136 | $(LN_SF) $@ liblz4.$(SHARED_EXT_MAJOR) 137 | $(LN_SF) $@ liblz4.$(SHARED_EXT) 138 | endif 139 | 140 | endif 141 | CLEAN += $(LIBLZ4) 142 | 143 | .PHONY: liblz4 144 | liblz4: $(LIBLZ4) 145 | 146 | CLEAN += liblz4.pc 147 | liblz4.pc: liblz4.pc.in Makefile 148 | @echo creating pkgconfig 149 | $(SED) -e 's|@PREFIX@|$(prefix)|' \ 150 | -e 's|@LIBDIR@|$(libdir)|' \ 151 | -e 's|@INCLUDEDIR@|$(includedir)|' \ 152 | -e 's|@VERSION@|$(LIBVER)|' \ 153 | -e 's|=${prefix}/|=$${prefix}/|' \ 154 | $< >$@ 155 | 156 | .PHONY: clean 157 | clean: 158 | ifeq ($(WINBASED),yes) 159 | $(RM) *.rc 160 | endif 161 | $(RM) $(CLEAN) core *.o *.a 162 | $(RM) *.$(SHARED_EXT) *.$(SHARED_EXT_MAJOR) *.$(SHARED_EXT_VER) 163 | @echo Cleaning library completed 164 | 165 | #----------------------------------------------------------------------------- 166 | # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets 167 | #----------------------------------------------------------------------------- 168 | ifeq ($(POSIX_ENV),Yes) 169 | 170 | .PHONY: listL120 171 | listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility) 172 | find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done 173 | 174 | DESTDIR ?= 175 | # directory variables : GNU conventions prefer lowercase 176 | # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html 177 | # support both lower and uppercase (BSD), use lower in script 178 | PREFIX ?= /usr/local 179 | prefix ?= $(PREFIX) 180 | EXEC_PREFIX ?= $(prefix) 181 | exec_prefix ?= $(EXEC_PREFIX) 182 | BINDIR ?= $(exec_prefix)/bin 183 | bindir ?= $(BINDIR) 184 | LIBDIR ?= $(exec_prefix)/lib 185 | libdir ?= $(LIBDIR) 186 | INCLUDEDIR ?= $(prefix)/include 187 | includedir ?= $(INCLUDEDIR) 188 | 189 | ifneq (,$(filter $(TARGET_OS),OpenBSD FreeBSD NetBSD DragonFly MidnightBSD)) 190 | PKGCONFIGDIR ?= $(prefix)/libdata/pkgconfig 191 | else 192 | PKGCONFIGDIR ?= $(libdir)/pkgconfig 193 | endif 194 | pkgconfigdir ?= $(PKGCONFIGDIR) 195 | 196 | .PHONY: install 197 | install: lib liblz4.pc 198 | $(MAKE_DIR) $(DESTDIR)$(pkgconfigdir)/ $(DESTDIR)$(includedir)/ $(DESTDIR)$(libdir)/ $(DESTDIR)$(bindir)/ 199 | $(INSTALL_DATA) liblz4.pc $(DESTDIR)$(pkgconfigdir)/ 200 | @echo Installing libraries in $(DESTDIR)$(libdir) 201 | ifeq ($(BUILD_STATIC),yes) 202 | $(INSTALL_DATA) liblz4.a $(DESTDIR)$(libdir)/liblz4.a 203 | $(INSTALL_DATA) lz4frame_static.h $(DESTDIR)$(includedir)/lz4frame_static.h 204 | $(INSTALL_DATA) lz4file.h $(DESTDIR)$(includedir)/lz4file.h 205 | endif 206 | ifeq ($(BUILD_SHARED),yes) 207 | # Traditionally, one installs the DLLs in the bin directory as programs 208 | # search them first in their directory. This allows to not pollute system 209 | # directories (like c:/windows/system32), nor modify the PATH variable. 210 | ifeq ($(WINBASED),yes) 211 | $(INSTALL_PROGRAM) $(LIBLZ4) $(DESTDIR)$(bindir) 212 | $(INSTALL_PROGRAM) $(LIBLZ4_EXP) $(DESTDIR)$(libdir) 213 | else 214 | $(INSTALL_PROGRAM) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir) 215 | $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR) 216 | $(LN_SF) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT) 217 | endif 218 | endif 219 | @echo Installing headers in $(DESTDIR)$(includedir) 220 | $(INSTALL_DATA) lz4.h $(DESTDIR)$(includedir)/lz4.h 221 | $(INSTALL_DATA) lz4hc.h $(DESTDIR)$(includedir)/lz4hc.h 222 | $(INSTALL_DATA) lz4frame.h $(DESTDIR)$(includedir)/lz4frame.h 223 | @echo lz4 libraries installed 224 | 225 | .PHONY: uninstall 226 | uninstall: 227 | $(RM) $(DESTDIR)$(pkgconfigdir)/liblz4.pc 228 | ifeq (WINBASED,yes) 229 | $(RM) $(DESTDIR)$(bindir)/$(LIBLZ4) 230 | $(RM) $(DESTDIR)$(libdir)/$(LIBLZ4_EXP) 231 | else 232 | $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT) 233 | $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_MAJOR) 234 | $(RM) $(DESTDIR)$(libdir)/liblz4.$(SHARED_EXT_VER) 235 | endif 236 | $(RM) $(DESTDIR)$(libdir)/liblz4.a 237 | $(RM) $(DESTDIR)$(includedir)/lz4.h 238 | $(RM) $(DESTDIR)$(includedir)/lz4hc.h 239 | $(RM) $(DESTDIR)$(includedir)/lz4frame.h 240 | $(RM) $(DESTDIR)$(includedir)/lz4frame_static.h 241 | $(RM) $(DESTDIR)$(includedir)/lz4file.h 242 | @echo lz4 libraries successfully uninstalled 243 | 244 | endif 245 | -------------------------------------------------------------------------------- /vendor/liblz4/README.md: -------------------------------------------------------------------------------- 1 | LZ4 - Library Files 2 | ================================ 3 | 4 | The `/lib` directory contains many files, but depending on project's objectives, 5 | not all of them are required. 6 | Limited systems may want to reduce the nb of source files to include 7 | as a way to reduce binary size and dependencies. 8 | 9 | Capabilities are added at the "level" granularity, detailed below. 10 | 11 | #### Level 1 : Minimal LZ4 build 12 | 13 | The minimum required is **`lz4.c`** and **`lz4.h`**, 14 | which provides the fast compression and decompression algorithms. 15 | They generate and decode data using the [LZ4 block format]. 16 | 17 | 18 | #### Level 2 : High Compression variant 19 | 20 | For more compression ratio at the cost of compression speed, 21 | the High Compression variant called **lz4hc** is available. 22 | Add files **`lz4hc.c`** and **`lz4hc.h`**. 23 | This variant also compresses data using the [LZ4 block format], 24 | and depends on regular `lib/lz4.*` source files. 25 | 26 | 27 | #### Level 3 : Frame support, for interoperability 28 | 29 | In order to produce compressed data compatible with `lz4` command line utility, 30 | it's necessary to use the [official interoperable frame format]. 31 | This format is generated and decoded automatically by the **lz4frame** library. 32 | Its public API is described in `lib/lz4frame.h`. 33 | In order to work properly, lz4frame needs all other modules present in `/lib`, 34 | including, lz4 and lz4hc, and also **xxhash**. 35 | So it's necessary to also include `xxhash.c` and `xxhash.h`. 36 | 37 | 38 | #### Level 4 : File compression operations 39 | 40 | As a helper around file operations, 41 | the library has been recently extended with `lz4file.c` and `lz4file.h` 42 | (still considered experimental at the time of this writing). 43 | These helpers allow opening, reading, writing, and closing files 44 | using transparent LZ4 compression / decompression. 45 | As a consequence, using `lz4file` adds a dependency on ``. 46 | 47 | `lz4file` relies on `lz4frame` in order to produce compressed data 48 | conformant to the [LZ4 Frame format] specification. 49 | Consequently, to enable this capability, 50 | it's necessary to include all `*.c` and `*.h` files from `lib/` directory. 51 | 52 | 53 | #### Advanced / Experimental API 54 | 55 | Definitions which are not guaranteed to remain stable in future versions, 56 | are protected behind macros, such as `LZ4_STATIC_LINKING_ONLY`. 57 | As the name suggests, these definitions should only be invoked 58 | in the context of static linking ***only***. 59 | Otherwise, dependent application may fail on API or ABI break in the future. 60 | The associated symbols are also not exposed by the dynamic library by default. 61 | Should they be nonetheless needed, it's possible to force their publication 62 | by using build macros `LZ4_PUBLISH_STATIC_FUNCTIONS` 63 | and `LZ4F_PUBLISH_STATIC_FUNCTIONS`. 64 | 65 | 66 | #### Build macros 67 | 68 | The following build macro can be selected to adjust source code behavior at compilation time : 69 | 70 | - `LZ4_FAST_DEC_LOOP` : this triggers a speed optimized decompression loop, more powerful on modern cpus. 71 | This loop works great on `x86`, `x64` and `aarch64` cpus, and is automatically enabled for them. 72 | It's also possible to enable or disable it manually, by passing `LZ4_FAST_DEC_LOOP=1` or `0` to the preprocessor. 73 | For example, with `gcc` : `-DLZ4_FAST_DEC_LOOP=1`, 74 | and with `make` : `CPPFLAGS+=-DLZ4_FAST_DEC_LOOP=1 make lz4`. 75 | 76 | - `LZ4_DISTANCE_MAX` : control the maximum offset that the compressor will allow. 77 | Set to 65535 by default, which is the maximum value supported by lz4 format. 78 | Reducing maximum distance will reduce opportunities for LZ4 to find matches, 79 | hence will produce a worse compression ratio. 80 | Setting a smaller max distance could allow compatibility with specific decoders with limited memory budget. 81 | This build macro only influences the compressed output of the compressor. 82 | 83 | - `LZ4_DISABLE_DEPRECATE_WARNINGS` : invoking a deprecated function will make the compiler generate a warning. 84 | This is meant to invite users to update their source code. 85 | Should this be a problem, it's generally possible to make the compiler ignore these warnings, 86 | for example with `-Wno-deprecated-declarations` on `gcc`, 87 | or `_CRT_SECURE_NO_WARNINGS` for Visual Studio. 88 | This build macro offers another project-specific method 89 | by defining `LZ4_DISABLE_DEPRECATE_WARNINGS` before including the LZ4 header files. 90 | 91 | - `LZ4_FORCE_SW_BITCOUNT` : by default, the compression algorithm tries to determine lengths 92 | by using bitcount instructions, generally implemented as fast single instructions in many cpus. 93 | In case the target cpus doesn't support it, or compiler intrinsic doesn't work, or feature bad performance, 94 | it's possible to use an optimized software path instead. 95 | This is achieved by setting this build macros. 96 | In most cases, it's not expected to be necessary, 97 | but it can be legitimately considered for less common platforms. 98 | 99 | - `LZ4_ALIGN_TEST` : alignment test ensures that the memory area 100 | passed as argument to become a compression state is suitably aligned. 101 | This test can be disabled if it proves flaky, by setting this value to 0. 102 | 103 | - `LZ4_USER_MEMORY_FUNCTIONS` : replace calls to ``'s `malloc()`, `calloc()` and `free()` 104 | by user-defined functions, which must be named `LZ4_malloc()`, `LZ4_calloc()` and `LZ4_free()`. 105 | User functions must be available at link time. 106 | 107 | - `LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION` : 108 | Remove support of dynamic memory allocation. 109 | For more details, see description of this macro in `lib/lz4.c`. 110 | 111 | - `LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT` : experimental feature aimed at producing the same 112 | compressed output on platforms of different endianness (i.e. little-endian and big-endian). 113 | Output on little-endian platforms shall remain unchanged, while big-endian platforms will start producing 114 | the same output as little-endian ones. This isn't expected to impact backward- and forward-compatibility 115 | in any way. 116 | 117 | - `LZ4_FREESTANDING` : by setting this build macro to 1, 118 | LZ4/HC removes dependencies on the C standard library, 119 | including allocation functions and `memmove()`, `memcpy()`, and `memset()`. 120 | This build macro is designed to help use LZ4/HC in restricted environments 121 | (embedded, bootloader, etc). 122 | For more details, see description of this macro in `lib/lz4.h`. 123 | 124 | - `LZ4_HEAPMODE` : Select how stateless compression functions like `LZ4_compress_default()` 125 | allocate memory for their hash table, 126 | in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). 127 | 128 | - `LZ4HC_HEAPMODE` : Select how stateless HC compression functions like `LZ4_compress_HC()` 129 | allocate memory for their workspace: 130 | in stack (0), or in heap (1:default). 131 | Since workspace is rather large, stack can be inconvenient, hence heap mode is recommended. 132 | 133 | - `LZ4F_HEAPMODE` : selects how `LZ4F_compressFrame()` allocates the compression state, 134 | either on stack (default, value 0) or using heap memory (value 1). 135 | 136 | 137 | #### Makefile variables 138 | 139 | The following `Makefile` variables can be selected to alter the profile of produced binaries : 140 | - `BUILD_SHARED` : generate `liblz4` dynamic library (enabled by default) 141 | - `BUILD_STATIC` : generate `liblz4` static library (enabled by default) 142 | 143 | 144 | #### Amalgamation 145 | 146 | lz4 source code can be amalgamated into a single file. 147 | One can combine all source code into `lz4_all.c` by using following command: 148 | ``` 149 | cat lz4.c lz4hc.c lz4frame.c > lz4_all.c 150 | ``` 151 | (`cat` file order is important) then compile `lz4_all.c`. 152 | All `*.h` files present in `/lib` remain necessary to compile `lz4_all.c`. 153 | 154 | 155 | #### Windows : using MinGW+MSYS to create DLL 156 | 157 | DLL can be created using MinGW+MSYS with the `make liblz4` command. 158 | This command creates `dll\liblz4.dll` and the import library `dll\liblz4.lib`. 159 | To override the `dlltool` command when cross-compiling on Linux, just set the `DLLTOOL` variable. Example of cross compilation on Linux with mingw-w64 64 bits: 160 | ``` 161 | make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT 162 | ``` 163 | The import library is only required with Visual C++. 164 | The header files `lz4.h`, `lz4hc.h`, `lz4frame.h` and the dynamic library 165 | `dll\liblz4.dll` are required to compile a project using gcc/MinGW. 166 | The dynamic library has to be added to linking options. 167 | It means that if a project that uses LZ4 consists of a single `test-dll.c` 168 | file it should be linked with `dll\liblz4.dll`. For example: 169 | ``` 170 | $(CC) $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\liblz4.dll 171 | ``` 172 | The compiled executable will require LZ4 DLL which is available at `dll\liblz4.dll`. 173 | 174 | 175 | #### Miscellaneous 176 | 177 | Other files present in the directory are not source code. They are : 178 | 179 | - `LICENSE` : contains the BSD license text 180 | - `Makefile` : `make` script to compile and install lz4 library (static and dynamic) 181 | - `liblz4.pc.in` : for `pkg-config` (used in `make install`) 182 | - `README.md` : this file 183 | 184 | [official interoperable frame format]: ../doc/lz4_Frame_format.md 185 | [LZ4 Frame format]: ../doc/lz4_Frame_format.md 186 | [LZ4 block format]: ../doc/lz4_Block_format.md 187 | 188 | 189 | #### License 190 | 191 | All source material within __lib__ directory are BSD 2-Clause licensed. 192 | See [LICENSE](LICENSE) for details. 193 | The license is also reminded at the top of each source file. 194 | -------------------------------------------------------------------------------- /vendor/liblz4/dll/example/Makefile: -------------------------------------------------------------------------------- 1 | # ########################################################################## 2 | # LZ4 programs - Makefile 3 | # Copyright (C) Yann Collet 2016-2020 4 | # 5 | # GPL v2 License 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation; either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | # 21 | # You can contact the author at : 22 | # - LZ4 homepage : http://www.lz4.org 23 | # - LZ4 source repository : https://github.com/lz4/lz4 24 | # ########################################################################## 25 | 26 | VOID := /dev/null 27 | LZ4DIR := ../include 28 | LIBDIR := ../static 29 | DLLDIR := ../dll 30 | 31 | CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make 32 | CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ 33 | -Wdeclaration-after-statement -Wstrict-prototypes \ 34 | -Wpointer-arith -Wstrict-aliasing=1 35 | CFLAGS += $(MOREFLAGS) 36 | CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ 37 | FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 38 | 39 | 40 | # Define *.exe as extension for Windows systems 41 | ifneq (,$(filter Windows%,$(OS))) 42 | EXT =.exe 43 | else 44 | EXT = 45 | endif 46 | 47 | .PHONY: default fullbench-dll fullbench-lib 48 | 49 | 50 | default: all 51 | 52 | all: fullbench-dll fullbench-lib 53 | 54 | 55 | fullbench-lib: fullbench.c xxhash.c 56 | $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4_static.lib 57 | 58 | fullbench-dll: fullbench.c xxhash.c 59 | $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(DLLDIR)/liblz4.dll 60 | 61 | clean: 62 | @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ 63 | @echo Cleaning completed 64 | -------------------------------------------------------------------------------- /vendor/liblz4/dll/example/README.md: -------------------------------------------------------------------------------- 1 | LZ4 Windows binary package 2 | ==================================== 3 | 4 | #### The package contents 5 | 6 | - `lz4.exe` : Command Line Utility, supporting gzip-like arguments 7 | - `dll\msys-lz4-1.dll` : The DLL of LZ4 library, compiled by msys 8 | - `dll\liblz4.dll.a` : The import library of LZ4 library for Visual C++ 9 | - `example\` : The example of usage of LZ4 library 10 | - `include\` : Header files required with LZ4 library 11 | - `static\liblz4_static.lib` : The static LZ4 library 12 | 13 | 14 | #### Usage of Command Line Interface 15 | 16 | Command Line Interface (CLI) supports gzip-like arguments. 17 | By default CLI takes an input file and compresses it to an output file: 18 | ``` 19 | Usage: lz4 [arg] [input] [output] 20 | ``` 21 | The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can 22 | be improved with commands from `-3` to `-16` but higher levels also have slower 23 | compression. CLI includes in-memory compression benchmark module with compression 24 | levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds. 25 | CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined 26 | into `-b1e18i1`. 27 | 28 | 29 | #### The example of usage of static and dynamic LZ4 libraries with gcc/MinGW 30 | 31 | Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`. 32 | `fullbench-dll` uses a dynamic LZ4 library from the `dll` directory. 33 | `fullbench-lib` uses a static LZ4 library from the `lib` directory. 34 | 35 | 36 | #### Using LZ4 DLL with gcc/MinGW 37 | 38 | The header files from `include\` and the dynamic library `dll\msys-lz4-1.dll` 39 | are required to compile a project using gcc/MinGW. 40 | The dynamic library has to be added to linking options. 41 | It means that if a project that uses LZ4 consists of a single `test-dll.c` 42 | file it should be linked with `dll\msys-lz4-1.dll`. For example: 43 | ``` 44 | gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\msys-lz4-1.dll 45 | ``` 46 | The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`. 47 | 48 | 49 | #### The example of usage of static and dynamic LZ4 libraries with Visual C++ 50 | 51 | Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a 52 | dynamic LZ4 library from the `dll` directory. The solution works with Visual C++ 53 | 2010 or newer. When one will open the solution with Visual C++ newer than 2010 54 | then the solution will be upgraded to the current version. 55 | 56 | 57 | #### Using LZ4 DLL with Visual C++ 58 | 59 | The header files from `include\` and the import library `dll\liblz4.dll.a` 60 | are required to compile a project using Visual C++. 61 | 62 | 1. The header files should be added to `Additional Include Directories` that can 63 | be found in project properties `C/C++` then `General`. 64 | 2. The import library has to be added to `Additional Dependencies` that can 65 | be found in project properties `Linker` then `Input`. 66 | If one will provide only the name `liblz4.dll.a` without a full path to the library 67 | the directory has to be added to `Linker\General\Additional Library Directories`. 68 | 69 | The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`. 70 | -------------------------------------------------------------------------------- /vendor/liblz4/dll/example/fullbench-dll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Express 2012 for Windows Desktop 3 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" 4 | EndProject 5 | Global 6 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 7 | Debug|Win32 = Debug|Win32 8 | Debug|x64 = Debug|x64 9 | Release|Win32 = Release|Win32 10 | Release|x64 = Release|x64 11 | EndGlobalSection 12 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 13 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 14 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 15 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 16 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 17 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 18 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 19 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 20 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /vendor/liblz4/dll/example/fullbench-dll.vcxproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Debug 10 | x64 11 | 12 | 13 | Release 14 | Win32 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | {13992FD2-077E-4954-B065-A428198201A9} 23 | Win32Proj 24 | fullbench-dll 25 | $(SolutionDir)bin\$(Platform)_$(Configuration)\ 26 | $(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\ 27 | 28 | 29 | 30 | Application 31 | true 32 | Unicode 33 | 34 | 35 | Application 36 | true 37 | Unicode 38 | 39 | 40 | Application 41 | false 42 | true 43 | Unicode 44 | 45 | 46 | Application 47 | false 48 | true 49 | Unicode 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | true 69 | $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); 70 | 71 | 72 | true 73 | $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); 74 | true 75 | 76 | 77 | false 78 | $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); 79 | 80 | 81 | false 82 | $(IncludePath);$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); 83 | true 84 | 85 | 86 | 87 | 88 | 89 | Level4 90 | Disabled 91 | WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) 92 | true 93 | false 94 | ..\include 95 | 96 | 97 | Console 98 | true 99 | $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) 100 | liblz4.lib;%(AdditionalDependencies) 101 | false 102 | 103 | 104 | 105 | 106 | 107 | 108 | Level4 109 | Disabled 110 | WIN32;_DEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) 111 | true 112 | true 113 | /analyze:stacksize295252 %(AdditionalOptions) 114 | ..\include 115 | 116 | 117 | Console 118 | true 119 | $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) 120 | liblz4.lib;%(AdditionalDependencies) 121 | 122 | 123 | 124 | 125 | Level4 126 | 127 | 128 | MaxSpeed 129 | true 130 | true 131 | WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) 132 | false 133 | false 134 | ..\include 135 | 136 | 137 | Console 138 | true 139 | true 140 | true 141 | $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) 142 | liblz4.lib;%(AdditionalDependencies) 143 | false 144 | 145 | 146 | 147 | 148 | Level4 149 | 150 | 151 | MaxSpeed 152 | true 153 | true 154 | WIN32;NDEBUG;_CONSOLE;LZ4_DLL_IMPORT=1;%(PreprocessorDefinitions) 155 | false 156 | true 157 | /analyze:stacksize295252 %(AdditionalOptions) 158 | ..\include 159 | 160 | 161 | Console 162 | true 163 | true 164 | true 165 | $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) 166 | liblz4.lib;%(AdditionalDependencies) 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | -------------------------------------------------------------------------------- /vendor/liblz4/liblz4-dll.rc.in: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // DLL version information. 4 | 1 VERSIONINFO 5 | FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 6 | PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 7 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 8 | #ifdef _DEBUG 9 | FILEFLAGS VS_FF_DEBUG | VS_FF_PRERELEASE 10 | #else 11 | FILEFLAGS 0 12 | #endif 13 | FILEOS VOS_NT_WINDOWS32 14 | FILETYPE VFT_DLL 15 | FILESUBTYPE VFT2_UNKNOWN 16 | BEGIN 17 | BLOCK "StringFileInfo" 18 | BEGIN 19 | BLOCK "040904B0" 20 | BEGIN 21 | VALUE "CompanyName", "Yann Collet" 22 | VALUE "FileDescription", "Extremely fast compression" 23 | VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 24 | VALUE "InternalName", "@LIBLZ4@" 25 | VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet" 26 | VALUE "OriginalFilename", "@LIBLZ4@.dll" 27 | VALUE "ProductName", "LZ4" 28 | VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 29 | END 30 | END 31 | BLOCK "VarFileInfo" 32 | BEGIN 33 | VALUE "Translation", 0x0409, 1200 34 | END 35 | END 36 | -------------------------------------------------------------------------------- /vendor/liblz4/liblz4.pc.in: -------------------------------------------------------------------------------- 1 | # LZ4 - Fast LZ compression algorithm 2 | # Copyright (C) 2011-2020, Yann Collet. 3 | # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | libdir=@LIBDIR@ 7 | includedir=@INCLUDEDIR@ 8 | 9 | Name: lz4 10 | Description: extremely fast lossless compression algorithm library 11 | URL: http://www.lz4.org/ 12 | Version: @VERSION@ 13 | Libs: -L${libdir} -llz4 14 | Cflags: -I${includedir} 15 | -------------------------------------------------------------------------------- /vendor/liblz4/lz4file.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 file library 3 | Header File 4 | Copyright (C) 2022, Xiaomi Inc. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | #if defined (__cplusplus) 35 | extern "C" { 36 | #endif 37 | 38 | #ifndef LZ4FILE_H 39 | #define LZ4FILE_H 40 | 41 | #include /* FILE* */ 42 | #include "lz4frame_static.h" 43 | 44 | typedef struct LZ4_readFile_s LZ4_readFile_t; 45 | typedef struct LZ4_writeFile_s LZ4_writeFile_t; 46 | 47 | /*! LZ4F_readOpen() : 48 | * Set read lz4file handle. 49 | * `lz4f` will set a lz4file handle. 50 | * `fp` must be the return value of the lz4 file opened by fopen. 51 | */ 52 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp); 53 | 54 | /*! LZ4F_read() : 55 | * Read lz4file content to buffer. 56 | * `lz4f` must use LZ4_readOpen to set first. 57 | * `buf` read data buffer. 58 | * `size` read data buffer size. 59 | */ 60 | LZ4FLIB_STATIC_API size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size); 61 | 62 | /*! LZ4F_readClose() : 63 | * Close lz4file handle. 64 | * `lz4f` must use LZ4_readOpen to set first. 65 | */ 66 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead); 67 | 68 | /*! LZ4F_writeOpen() : 69 | * Set write lz4file handle. 70 | * `lz4f` will set a lz4file handle. 71 | * `fp` must be the return value of the lz4 file opened by fopen. 72 | */ 73 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr); 74 | 75 | /*! LZ4F_write() : 76 | * Write buffer to lz4file. 77 | * `lz4f` must use LZ4F_writeOpen to set first. 78 | * `buf` write data buffer. 79 | * `size` write data buffer size. 80 | */ 81 | LZ4FLIB_STATIC_API size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, const void* buf, size_t size); 82 | 83 | /*! LZ4F_writeClose() : 84 | * Close lz4file handle. 85 | * `lz4f` must use LZ4F_writeOpen to set first. 86 | */ 87 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite); 88 | 89 | #endif /* LZ4FILE_H */ 90 | 91 | #if defined (__cplusplus) 92 | } 93 | #endif 94 | -------------------------------------------------------------------------------- /vendor/liblz4/lz4frame_static.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 auto-framing library 3 | Header File for static linking only 4 | Copyright (C) 2011-2020, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #ifndef LZ4FRAME_STATIC_H_0398209384 37 | #define LZ4FRAME_STATIC_H_0398209384 38 | 39 | /* The declarations that formerly were made here have been merged into 40 | * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward, 41 | * it is recommended to simply include that header directly. 42 | */ 43 | 44 | #define LZ4F_STATIC_LINKING_ONLY 45 | #include "lz4frame.h" 46 | 47 | #endif /* LZ4FRAME_STATIC_H_0398209384 */ 48 | --------------------------------------------------------------------------------