├── .github └── workflows │ └── test.yml ├── .gitignore ├── Cargo.toml ├── LICENSE.md ├── README.md ├── build.rs ├── src ├── error.rs ├── lib.rs ├── lzma_stream_wrapper.rs ├── lzma_sys.rs ├── reader.rs └── writer.rs └── tests ├── test.rs ├── test_file.lzma └── test_file.txt /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | check: 6 | name: Check 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: actions-rs/toolchain@v1 11 | with: 12 | profile: minimal 13 | toolchain: stable 14 | override: true 15 | - uses: actions-rs/cargo@v1 16 | with: 17 | command: check 18 | 19 | test: 20 | name: Run test suite 21 | runs-on: ubuntu-latest 22 | container: rust:latest 23 | 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v3 27 | 28 | - name: Install dependencies 29 | run: | 30 | apt-get update -y 31 | apt-get install -y pkg-config liblzma-dev 32 | 33 | - name: Run tests 34 | run: cargo test 35 | 36 | clippy: 37 | name: Run clippy 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v3 41 | - uses: actions-rs/toolchain@v1 42 | with: 43 | profile: minimal 44 | toolchain: stable 45 | override: true 46 | - run: rustup component add clippy 47 | - uses: actions-rs/cargo@v1 48 | with: 49 | command: clippy 50 | args: -- -D warnings -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | target 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust-lzma" 3 | version = "0.6.0" 4 | authors = ["fpgaminer "] 5 | description = "Simple interface for LZMA compression and decompression." 6 | documentation = "https://docs.rs/rust-lzma/" 7 | repository = "https://github.com/fpgaminer/rust-lzma" 8 | readme = "README.md" 9 | keywords = ["lzma", "compression", "decompression", "xz", "liblzma"] 10 | license = "MIT" 11 | build = "build.rs" 12 | links = "lzma" 13 | 14 | [lib] 15 | name = "lzma" 16 | 17 | [dependencies] 18 | 19 | [build-dependencies] 20 | pkg-config = "^0.3.3" 21 | 22 | [target.'cfg(target_env = "msvc")'.build-dependencies] 23 | vcpkg = "0.2" 24 | 25 | [features] 26 | static = [] 27 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 fpgaminer@bitcoin-mining.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rust-lzma [![Crates.io](https://img.shields.io/crates/v/rust-lzma.svg)](https://crates.io/crates/rust-lzma) # 2 | 3 | [Documentation](https://docs.rs/rust-lzma/) 4 | 5 | This crate provides a simple interface to liblzma. LZMA is more commonly known 6 | as XZ or 7zip, (as in, files with the `.xz` or `.7z` file extension). LZMA 7 | compression is fast and aggressive, compressing better than bzip2. liblzma 8 | implements the XZ variant, so it can read and write `.xz` files/streams. 9 | 10 | Two interfaces are provided. `LzmaReader`/`LzmaWriter` are generic Readers and 11 | Writers that can be composed with other `Read`/`Write` interfaces. For example, 12 | wrap them around a `File` and you can write data to a file while compressing it 13 | on the fly, or stream in an `xz` file from disk. 14 | 15 | `compress`/`decompress` are easy to use functions for simple use cases. 16 | 17 | See the documentation for details on usage. 18 | 19 | 20 | ## Example ## 21 | Cargo.toml: 22 | ```toml 23 | [dependencies] 24 | rust-lzma = "0.6" 25 | ``` 26 | main.rs: 27 | ```Rust 28 | extern crate lzma; 29 | 30 | use lzma::LzmaWriter; 31 | use std::io::prelude::*; 32 | use std::fs::File; 33 | 34 | fn main() { 35 | let f = File::create("foo.xz").unwrap(); 36 | let mut f = LzmaWriter::new_compressor(f, 6).unwrap(); 37 | 38 | write!(f, "It's a small world!").unwrap(); 39 | f.finish().unwrap(); 40 | } 41 | ``` 42 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | fn main() { 4 | let target = env::var("TARGET").unwrap(); 5 | let is_windows = target.contains("windows"); 6 | let statik = env::var("CARGO_FEATURE_STATIC").is_ok(); 7 | 8 | if is_windows { 9 | #[cfg(windows)] 10 | vcpkg::Config::new() 11 | .emit_includes(true) 12 | .find_package("liblzma") 13 | .expect("Could not find liblzma using vcpkg"); 14 | } else { 15 | #[cfg(not(windows))] 16 | pkg_config::Config::new() 17 | .statik(statik) 18 | .probe("liblzma") 19 | .expect("Could not find liblzma using pkg-config"); 20 | } 21 | 22 | if statik { 23 | println!("cargo:rustc-link-lib=static=lzma"); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use lzma_sys::*; 2 | use std::convert::From; 3 | use std::result::Result; 4 | use std::error; 5 | use std::fmt; 6 | use std::error::Error as StdError; 7 | use std::io::Error as IoError; 8 | 9 | 10 | /// An error produced by an operation on LZMA data 11 | #[derive(Debug)] 12 | pub enum LzmaError { 13 | /// Failed Memory Allocation 14 | Mem, 15 | /// Memory limit would be violated 16 | MemLimit, 17 | /// XZ magic bytes weren't found 18 | Format, 19 | /// Unsupported compression options 20 | Options, 21 | /// Corrupt data 22 | Data, 23 | /// Data looks truncated 24 | Buf, 25 | /// std::io::Error 26 | Io(IoError), 27 | /// An unknown error 28 | Other, 29 | } 30 | 31 | impl fmt::Display for LzmaError { 32 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 33 | match *self { 34 | LzmaError::Mem => write!(f, "Memory allocation failed"), 35 | LzmaError::MemLimit => write!(f, "Memory limit would be violated"), 36 | LzmaError::Format => write!(f, "XZ magic bytes were not found"), 37 | LzmaError::Options => write!(f, "Unsupported compression options"), 38 | LzmaError::Data => write!(f, "Corrupt data"), 39 | LzmaError::Buf => write!(f, "Data look like it was truncated or possibly corrupt"), 40 | LzmaError::Io(ref err) => write!(f, "{}", err), 41 | LzmaError::Other => write!(f, "Unknown error"), 42 | } 43 | } 44 | } 45 | 46 | impl StdError for LzmaError { 47 | fn description(&self) -> &str { 48 | match *self { 49 | LzmaError::Mem => "Memory allocation failed", 50 | LzmaError::MemLimit => "Memory limit would be violated", 51 | LzmaError::Format => "XZ magic bytes were not found", 52 | LzmaError::Options => "Unsupported compression options", 53 | LzmaError::Data => "Corrupt data", 54 | LzmaError::Buf => "Data look like it was truncated or possibly corrupt", 55 | LzmaError::Io(..) => "IO error", 56 | LzmaError::Other => "Unknown error", 57 | } 58 | } 59 | 60 | fn cause(&self) -> Option<&dyn error::Error> { 61 | match *self { 62 | LzmaError::Io(ref err) => Some(err), 63 | _ => None, 64 | } 65 | } 66 | } 67 | 68 | impl From for LzmaError { 69 | fn from(err: IoError) -> LzmaError { 70 | LzmaError::Io(err) 71 | } 72 | } 73 | 74 | 75 | /* Return values from liblzma are converted into this for easier handling */ 76 | pub type LzmaLibResult = Result; 77 | 78 | impl From for LzmaLibResult { 79 | fn from(ret: lzma_ret) -> LzmaLibResult { 80 | match ret { 81 | lzma_ret::LzmaOk => Ok(ret), 82 | lzma_ret::LzmaStreamEnd => Ok(ret), 83 | lzma_ret::LzmaNoCheck => Ok(ret), 84 | lzma_ret::LzmaUnsupportedCheck => Ok(ret), // NOTE: This is an error in some cases. Not sure how to handle properly. 85 | lzma_ret::LzmaGetCheck => Ok(ret), 86 | lzma_ret::LzmaMemError => Err(LzmaError::Mem), 87 | lzma_ret::LzmaMemlimitError => Err(LzmaError::MemLimit), 88 | lzma_ret::LzmaFormatError => Err(LzmaError::Format), 89 | lzma_ret::LzmaOptionsError => Err(LzmaError::Options), 90 | lzma_ret::LzmaDataError => Err(LzmaError::Data), 91 | lzma_ret::LzmaBufError => Err(LzmaError::Buf), 92 | _ => Err(LzmaError::Other), 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides a simple interface to liblzma. LZMA is more commonly known as XZ or 7zip, 2 | //! (as in, files with the `.xz` or `.7z` file extension). LZMA compression is fast and aggressive, 3 | //! compressing better than bzip2. liblzma implements the XZ variant, so it can read and write 4 | //! `.xz` files/streams. 5 | //! 6 | //! Two interfaces are provided. `LzmaReader`/`LzmaWriter` are generic Readers and Writers that 7 | //! can be composed with other `Read`/`Write` interfaces. For example, wrap them around a `File` 8 | //! and you can write data to a file while compressing it on the fly, or stream in an `xz` file 9 | //! from disk. 10 | //! 11 | //! `compress`/`decompress` are easy to use functions for simple use cases. 12 | //! 13 | //! See the `LzmaReader` and `LzmaWriter` documentation for further details on that interface. 14 | //! `compress` and `decompress` are documented here. 15 | //! 16 | //! # Examples 17 | //! 18 | //! ``` 19 | //! let test_string = "Like tears in rain"; 20 | //! let mut compressed = lzma::compress(test_string.as_bytes(), 6).unwrap(); 21 | //! let decompressed = lzma::decompress(&mut compressed).unwrap(); 22 | //! let decompressed_str = String::from_utf8(decompressed).unwrap(); 23 | //! 24 | //! assert_eq!(test_string, decompressed_str); 25 | //! ``` 26 | 27 | pub enum Direction { 28 | Compress, 29 | Decompress, 30 | } 31 | 32 | mod lzma_sys; 33 | mod lzma_stream_wrapper; 34 | pub mod reader; 35 | pub mod writer; 36 | pub mod error; 37 | 38 | use std::io::Read; 39 | pub use reader::LzmaReader; 40 | pub use writer::LzmaWriter; 41 | pub use error::LzmaError; 42 | 43 | 44 | pub const EXTREME_PRESET: u32 = 1 << 31; 45 | 46 | 47 | /// Compress `buf` and return the result. 48 | /// 49 | /// preset is [0-9] and corresponds to xz's presets. 50 | /// Binary-or with EXTREME_PRESET for --extreme (e.g. 9 | EXTREME_PRESET). 51 | pub fn compress(buf: &[u8], preset: u32) -> Result, LzmaError> { 52 | let mut output: Vec = Vec::new(); 53 | 54 | { 55 | let mut reader = LzmaReader::new_compressor(buf, preset)?; 56 | 57 | reader.read_to_end(&mut output)?; 58 | } 59 | 60 | Ok(output) 61 | } 62 | 63 | 64 | /// Decompress `buf` and return the result. 65 | pub fn decompress(buf: &[u8]) -> Result, LzmaError> { 66 | let mut output: Vec = Vec::new(); 67 | 68 | { 69 | let mut reader = LzmaReader::new_decompressor(buf)?; 70 | 71 | reader.read_to_end(&mut output)?; 72 | } 73 | 74 | Ok(output) 75 | } 76 | -------------------------------------------------------------------------------- /src/lzma_stream_wrapper.rs: -------------------------------------------------------------------------------- 1 | //! Wraps the underlying FFI struct `lzma_stream` to provide various safety guarantees, like the Send trait. 2 | 3 | use lzma_sys::*; 4 | use error::{LzmaError, LzmaLibResult}; 5 | use std::ptr; 6 | use std::ops::Drop; 7 | 8 | 9 | pub struct LzmaStreamWrapper { 10 | stream: lzma_stream, 11 | } 12 | 13 | pub struct LzmaCodeResult { 14 | /// The return value of lzma_code 15 | pub ret: LzmaLibResult, 16 | /// The number of bytes read from input 17 | pub bytes_read: usize, 18 | /// The number of bytes written to output 19 | pub bytes_written: usize, 20 | } 21 | 22 | 23 | // I believe liblzma is at least Send thread safe, though using it like that will result in 24 | // malloc being called in one thread and free being called in another. That's usually safe, 25 | // but depends on how liblzma was compiled. 26 | unsafe impl Send for LzmaStreamWrapper {} 27 | 28 | 29 | impl LzmaStreamWrapper { 30 | pub fn new() -> LzmaStreamWrapper { 31 | LzmaStreamWrapper { 32 | stream: lzma_stream::new(), 33 | } 34 | } 35 | 36 | pub fn easy_encoder(&mut self, preset: u32, check: lzma_check) -> Result<(), LzmaError> { 37 | unsafe { 38 | LzmaLibResult::from(lzma_easy_encoder(&mut self.stream, preset, check)).map(|_| ()) 39 | } 40 | } 41 | 42 | pub fn stream_decoder(&mut self, memlimit: u64, flags: u32) -> Result<(), LzmaError> { 43 | unsafe { 44 | LzmaLibResult::from(lzma_auto_decoder(&mut self.stream, memlimit, flags)).map(|_| ()) 45 | } 46 | } 47 | 48 | pub fn end(&mut self) { 49 | unsafe { 50 | lzma_end(&mut self.stream) 51 | } 52 | } 53 | 54 | /// Pointers to input and output are given to liblzma during execution of this function, 55 | /// but they are removed before returning. So that should keep everything safe. 56 | pub fn code(&mut self, input: &[u8], output: &mut [u8], action: lzma_action) -> LzmaCodeResult { 57 | // Prepare lzma_stream 58 | self.stream.next_in = input.as_ptr(); 59 | self.stream.avail_in = input.len(); 60 | self.stream.next_out = output.as_mut_ptr(); 61 | self.stream.avail_out = output.len(); 62 | 63 | // Execute lzma_code and get results 64 | let ret = unsafe { 65 | LzmaLibResult::from(lzma_code(&mut self.stream, action)) 66 | }; 67 | let bytes_read = input.len() - self.stream.avail_in; 68 | let bytes_written = output.len() - self.stream.avail_out; 69 | 70 | // Clear pointers from lzma_stream 71 | self.stream.next_in = ptr::null(); 72 | self.stream.avail_in = 0; 73 | self.stream.next_out = ptr::null_mut(); 74 | self.stream.avail_out = 0; 75 | 76 | LzmaCodeResult { 77 | ret, 78 | bytes_read, 79 | bytes_written, 80 | } 81 | } 82 | } 83 | 84 | // This makes sure to call lzma_end, which frees memory that liblzma has allocated internally 85 | // Note: It appears to be safe to call lzma_end multiple times; so this Drop is safe 86 | // even if the user has already called end. 87 | impl Drop for LzmaStreamWrapper { 88 | fn drop(&mut self) { 89 | self.end(); 90 | } 91 | } -------------------------------------------------------------------------------- /src/lzma_sys.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::c_void; 2 | use std::mem; 3 | 4 | 5 | #[repr(C)] 6 | pub struct lzma_stream { 7 | /// < Pointer to the next input byte. 8 | pub next_in: *const u8, 9 | /// < Number of available input bytes in next_in. 10 | pub avail_in: usize, 11 | /// < Total number of bytes read by liblzma. 12 | pub total_in: u64, 13 | 14 | /// < Pointer to the next output position. 15 | pub next_out: *mut u8, 16 | /// < Amount of free space in next_out. 17 | pub avail_out: usize, 18 | /// < Total number of bytes written by liblzma. 19 | pub total_out: u64, 20 | 21 | /// \brief Custom memory allocation functions 22 | /// 23 | /// In most cases this is NULL which makes liblzma use 24 | /// the standard malloc() and free(). 25 | /// 26 | /// \note In 5.0.x this is not a const pointer. 27 | pub allocator: *const lzma_allocator, 28 | 29 | /// Internal state is not visible to applications. 30 | pub internal: *mut c_void, // Actually a pointer to lzma_internal, but lzma_internal is opaque 31 | 32 | pub reserved_ptr1: *mut c_void, 33 | pub reserved_ptr2: *mut c_void, 34 | pub reserved_ptr3: *mut c_void, 35 | pub reserved_ptr4: *mut c_void, 36 | pub reserved_int1: u64, 37 | pub reserved_int2: u64, 38 | pub reserved_int3: usize, 39 | pub reserved_int4: usize, 40 | pub reserved_enum1: u32, // Actually an enum, but it's opaque so we stub with u32 41 | pub reserved_enum2: u32, // Actually an enum, but it's opaque so we stub with u32 42 | } 43 | 44 | impl lzma_stream { 45 | // base.h defines LZMA_STREAM_INIT; we declare new instead. 46 | pub fn new() -> lzma_stream { 47 | unsafe { 48 | mem::zeroed() 49 | } 50 | } 51 | } 52 | 53 | 54 | #[repr(C)] 55 | pub struct lzma_allocator { 56 | pub alloc: *mut extern fn(opaque: *mut c_void, nmemb: usize, size: usize), 57 | pub free: extern fn(opaque: *mut c_void, ptr: *mut c_void), 58 | pub opaque: *mut c_void, 59 | } 60 | 61 | 62 | #[allow(clippy::enum_variant_names)] 63 | #[repr(C)] 64 | #[derive(Clone, Copy, Debug)] 65 | #[must_use] 66 | pub enum lzma_ret { 67 | LzmaOk = 0, 68 | LzmaStreamEnd = 1, 69 | LzmaNoCheck = 2, 70 | LzmaUnsupportedCheck = 3, 71 | LzmaGetCheck = 4, 72 | LzmaMemError = 5, 73 | LzmaMemlimitError = 6, 74 | LzmaFormatError = 7, 75 | LzmaOptionsError = 8, 76 | LzmaDataError = 9, 77 | LzmaBufError = 10, 78 | LzmaProgError = 11, 79 | } 80 | 81 | 82 | #[allow(clippy::enum_variant_names)] 83 | #[repr(C)] 84 | #[derive(Clone, Copy)] 85 | #[allow(dead_code)] 86 | pub enum lzma_action { 87 | LzmaRun = 0, 88 | LzmaSyncFlush = 1, 89 | LzmaFullFlush = 2, 90 | LzmaFullBarrier = 4, 91 | LzmaFinish = 3, 92 | } 93 | 94 | 95 | #[repr(C)] 96 | #[derive(Clone, Copy)] 97 | #[allow(dead_code)] 98 | pub enum lzma_check { 99 | LzmaCheckNone = 0, 100 | LzmaCheckCrc32 = 1, 101 | LzmaCheckCrc64 = 4, 102 | LzmaCheckSha256 = 10, 103 | } 104 | 105 | 106 | extern { 107 | pub fn lzma_easy_encoder(stream: *mut lzma_stream, preset: u32, check: lzma_check) -> lzma_ret; 108 | pub fn lzma_code(stream: *mut lzma_stream, action: lzma_action) -> lzma_ret; 109 | pub fn lzma_end(stream: *mut lzma_stream); 110 | pub fn lzma_auto_decoder(stream: *mut lzma_stream, memlimit: u64, flags: u32) -> lzma_ret; 111 | } 112 | -------------------------------------------------------------------------------- /src/reader.rs: -------------------------------------------------------------------------------- 1 | //! This module implements `LzmaReader`. 2 | //! 3 | //! `LzmaReader` implements the LZMA (XZ) compression/decompression algorithm as a generic 4 | //! Reader. In other words, it behaves similar to BufReader. Instead of buffering the `Read` 5 | //! object passed to it, `LzmaReader` applies compression or decompression to it as it's read. 6 | //! 7 | //! 8 | //! # Examples 9 | //! 10 | //! ```no_run 11 | //! use lzma::LzmaReader; 12 | //! use std::io::prelude::*; 13 | //! use std::fs::File; 14 | //! 15 | //! let f = File::open("foo.xz").unwrap(); 16 | //! let mut f = LzmaReader::new_decompressor(f).unwrap(); 17 | //! let mut s = String::new(); 18 | //! 19 | //! f.read_to_string(&mut s).unwrap(); 20 | //! println!("{}", s); 21 | //! ``` 22 | 23 | use std::io::{self, Read}; 24 | use lzma_sys::*; 25 | use std; 26 | use error::LzmaError; 27 | use ::Direction; 28 | use lzma_stream_wrapper::LzmaStreamWrapper; 29 | 30 | 31 | const DEFAULT_BUF_SIZE: usize = 4 * 1024; 32 | 33 | 34 | pub struct LzmaReader { 35 | inner: T, 36 | stream: LzmaStreamWrapper, 37 | buffer: Vec, 38 | buffer_offset: usize, 39 | buffer_len: usize, 40 | direction: Direction, 41 | } 42 | 43 | 44 | impl LzmaReader { 45 | pub fn new_compressor(inner: T, preset: u32) -> Result, LzmaError> { 46 | LzmaReader::with_capacity(DEFAULT_BUF_SIZE, inner, Direction::Compress, preset) 47 | } 48 | 49 | pub fn new_decompressor(inner: T) -> Result, LzmaError> { 50 | LzmaReader::with_capacity(DEFAULT_BUF_SIZE, inner, Direction::Decompress, 0) 51 | } 52 | 53 | pub fn with_capacity(capacity: usize, inner: T, direction: Direction, preset: u32) -> Result, LzmaError> { 54 | let mut reader = LzmaReader { 55 | inner, 56 | stream: LzmaStreamWrapper::new(), 57 | buffer: vec![0; capacity], 58 | buffer_offset: 0, 59 | buffer_len: 0, 60 | direction, 61 | }; 62 | 63 | match reader.direction { 64 | Direction::Compress => { 65 | reader.stream.easy_encoder(preset, lzma_check::LzmaCheckCrc64)? 66 | }, 67 | Direction::Decompress => { 68 | reader.stream.stream_decoder(std::u64::MAX, 0)? 69 | }, 70 | } 71 | 72 | Ok(reader) 73 | } 74 | 75 | pub fn into_inner(self) -> T { self.inner } 76 | } 77 | 78 | 79 | impl Read for LzmaReader { 80 | /// Reads data from the wrapped object, applies compression/decompression, and puts the results 81 | /// into buf. 82 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 83 | // Our code doesn't handle buf.len() being 0, so exit early 84 | if buf.is_empty() { 85 | return Ok(0); 86 | } 87 | 88 | loop { 89 | let mut action = lzma_action::LzmaRun; 90 | 91 | // If our internal read buffer is empty, re-fill it by calling read on the inner Read object. 92 | if self.buffer_len == 0 { 93 | self.buffer_offset = 0; 94 | self.buffer_len = self.inner.read(&mut self.buffer)?; 95 | 96 | if self.buffer_len == 0 { 97 | action = lzma_action::LzmaFinish; 98 | } 99 | } 100 | 101 | // Instruct liblzma to compress/decompress data from the buffer, and write the results to buf 102 | let result = self.stream.code(&self.buffer[self.buffer_offset..(self.buffer_offset+self.buffer_len)], buf, action); 103 | self.buffer_offset += result.bytes_read; 104 | self.buffer_len -= result.bytes_read; 105 | 106 | let stream_end = match result.ret { 107 | Ok(lzma_ret::LzmaStreamEnd) => true, 108 | Ok(_) => false, 109 | Err(err) => return Err(io::Error::new(io::ErrorKind::Other, err)), 110 | }; 111 | 112 | // We have to loop until we get at least 1 byte or EOF, because most users of 113 | // Read::read assume that a return value of 0 is EOF. 114 | if stream_end || result.bytes_written > 0 { 115 | return Ok(result.bytes_written); 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/writer.rs: -------------------------------------------------------------------------------- 1 | //! This module implements `LzmaWriter`. 2 | //! 3 | //! `LzmaWriter` implements the LZMA (XZ) compression/decompression algorithm as a generic 4 | //! Writer. In other words, it behaves similar to BufWriter. Instead of buffering the `Write` 5 | //! object passed to it, `LzmaWriter` applies compression or decompression to it as it's write. 6 | //! 7 | //! 8 | //! # Examples 9 | //! 10 | //! ```no_run 11 | //! use lzma::LzmaWriter; 12 | //! use std::io::prelude::*; 13 | //! use std::fs::File; 14 | //! 15 | //! let f = File::create("foo.xz").unwrap(); 16 | //! let mut f = LzmaWriter::new_compressor(f, 6).unwrap(); 17 | //! 18 | //! write!(f, "It's a small world!").unwrap(); 19 | //! f.finish().unwrap(); 20 | //! ``` 21 | 22 | use std::io::{self, Write}; 23 | use lzma_sys::*; 24 | use std; 25 | use error::LzmaError; 26 | use ::Direction; 27 | use lzma_stream_wrapper::{LzmaStreamWrapper, LzmaCodeResult}; 28 | 29 | 30 | const DEFAULT_BUF_SIZE: usize = 4 * 1024; 31 | 32 | 33 | pub struct LzmaWriter { 34 | inner: T, 35 | stream: LzmaStreamWrapper, 36 | buffer: Vec, 37 | direction: Direction, 38 | } 39 | 40 | 41 | impl LzmaWriter { 42 | pub fn new_compressor(inner: T, preset: u32) -> Result, LzmaError> { 43 | LzmaWriter::with_capacity(DEFAULT_BUF_SIZE, inner, Direction::Compress, preset) 44 | } 45 | 46 | pub fn new_decompressor(inner: T) -> Result, LzmaError> { 47 | LzmaWriter::with_capacity(DEFAULT_BUF_SIZE, inner, Direction::Decompress, 0) 48 | } 49 | 50 | pub fn with_capacity(capacity: usize, inner: T, direction: Direction, preset: u32) -> Result, LzmaError> { 51 | let mut writer = LzmaWriter { 52 | inner, 53 | stream: LzmaStreamWrapper::new(), 54 | buffer: vec![0; capacity], 55 | direction, 56 | }; 57 | 58 | match writer.direction { 59 | Direction::Compress => { 60 | writer.stream.easy_encoder(preset, lzma_check::LzmaCheckCrc64)? 61 | }, 62 | Direction::Decompress => { 63 | writer.stream.stream_decoder(std::u64::MAX, 0)? 64 | }, 65 | } 66 | 67 | Ok(writer) 68 | } 69 | } 70 | 71 | impl LzmaWriter { 72 | /// Finalizes the LZMA stream so that it finishes compressing or decompressing. 73 | /// 74 | /// This *must* be called after all writing is done to ensure the last pieces of the compressed 75 | /// or decompressed stream get written out. 76 | pub fn finish(mut self) -> Result { 77 | loop { 78 | match self.lzma_code_and_write(&[], lzma_action::LzmaFinish) { 79 | Ok(LzmaCodeResult { 80 | ret: Ok(lzma_ret::LzmaStreamEnd), 81 | bytes_read: _, 82 | bytes_written: _, 83 | }) => break, 84 | Ok(_) => continue, 85 | Err(err) => return Err(err), 86 | } 87 | } 88 | 89 | Ok(self.inner) 90 | } 91 | 92 | #[allow(clippy::question_mark)] 93 | fn lzma_code_and_write(&mut self, input: &[u8], action: lzma_action) -> Result { 94 | let result = self.stream.code(input, &mut self.buffer, action); 95 | if let Err(err) = result.ret { 96 | return Err(err); 97 | } 98 | 99 | if result.bytes_written > 0 { 100 | Write::write_all(&mut self.inner, &self.buffer[..result.bytes_written])?; 101 | } 102 | 103 | Ok(result) 104 | } 105 | } 106 | 107 | 108 | impl Write for LzmaWriter { 109 | fn write(&mut self, buf: &[u8]) -> io::Result { 110 | // Loop until at least one byte from buf was consumed in order to be 111 | // compliant with std::io::Write trait API. 112 | loop { 113 | match self.lzma_code_and_write(buf, lzma_action::LzmaRun) { 114 | Ok(result) => if result.bytes_read == 0 && result.bytes_written > 0 { 115 | continue 116 | } else { 117 | // If result.bytes_read is zero, then neither was something 118 | // written nor read. This indicates, something went wrong. 119 | return Ok(result.bytes_read) 120 | }, 121 | Err(LzmaError::Io(err)) => return Err(err), 122 | Err(err) => return Err(io::Error::new(io::ErrorKind::Other, err)), 123 | } 124 | } 125 | } 126 | 127 | fn flush(&mut self) -> io::Result<()> { 128 | self.inner.flush() 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /tests/test.rs: -------------------------------------------------------------------------------- 1 | extern crate lzma; 2 | 3 | use lzma::error::LzmaError; 4 | use std::io::{Read, Cursor, Write}; 5 | use std::thread; 6 | 7 | 8 | // A large text file used for testing 9 | const TEST_STRING: &'static str = include_str!("test_file.txt"); 10 | // Should be test_file.txt compressed in the legacy lzma format. 11 | const TEST_LEGACY_DATA: &'static [u8] = include_bytes!("test_file.lzma"); 12 | 13 | 14 | #[test] 15 | fn simple_compress_decompress() { 16 | let mut compressed = lzma::compress(&TEST_STRING.to_string().into_bytes(), 6).unwrap(); 17 | let decompressed = String::from_utf8(lzma::decompress(&mut compressed).unwrap()).unwrap(); 18 | 19 | assert!(compressed.len() < TEST_STRING.len()); 20 | assert_eq!(TEST_STRING, decompressed); 21 | } 22 | 23 | 24 | #[test] 25 | fn extreme() { 26 | let mut compressed = lzma::compress(&TEST_STRING.to_string().into_bytes(), 9).unwrap(); 27 | let mut extreme_compressed = lzma::compress(&TEST_STRING.to_string().into_bytes(), 9 | lzma::EXTREME_PRESET).unwrap(); 28 | let decompressed = String::from_utf8(lzma::decompress(&mut compressed).unwrap()).unwrap(); 29 | let extreme_decompressed = String::from_utf8(lzma::decompress(&mut extreme_compressed).unwrap()).unwrap(); 30 | 31 | // TODO: This test is not great. We just want to know if the EXTREME_PRESET flag is working. 32 | // It might occur that the len's are equal because EXTREME wasn't able to compress more (or less). 33 | assert!(extreme_compressed.len() != compressed.len()); 34 | assert_eq!(TEST_STRING, decompressed); 35 | assert_eq!(TEST_STRING, extreme_decompressed); 36 | } 37 | 38 | 39 | #[test] 40 | fn reader_wormhole() { 41 | let compressor = lzma::LzmaReader::new_compressor(Cursor::new(TEST_STRING), 5).unwrap(); 42 | let mut decompressor = lzma::LzmaReader::new_decompressor(compressor).unwrap(); 43 | let mut s = String::new(); 44 | 45 | decompressor.read_to_string(&mut s).unwrap(); 46 | 47 | assert_eq!(TEST_STRING, s); 48 | } 49 | 50 | 51 | #[test] 52 | fn writer_wormhole() { 53 | let mut output = vec![0u8; 0]; 54 | { 55 | let decompressor = lzma::LzmaWriter::new_decompressor(&mut output).unwrap(); 56 | let mut compressor = lzma::LzmaWriter::new_compressor(decompressor, 2).unwrap(); 57 | 58 | write!(compressor, "{}", TEST_STRING).unwrap(); 59 | compressor.finish().unwrap(); 60 | } 61 | 62 | assert_eq!(TEST_STRING, String::from_utf8(output).unwrap()); 63 | } 64 | 65 | 66 | #[test] 67 | fn truncation_causes_error() { 68 | let mut compressed = lzma::compress(&"Like tears in rain".to_string().into_bytes(), 6).unwrap(); 69 | let bad_len = compressed.len() - 1; 70 | compressed.truncate(bad_len); 71 | match lzma::decompress(&mut compressed) { 72 | Err(lzma::LzmaError::Io(err)) => { 73 | match *err.get_ref().unwrap().downcast_ref::().unwrap() { 74 | LzmaError::Buf => (), 75 | _ => panic!("Decompressing a truncated buffer should return an LzmaError::Buf error"), 76 | } 77 | }, 78 | _ => panic!("Decompressing a truncated buffer should return an LzmaError::Buf error"), 79 | } 80 | } 81 | 82 | 83 | // Test to make sure that LzmaReader implements the Send trait correctly 84 | #[test] 85 | fn reader_thread_send() { 86 | let compressor = lzma::LzmaReader::new_compressor(Cursor::new(TEST_STRING), 5).unwrap(); 87 | let mut decompressor = lzma::LzmaReader::new_decompressor(compressor).unwrap(); 88 | 89 | let output = thread::spawn(move || { 90 | let mut s = String::new(); 91 | decompressor.read_to_string(&mut s).unwrap(); 92 | s 93 | 94 | }).join().unwrap(); 95 | 96 | assert_eq!(TEST_STRING, output); 97 | } 98 | 99 | 100 | // Decompressing the test string in here causes LZMA to return 0 (bytes consumed from input) occasionally. 101 | // Good for testing that our Writer implementation handles that correctly. 102 | #[test] 103 | fn test_string_1() { 104 | let input = b"\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x12\x00\x00\x00\x23\xb8\x87\x2c\xe1\x20\xff\x00\x4c\x5d\x00\x30\xef\xfb\xbf\xfe\xa3\xb1\x5e\xe5\xf8\x3f\xb2\xaa\x26\x55\xf8\x68\x70\x41\x70\x15\x0f\x8d\xfd\x1e\x4c\x1b\x8a\x42\xb7\x19\xf4\x69\x18\x71\xae\x66\x23\x8a\x8a\x4d\x2f\xa3\x0d\xd9\x7f\xa6\xe3\x8c\x23\x11\x53\xe0\x59\x18\xc5\x75\x8a\xe2\x77\xf8\xb6\x94\x7f\x0c\x6a\xc0\xde\x74\x49\x64\xe2\xe8\x22\xa0\xfd\x00\xb3\xfd\x27\xbc\xcd\x64\x62\xb1\x00\x01\x68\x80\xc2\x04\x00\x00\x29\x67\x59\x4d\xb1\xc4\x67\xfb\x02\x00\x00\x00\x00\x04\x59\x5a"; 105 | let mut writer = lzma::LzmaWriter::new_decompressor(Vec::new()).unwrap(); 106 | writer.write_all(input).unwrap(); 107 | let buffer = writer.finish().unwrap(); 108 | assert_eq!(buffer.len(), 73984); // Original string is 73,984 * b'a'. 109 | } 110 | 111 | 112 | // Test that we can decompress a legacy .lzma file 113 | #[test] 114 | fn test_legacy_format() { 115 | let decompressed = lzma::decompress(TEST_LEGACY_DATA).unwrap(); 116 | assert_eq!(decompressed, TEST_STRING.as_bytes()); 117 | } -------------------------------------------------------------------------------- /tests/test_file.lzma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fpgaminer/rust-lzma/1dd93581379c455fe2813ec72ca27c334b542669/tests/test_file.lzma --------------------------------------------------------------------------------