├── testdata ├── good │ ├── empty │ ├── block_raw │ ├── block_rle │ ├── frame_nosum │ ├── frame_skip │ ├── block_many │ ├── frame_many │ ├── block_comp_offs_n │ ├── block_comp_endlit │ ├── block_comp_manyseqs │ ├── block_comp_offs_1 │ ├── empty.zst │ ├── block_many.zst │ ├── block_raw.zst │ ├── block_rle.zst │ ├── frame_many.zst │ ├── frame_skip.zst │ ├── frame_nosum.zst │ ├── block_comp_endlit.zst │ ├── block_comp_lithead_2B │ ├── block_comp_offs_1.zst │ ├── block_comp_offs_n.zst │ ├── block_comp_manyseqs.zst │ ├── block_comp_lithead_2B.zst │ ├── block_comp_offs_overlap.zst │ └── block_comp_offs_overlap ├── bad │ ├── frame_skip_nodata.zst │ ├── frame_skip_nosize.zst │ ├── frame_badmagic │ ├── block_noheader │ ├── block_nolast │ ├── frame_badsum │ ├── frame_noheader │ ├── frame_nosum │ ├── block_raw_nodata │ ├── frame_bigcontsize │ ├── frame_nocontsize │ ├── frame_resvbit │ ├── frame_skip_nodata │ ├── frame_skip_nosize │ ├── frame_nosum.zst │ ├── block_noheader.zst │ ├── block_nolast.zst │ ├── frame_badmagic.zst │ ├── frame_badsum.zst │ ├── frame_noheader.zst │ ├── frame_resvbit.zst │ ├── block_raw_nodata.zst │ ├── frame_nocontsize.zst │ └── frame_bigcontsize.zst ├── large │ ├── Zeros-100KiB.zst │ └── Zeros-10MiB.zst └── verify_inputs ├── .gitignore ├── tools.go ├── go.mod ├── go.sum ├── .travis.yml ├── consts.wuffs ├── test ├── LICENSE ├── README.md ├── wuffsmain.c └── decode.wuffs /testdata/good/empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/good/block_raw: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /testdata/good/block_rle: -------------------------------------------------------------------------------- 1 | AAAAAAAAA -------------------------------------------------------------------------------- /testdata/good/frame_nosum: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /testdata/good/frame_skip: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_skip_nodata.zst: -------------------------------------------------------------------------------- 1 | _*M -------------------------------------------------------------------------------- /testdata/bad/frame_skip_nosize.zst: -------------------------------------------------------------------------------- 1 | _*M -------------------------------------------------------------------------------- /testdata/good/block_many: -------------------------------------------------------------------------------- 1 | AAAAAAAAABBBBBBBBB -------------------------------------------------------------------------------- /testdata/good/frame_many: -------------------------------------------------------------------------------- 1 | foo 2 | foo 3 | -------------------------------------------------------------------------------- /testdata/bad/frame_badmagic: -------------------------------------------------------------------------------- 1 | not a zstd frame 2 | -------------------------------------------------------------------------------- /testdata/bad/block_noheader: -------------------------------------------------------------------------------- 1 | missing block header 2 | -------------------------------------------------------------------------------- /testdata/bad/block_nolast: -------------------------------------------------------------------------------- 1 | missing block header 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_badsum: -------------------------------------------------------------------------------- 1 | frame xxhash mismatch 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_noheader: -------------------------------------------------------------------------------- 1 | missing frame header 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_nosum: -------------------------------------------------------------------------------- 1 | missing frame xxhash 2 | -------------------------------------------------------------------------------- /testdata/bad/block_raw_nodata: -------------------------------------------------------------------------------- 1 | missing raw block content 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_bigcontsize: -------------------------------------------------------------------------------- 1 | zstd window size is too big 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_nocontsize: -------------------------------------------------------------------------------- 1 | missing frame content size 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_resvbit: -------------------------------------------------------------------------------- 1 | zstd frame reserved bit was set 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_skip_nodata: -------------------------------------------------------------------------------- 1 | missing skippable frame data 2 | -------------------------------------------------------------------------------- /testdata/bad/frame_skip_nosize: -------------------------------------------------------------------------------- 1 | missing skippable frame size 2 | -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_n: -------------------------------------------------------------------------------- 1 | 0123456789 2 | 0123456789 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /*.c 2 | /*.h 3 | !/wuffsmain.c 4 | /zstd 5 | /got 6 | -------------------------------------------------------------------------------- /testdata/good/block_comp_endlit: -------------------------------------------------------------------------------- 1 | 0123456789qwe 2 | 0123456789abc 3 | -------------------------------------------------------------------------------- /testdata/good/block_comp_manyseqs: -------------------------------------------------------------------------------- 1 | 000000000000 2 | 111111111111 3 | -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_1: -------------------------------------------------------------------------------- 1 | 00000000000000000000000000000000000000000000000000 2 | -------------------------------------------------------------------------------- /testdata/good/empty.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/empty.zst -------------------------------------------------------------------------------- /testdata/bad/frame_nosum.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_nosum.zst -------------------------------------------------------------------------------- /testdata/good/block_many.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_many.zst -------------------------------------------------------------------------------- /testdata/good/block_raw.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_raw.zst -------------------------------------------------------------------------------- /testdata/good/block_rle.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_rle.zst -------------------------------------------------------------------------------- /testdata/good/frame_many.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/frame_many.zst -------------------------------------------------------------------------------- /testdata/good/frame_skip.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/frame_skip.zst -------------------------------------------------------------------------------- /testdata/bad/block_noheader.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/block_noheader.zst -------------------------------------------------------------------------------- /testdata/bad/block_nolast.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/block_nolast.zst -------------------------------------------------------------------------------- /testdata/bad/frame_badmagic.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_badmagic.zst -------------------------------------------------------------------------------- /testdata/bad/frame_badsum.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_badsum.zst -------------------------------------------------------------------------------- /testdata/bad/frame_noheader.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_noheader.zst -------------------------------------------------------------------------------- /testdata/bad/frame_resvbit.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_resvbit.zst -------------------------------------------------------------------------------- /testdata/good/frame_nosum.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/frame_nosum.zst -------------------------------------------------------------------------------- /testdata/large/Zeros-100KiB.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/large/Zeros-100KiB.zst -------------------------------------------------------------------------------- /testdata/large/Zeros-10MiB.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/large/Zeros-10MiB.zst -------------------------------------------------------------------------------- /testdata/bad/block_raw_nodata.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/block_raw_nodata.zst -------------------------------------------------------------------------------- /testdata/bad/frame_nocontsize.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_nocontsize.zst -------------------------------------------------------------------------------- /testdata/bad/frame_bigcontsize.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/bad/frame_bigcontsize.zst -------------------------------------------------------------------------------- /testdata/good/block_comp_endlit.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_endlit.zst -------------------------------------------------------------------------------- /testdata/good/block_comp_lithead_2B: -------------------------------------------------------------------------------- 1 | 0123456789qwertyuiopasdfghjklzxcvbnm 2 | 0123456789qwertyuiopasdfghjklzxcvbnm 3 | -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_1.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_offs_1.zst -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_n.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_offs_n.zst -------------------------------------------------------------------------------- /testdata/good/block_comp_manyseqs.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_manyseqs.zst -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | package tools 4 | 5 | import ( 6 | _ "github.com/google/wuffs/cmd/wuffs-c" 7 | ) 8 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module mvdan.cc/zstd 2 | 3 | require github.com/google/wuffs v0.1.1-0.20191116223715-191c6bc6f28e 4 | 5 | go 1.13 6 | -------------------------------------------------------------------------------- /testdata/good/block_comp_lithead_2B.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_lithead_2B.zst -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_overlap.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mvdan/zstd/HEAD/testdata/good/block_comp_offs_overlap.zst -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/wuffs v0.1.1-0.20191116223715-191c6bc6f28e h1:hCgErW5Xt0oWHmU31xBVZWd5uvDAfmkEqgYqgUH9N88= 2 | github.com/google/wuffs v0.1.1-0.20191116223715-191c6bc6f28e/go.mod h1:CXN6/+deEtlZkYTWeZJOHoXibYZ0HYO9qPmXSUcyFwI= 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.12.x 5 | 6 | matrix: 7 | include: 8 | - os: linux 9 | addons: 10 | apt: 11 | sources: 12 | - llvm-toolchain-trusty-5.0 13 | packages: 14 | - clang-format-5.0 15 | 16 | env: 17 | - GO111MODULE=on 18 | 19 | install: true 20 | 21 | script: 22 | - ./build 23 | - ./test 24 | -------------------------------------------------------------------------------- /consts.wuffs: -------------------------------------------------------------------------------- 1 | pri const frame_magic_number base.u32 = 0xFD2FB528 2 | 3 | pri const frame_magic_skip_first base.u32 = 0x184D2A50 4 | pri const frame_magic_skip_last base.u32 = 0x184D2A5F 5 | 6 | pri const fcs_field_sizes array[4] base.u8 = [0, 2, 4, 8] 7 | 8 | pri const block_type_raw base.u8 = 0 9 | pri const block_type_rle base.u8 = 1 10 | pri const block_type_compressed base.u8 = 2 11 | 12 | pri const min_window_size base.u64 = 1 << 10 // 1KiB 13 | pri const max_block_size base.u32 = 128 << 10 // 128KiB 14 | -------------------------------------------------------------------------------- /testdata/good/block_comp_offs_overlap: -------------------------------------------------------------------------------- 1 | 000000000000000000000000000000000000000000000000000000000000000 2 | 111111111111111111111111111111111111111111111111111111111111111 3 | 000000000000000000000000000000000000000000000000000000000000000 4 | 111111111111111111111111111111111111111111111111111111111111111 5 | 000000000000000000000000000000000000000000000000000000000000000 6 | 111111111111111111111111111111111111111111111111111111111111111 7 | 000000000000000000000000000000000000000000000000000000000000000 8 | 111111111111111111111111111111111111111111111111111111111111111 9 | -------------------------------------------------------------------------------- /test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | failed=false 4 | 5 | for f in testdata/good/*.zst; do 6 | want=${f%%.zst} 7 | ./zstd <$f >got 8 | if [[ "$f" == *block_comp* ]]; then 9 | echo "ignoring compressed block test" 10 | continue 11 | fi 12 | if [[ $? -ne 0 ]]; then 13 | echo "$f is not valid" 14 | echo 15 | failed=true 16 | continue 17 | fi 18 | # we must use files, as otherwise shell command substution and 19 | # vars are going to nuke whitespace and null characters 20 | diff=$(diff -u $want got) 21 | if [[ $? -ne 0 ]]; then 22 | echo "$f has mismatching output:" 23 | echo "$diff" | sed 1,2d 24 | echo 25 | failed=true 26 | continue 27 | fi 28 | done 29 | 30 | rm -f got 31 | if $failed; then 32 | exit 1 33 | fi 34 | -------------------------------------------------------------------------------- /testdata/verify_inputs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script uses the reference zstd implementation to verify that: 4 | # * all inputs in good/*.zst are valid 5 | # * all inputs in bad/*.zst are invalid 6 | 7 | failed=false 8 | 9 | for f in good/*.zst; do 10 | want=${f%%.zst} 11 | rm -f got 12 | out=$(zstd -d $f -o got 2>&1) 13 | if [[ $? -ne 0 ]]; then 14 | echo "$f is not valid:" 15 | echo $out | sed 's/^[^ ]* : //' 16 | echo 17 | failed=true 18 | continue 19 | fi 20 | # we must use files, as otherwise shell command substution and 21 | # vars are going to nuke whitespace and null characters 22 | diff=$(diff -u $want got) 23 | if [[ $? -ne 0 ]]; then 24 | echo "$f has mismatching output:" 25 | echo "$diff" | sed 1,2d 26 | echo 27 | failed=true 28 | continue 29 | fi 30 | done 31 | 32 | for f in bad/*.zst; do 33 | out=$(zstd -d $f -o /dev/null 2>&1) 34 | if [[ $? -eq 0 ]]; then 35 | echo "$f is valid" 36 | failed=true 37 | fi 38 | done 39 | 40 | rm -f got 41 | if $failed; then 42 | exit 1 43 | fi 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, Daniel Martí. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of the copyright holder nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zstd 2 | 3 | An implementation from scratch of [Zstandard] in [Wuffs]. It is being 4 | developed following the published [spec]. 5 | 6 | This is very much a work in progress, so it is not ready for use. 7 | 8 | To build a simple `zstd` binary that will use stdin and stdout: 9 | 10 | ./build 11 | 12 | And to test it with the input/output cases in `testdata`: 13 | 14 | ./test 15 | 16 | ### Why? 17 | 18 | Writing a decoder in Wuffs takes more time, but the end result is an 19 | implementation that is safe and can be used in many languages without 20 | linking against C. 21 | 22 | For example, that would mean no cgo overhead with Go, and safer code for 23 | languages like Rust. Though that is somewhere in the future - see the 24 | roadmap. 25 | 26 | If you're after a zstd implementation that works today, use 27 | https://github.com/DataDog/zstd. 28 | 29 | ### Roadmap 30 | 31 | This is the current progress of the decoder. 32 | 33 | - [x] Zstandard frames 34 | - [x] Raw blocks 35 | - [x] RLE blocks 36 | - [ ] Compressed blocks 37 | - [ ] Literals section 38 | - [ ] Raw literals block 39 | - [ ] RLE literals block 40 | - [ ] Compressed literals block 41 | - [ ] Treeless literals block 42 | - [ ] Sequences section 43 | - [ ] Predefined mode 44 | - [ ] RLE mode 45 | - [ ] Repeat mode 46 | - [ ] FSE compression mode 47 | - [ ] Sequence execution 48 | - [ ] Repeat offsets 49 | - [ ] Other offsets 50 | - [ ] XXH64 frame content checksum 51 | - [x] Skippable frames 52 | - [ ] Dictionaries 53 | 54 | These items are required for a stable 1.0 release: 55 | 56 | - [ ] Wuffs 1.0 release 57 | - [ ] Go support in Wuffs (generating a Go zstd library) 58 | - [ ] Full zstd decoder implemented 59 | 60 | [Zstandard]: https://facebook.github.io/zstd/ 61 | [Wuffs]: https://github.com/google/wuffs 62 | [spec]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md 63 | -------------------------------------------------------------------------------- /wuffsmain.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "decode.h" 5 | 6 | #ifndef DST_BUFFER_SIZE 7 | #define DST_BUFFER_SIZE (128 * 1024) 8 | #endif 9 | 10 | #ifndef SRC_BUFFER_SIZE 11 | #define SRC_BUFFER_SIZE (128 * 1024) 12 | #endif 13 | 14 | uint8_t dst_buffer[DST_BUFFER_SIZE]; 15 | uint8_t src_buffer[SRC_BUFFER_SIZE]; 16 | 17 | // ignore_return_value suppresses errors from -Wall -Werror. 18 | static void ignore_return_value(int ignored) {} 19 | 20 | static const char* decode() { 21 | wuffs_zstd__decoder dec; 22 | const char* status = 23 | wuffs_zstd__decoder__initialize(&dec, sizeof dec, WUFFS_VERSION, 0); 24 | if (status) { 25 | return status; 26 | } 27 | 28 | wuffs_base__io_buffer dst; 29 | dst.data.ptr = dst_buffer; 30 | dst.data.len = DST_BUFFER_SIZE; 31 | dst.meta.wi = 0; 32 | dst.meta.ri = 0; 33 | dst.meta.pos = 0; 34 | dst.meta.closed = false; 35 | 36 | wuffs_base__io_buffer src; 37 | src.data.ptr = src_buffer; 38 | src.data.len = SRC_BUFFER_SIZE; 39 | src.meta.wi = 0; 40 | src.meta.ri = 0; 41 | src.meta.pos = 0; 42 | src.meta.closed = false; 43 | 44 | while (true) { 45 | const int stdin_fd = 0; 46 | ssize_t n = 47 | read(stdin_fd, src.data.ptr + src.meta.wi, src.data.len - src.meta.wi); 48 | if (n < 0) { 49 | if (errno != EINTR) { 50 | return strerror(errno); 51 | } 52 | continue; 53 | } 54 | src.meta.wi += n; 55 | if (n == 0) { 56 | src.meta.closed = true; 57 | } 58 | 59 | while (true) { 60 | status = wuffs_zstd__decoder__decode( 61 | &dec, &dst, &src); 62 | 63 | if (dst.meta.wi) { 64 | // TODO: handle EINTR and other write errors; see "man 2 write". 65 | const int stdout_fd = 1; 66 | ignore_return_value(write(stdout_fd, dst_buffer, dst.meta.wi)); 67 | dst.meta.ri = dst.meta.wi; 68 | wuffs_base__io_buffer__compact(&dst); 69 | } 70 | 71 | if (status == wuffs_base__suspension__short_read) { 72 | break; 73 | } 74 | if (status == wuffs_base__suspension__short_write) { 75 | continue; 76 | } 77 | return status; 78 | } 79 | 80 | wuffs_base__io_buffer__compact(&src); 81 | if (src.meta.wi == src.data.len) { 82 | return "internal error: no I/O progress possible"; 83 | } 84 | } 85 | } 86 | 87 | int fail(const char* msg) { 88 | const int stderr_fd = 2; 89 | ignore_return_value(write(stderr_fd, msg, strnlen(msg, 4095))); 90 | ignore_return_value(write(stderr_fd, "\n", 1)); 91 | return 1; 92 | } 93 | 94 | int main(int argc, char** argv) { 95 | const char* status = decode(); 96 | int status_code = status ? fail(status) : 0; 97 | 98 | return status_code; 99 | } 100 | -------------------------------------------------------------------------------- /decode.wuffs: -------------------------------------------------------------------------------- 1 | pub struct decoder?( 2 | window_size : base.u64, 3 | ) 4 | 5 | pub status "?not a frame" 6 | pub status "?frame reserved bit was set" 7 | pub status "?reserved block type found" 8 | pub status "?block size is larger than 128KiB" 9 | pub status "?block size is larger than window size" 10 | 11 | pub status "?TODO" 12 | 13 | pub func decoder.decode?(dst: base.io_writer, src: base.io_reader) { 14 | var z : base.status 15 | while args.src.available() > 0 { 16 | z =? this.decode_frame?(dst: args.dst, src: args.src) 17 | if z.is_suspension() { 18 | yield? z 19 | } 20 | if z.is_error() { 21 | return z 22 | } 23 | } 24 | } 25 | 26 | pri func decoder.decode_frame?(dst: base.io_writer, src: base.io_reader) { 27 | var magic : base.u32 28 | var frame_size : base.u32 29 | var frame_header : base.u8 30 | var fcs_flag : base.u8[..= 0x3] 31 | var fcs_field_size : base.u8 32 | var single_segment : base.bool 33 | var frame_cont_size : base.u64 34 | var hashing : base.bool 35 | var dict_id_flag : base.u8 36 | var block_header : base.u32[..= 0xFFFFFF] 37 | var is_last_block : base.bool 38 | var z : base.status 39 | var sum : base.u32 40 | 41 | magic = args.src.read_u32le?() 42 | if (magic >= frame_magic_skip_first) and (magic <= frame_magic_skip_last) { 43 | frame_size = args.src.read_u32le?() 44 | args.src.skip?(n: frame_size) 45 | return ok 46 | } 47 | if magic <> frame_magic_number { 48 | return "?not a frame" 49 | } 50 | 51 | frame_header = args.src.read_u8?() 52 | 53 | fcs_flag = frame_header >> 6 54 | fcs_field_size = fcs_field_sizes[fcs_flag] 55 | 56 | single_segment = ((frame_header >> 5) & 1) == 1 57 | if (fcs_flag == 0) and single_segment { 58 | fcs_field_size = 1 59 | } 60 | 61 | if fcs_field_size == 1 { 62 | frame_cont_size = args.src.read_u8_as_u64?() 63 | } else if fcs_field_size == 2 { 64 | frame_cont_size = args.src.read_u16le_as_u64?() 65 | frame_cont_size += 256 66 | } else if fcs_field_size == 4 { 67 | frame_cont_size = args.src.read_u32le_as_u64?() 68 | } else if fcs_field_size == 8 { 69 | frame_cont_size = args.src.read_u64le?() 70 | } else { 71 | return "?TODO" // prove this can't happen? 72 | } 73 | 74 | this.window_size = min_window_size 75 | if not single_segment { 76 | return "?TODO" 77 | } else { 78 | this.window_size = frame_cont_size 79 | } 80 | 81 | if ((frame_header >> 3) & 1) == 1 { 82 | return "?frame reserved bit was set" 83 | } 84 | 85 | hashing = ((frame_header >> 2) & 1) == 1 86 | 87 | dict_id_flag = frame_header & 3 88 | if dict_id_flag > 0 { 89 | return "?TODO" // read and use dictionary ID 90 | } 91 | 92 | while true { 93 | block_header = args.src.read_u24le_as_u32?() 94 | 95 | is_last_block = (block_header & 1) == 1 96 | z =? this.decode_block?(dst: args.dst, src: args.src, header: block_header) 97 | if z.is_suspension() { 98 | yield? z 99 | } 100 | if z.is_error() { 101 | return z 102 | } 103 | if is_last_block { 104 | break 105 | } 106 | } 107 | if hashing { 108 | sum = args.src.read_u32le?() 109 | // TODO: check the hash 110 | } 111 | } 112 | 113 | pri func decoder.decode_block?(dst: base.io_writer, src: base.io_reader, header: base.u32[..= 0xFFFFFF]) { 114 | var block_size : base.u32 115 | var block_type : base.u8 116 | var b : base.u8 117 | var i : base.u32 118 | 119 | block_size = args.header >> 3 120 | if block_size > max_block_size { 121 | return "?block size is larger than 128KiB" 122 | } 123 | if (block_size as base.u64) > this.window_size { 124 | return "?block size is larger than window size" 125 | } 126 | block_type = ((args.header >> 1) & 3) as base.u8 127 | 128 | if block_type == block_type_raw { 129 | args.dst.copy_n_from_reader!(n: block_size, r: args.src) 130 | } else if block_type == block_type_rle { 131 | b = args.src.read_u8?() 132 | while i < block_size, 133 | inv block_size <= max_block_size { 134 | 135 | // TODO: hopefully clean up once this issue is 136 | // fixed: https://github.com/google/wuffs/issues/11 137 | assert i <= max_block_size via "a <= b: a <= c; c <= b"(c: block_size) 138 | assert i <= 131072 via "a <= b: a <= c; c == b"(c: max_block_size) 139 | 140 | args.dst.write_u8?(a: b) 141 | i += 1 142 | } 143 | } else if block_type == block_type_compressed { 144 | return "?TODO" 145 | } else { 146 | return "?reserved block type found" 147 | } 148 | } 149 | --------------------------------------------------------------------------------