├── .editorconfig ├── .github └── workflows │ ├── amd64_linux_clang.yml │ ├── amd64_linux_gcc.yml │ ├── amd64_linux_tcc.yml │ ├── amd64_macos_clang.yml │ ├── amd64_macos_gcc.yml │ ├── amd64_windows_clang.yml │ ├── amd64_windows_gcc.yml │ ├── amd64_windows_tcc.yml │ ├── amd64_windows_vs2019.yml │ ├── arm64_linux_gcc.yml │ ├── armhf_linux_gcc.yml │ ├── asan.yml │ ├── codestyle.yml │ ├── i586_dos_gcc_cross.yml │ ├── i686_linux_clang.yml │ ├── i686_linux_gcc.yml │ ├── i686_windows_tcc.yml │ ├── i686_windows_vs2019.yml │ ├── mips64_linux_gcc.yml │ ├── mips64el_linux_gcc.yml │ ├── mips_linux_gcc.yml │ ├── mipsel_linux_gcc.yml │ ├── powerpc_linux_gcc.yml │ ├── ppc64_linux_gcc.yml │ ├── ppc64le_linux_gcc.yml │ ├── riscv64_linux_gcc.yml │ ├── riscv_linux_gcc.yml │ └── s390x_linux_gcc.yml ├── .gitmodules ├── ChangeLog ├── LICENSE.MIT ├── README.md ├── _config.yml ├── examples ├── 6pack.c ├── 6unpack.c ├── Makefile └── Makefile.win ├── fastlz.c ├── fastlz.h ├── tests ├── Makefile ├── Makefile.win ├── refimpl.c └── test_roundtrip.c └── tools └── format-code.sh /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | end_of_line = lf 9 | insert_final_newline = true 10 | 11 | [*.{c,h}] 12 | indent_size = 2 13 | indent_style = space 14 | 15 | [Makefile] 16 | indent_style = tab 17 | 18 | [Makefile.win] 19 | indent_style = tab -------------------------------------------------------------------------------- /.github/workflows/amd64_linux_clang.yml: -------------------------------------------------------------------------------- 1 | name: amd64_linux_clang 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_linux_clang: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: clang 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt install -y make clang 21 | - run: clang --version 22 | - run: cd tests && make roundtrip 23 | name: Perform round-trip tests 24 | - name: 'Build examples: 6pack and 6unpack' 25 | run: cd examples && make 26 | - name: 'Run examples: 6pack and 6unpack' 27 | run: | 28 | cd examples 29 | ./6pack -v 30 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 31 | ./6unpack -v 32 | ./6unpack archive.6pk 33 | -------------------------------------------------------------------------------- /.github/workflows/amd64_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: amd64_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt install -y make gcc 21 | - run: gcc --version 22 | - run: cd tests && make roundtrip 23 | name: Perform round-trip tests 24 | - name: 'Build examples: 6pack and 6unpack' 25 | run: cd examples && make 26 | - name: 'Run examples: 6pack and 6unpack' 27 | run: | 28 | cd examples 29 | ./6pack -v 30 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 31 | ./6unpack -v 32 | ./6unpack archive.6pk 33 | -------------------------------------------------------------------------------- /.github/workflows/amd64_linux_tcc.yml: -------------------------------------------------------------------------------- 1 | name: amd64_linux_tcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_linux_tcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: tcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt install -y make tcc 21 | - run: tcc -v 22 | - run: cd tests && make roundtrip 23 | name: Perform round-trip tests 24 | - name: 'Build examples: 6pack and 6unpack' 25 | run: cd examples && make 26 | - name: 'Run examples: 6pack and 6unpack' 27 | run: | 28 | cd examples 29 | ./6pack -v 30 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 31 | ./6unpack -v 32 | ./6unpack archive.6pk 33 | -------------------------------------------------------------------------------- /.github/workflows/amd64_macos_clang.yml: -------------------------------------------------------------------------------- 1 | name: amd64_macos_clang 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_macos_clang: 8 | runs-on: macos-latest 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Retrieve test compression corpus 13 | run: | 14 | git clone https://github.com/ariya/compression-corpus.git 15 | cd compression-corpus 16 | cd enwik 17 | unzip enwik8.zip 18 | - run: cc --version 19 | - run: cd tests && make roundtrip 20 | name: Perform round-trip tests 21 | - name: 'Build examples: 6pack and 6unpack' 22 | run: cd examples && make 23 | - name: 'Run examples: 6pack and 6unpack' 24 | run: | 25 | cd examples 26 | ./6pack -v 27 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 28 | ./6unpack -v 29 | ./6unpack archive.6pk 30 | -------------------------------------------------------------------------------- /.github/workflows/amd64_macos_gcc.yml: -------------------------------------------------------------------------------- 1 | name: amd64_macos_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_macos_gcc: 8 | runs-on: macos-12 9 | timeout-minutes: 10 10 | env: 11 | CC: gcc-9 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: brew install gcc@9 21 | - run: gcc-9 --version 22 | - run: cd tests && make roundtrip 23 | name: Perform round-trip tests 24 | - name: 'Build examples: 6pack and 6unpack' 25 | run: cd examples && make 26 | - name: 'Run examples: 6pack and 6unpack' 27 | run: | 28 | cd examples 29 | ./6pack -v 30 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 31 | ./6unpack -v 32 | ./6unpack archive.6pk 33 | -------------------------------------------------------------------------------- /.github/workflows/amd64_windows_clang.yml: -------------------------------------------------------------------------------- 1 | name: amd64_windows_clang 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_windows_clang: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | env: 11 | CC: clang 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - uses: msys2/setup-msys2@v2 21 | with: 22 | install: make mingw-w64-x86_64-clang 23 | - run: clang --version 24 | - run: cd tests && make roundtrip 25 | name: Perform round-trip tests 26 | - name: 'Build examples: 6pack and 6unpack' 27 | run: cd examples && make 28 | - name: 'Run examples: 6pack and 6unpack' 29 | run: | 30 | cd examples 31 | ./6pack -v 32 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 33 | ./6unpack -v 34 | ./6unpack archive.6pk 35 | -------------------------------------------------------------------------------- /.github/workflows/amd64_windows_gcc.yml: -------------------------------------------------------------------------------- 1 | name: amd64_windows_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_windows_gcc: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | env: 11 | CC: gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - uses: msys2/setup-msys2@v2 21 | with: 22 | install: gcc make 23 | - run: gcc --version 24 | - run: cd tests && make roundtrip 25 | name: Perform round-trip tests 26 | - name: 'Build examples: 6pack and 6unpack' 27 | run: cd examples && make 28 | - name: 'Run examples: 6pack and 6unpack' 29 | run: | 30 | cd examples 31 | ./6pack -v 32 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 33 | ./6unpack -v 34 | ./6unpack archive.6pk 35 | -------------------------------------------------------------------------------- /.github/workflows/amd64_windows_tcc.yml: -------------------------------------------------------------------------------- 1 | name: amd64_windows_tcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_windows_tcc: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Retrieve test compression corpus 13 | run: | 14 | git clone https://github.com/ariya/compression-corpus.git 15 | cd compression-corpus 16 | cd enwik 17 | unzip enwik8.zip 18 | - name: Install tcc 19 | run: | 20 | echo "5a3979bd5044b795547a4948a5625a12 tcc.zip" > checksum.md5 21 | dos2unix checksum.md5 22 | curl -L -o tcc.zip https://archive.org/download/tinyccompiler/tcc-0.9.27-win32-bin.zip 23 | md5sum -c checksum.md5 && unzip -q tcc.zip 24 | - run: tcc\x86_64-win32-tcc.exe -v 25 | - run: cd tests && make roundtrip CC=..\tcc\x86_64-win32-tcc.exe 26 | name: Perform round-trip tests 27 | - name: 'Build examples: 6pack and 6unpack' 28 | run: cd examples && make CC=..\tcc\x86_64-win32-tcc.exe 29 | - name: 'Run examples: 6pack and 6unpack' 30 | run: | 31 | cd examples 32 | ./6pack -v 33 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 34 | ./6unpack -v 35 | ./6unpack archive.6pk 36 | -------------------------------------------------------------------------------- /.github/workflows/amd64_windows_vs2019.yml: -------------------------------------------------------------------------------- 1 | name: amd64_windows_vs2019 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | amd64_windows_vs2019: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Retrieve test compression corpus 13 | run: | 14 | git clone https://github.com/ariya/compression-corpus.git 15 | cd compression-corpus 16 | cd enwik 17 | unzip enwik8.zip 18 | - uses: ilammy/msvc-dev-cmd@v1 19 | - run: cl 20 | - run: cd tests && mingw32-make -f Makefile.win roundtrip 21 | name: Perform round-trip tests 22 | - name: 'Build examples: 6pack and 6unpack' 23 | run: cd examples && make -f Makefile.win 24 | - name: 'Run examples: 6pack and 6unpack' 25 | run: | 26 | cd examples 27 | ./6pack -v 28 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 29 | ./6unpack -v 30 | ./6unpack archive.6pk 31 | -------------------------------------------------------------------------------- /.github/workflows/arm64_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: arm64_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | arm64_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/aarch64-linux-musl-cross.tgz 25 | tar xzf aarch64-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-aarch64 ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-aarch64 ./6pack -v 39 | qemu-aarch64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-aarch64 ./6unpack -v 41 | qemu-aarch64 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/armhf_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: armhf_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | armhf_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/armel-linux-musleabihf-cross/bin/armel-linux-musleabihf-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O https://musl.cc/armel-linux-musleabihf-cross.tgz 25 | tar xzf armel-linux-musleabihf-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/armel-linux-musleabihf-cross/bin/armel-linux-musleabihf-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-arm ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-arm ./6pack -v 39 | qemu-arm ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-arm ./6unpack -v 41 | qemu-arm ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/asan.yml: -------------------------------------------------------------------------------- 1 | name: Address Sanitizer 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | asan: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt install -y make gcc 21 | - run: gcc --version 22 | - run: cd tests && make roundtrip 23 | name: Perform round-trip tests 24 | env: 25 | CFLAGS: "-g -fno-omit-frame-pointer -fsanitize=address" 26 | - run: cd tests && make roundtrip 27 | name: Perform round-trip tests with FASTLZ_USE_MEMMOVE=0 28 | env: 29 | CFLAGS: "-g -fno-omit-frame-pointer -fsanitize=address -DFASTLZ_USE_MEMMOVE=0" 30 | -------------------------------------------------------------------------------- /.github/workflows/codestyle.yml: -------------------------------------------------------------------------------- 1 | name: Code style 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | codestyle: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - run: sudo apt install -y clang-format-6.0 13 | name: Install clang-format 14 | - run: clang-format-6.0 --version 15 | - run: bash tools/format-code.sh 16 | name: Run code formatter 17 | - run: git diff 18 | - run: git diff --quiet HEAD 19 | name: Check if the styling guide is followed 20 | -------------------------------------------------------------------------------- /.github/workflows/i586_dos_gcc_cross.yml: -------------------------------------------------------------------------------- 1 | name: i586_dos_gcc_cross 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | i586_dos_gcc_cross: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 15 10 | env: 11 | CC: /opt/djgpp/bin/i586-pc-msdosdjgpp-gcc 12 | LDFLAGS: -static 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Retrieve test compression corpus 16 | run: | 17 | git clone https://github.com/ariya/compression-corpus.git 18 | cd compression-corpus 19 | cd enwik 20 | unzip enwik8.zip 21 | - run: sudo apt-get -y -qq update 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -OL https://github.com/andrewwutw/build-djgpp/releases/download/v3.3/djgpp-linux64-gcc1210.tar.bz2 25 | tar xf djgpp-linux64-gcc1210.tar.bz2 -C /opt 26 | - name: Verify compiler version 27 | run: /opt/djgpp/bin/i586-pc-msdosdjgpp-gcc --version 28 | - name: Install DOSEMU2 29 | run: | 30 | sudo add-apt-repository -y ppa:dosemu2/ppa 31 | sudo apt update -y 32 | sudo apt install -y dosemu2 33 | - run: dosemu --version 34 | - name: Perform round-trip tests 35 | run: | 36 | cd tests 37 | ln -s ../compression-corpus/ corpus 38 | make test_roundtrip TEST_ROUNDTRIP=testrr 39 | file ./testrr.exe 40 | dosemu -dumb -K . -t -E "testrr corpus/" 41 | - name: 'Build and run example: 6pack and 6unpack' 42 | run: | 43 | cd examples 44 | ln -s ../compression-corpus/enwik/enwik8.txt enwik8.txt 45 | make 46 | file ./6pack.exe 47 | dosemu -K . -t -E "6pack.exe -v" 48 | dosemu -K . -t -E "6pack.exe enwik8.txt archive.6pk" 49 | mv enwik8.txt enwik8.txt.orig 50 | dosemu -K . -t -E "6unpack.exe -v" 51 | dosemu -K . -t -E "6unpack.exe archive.6pk" 52 | ls -l enwik8* 53 | -------------------------------------------------------------------------------- /.github/workflows/i686_linux_clang.yml: -------------------------------------------------------------------------------- 1 | name: i686_linux_clang 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | i686_linux_clang: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: clang 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y make clang gcc-multilib qemu-user 22 | - run: clang --version 23 | - name: Perform round-trip tests 24 | run: | 25 | cd tests 26 | make test_roundtrip CFLAGS="-static -m32" 27 | file ./test_roundtrip 28 | qemu-i386 ./test_roundtrip 29 | - name: 'Build and run example: 6pack and 6unpack' 30 | run: | 31 | cd examples 32 | make CFLAGS="-static -m32" 33 | qemu-i386 ./6pack -v 34 | qemu-i386 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 35 | qemu-i386 ./6unpack -v 36 | qemu-i386 ./6unpack archive.6pk 37 | -------------------------------------------------------------------------------- /.github/workflows/i686_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: i686_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | i686_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/i686-linux-musl-cross/bin/i686-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O https://musl.cc/i686-linux-musl-cross.tgz 25 | tar xzf i686-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/i686-linux-musl-cross/bin/i686-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-i386 ./test_roundtrip 34 | - name: 'Build and run examples: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-i386 ./6pack -v 39 | qemu-i386 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-i386 ./6unpack -v 41 | qemu-i386 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/i686_windows_tcc.yml: -------------------------------------------------------------------------------- 1 | name: i686_windows_tcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | i686_windows_tcc: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Retrieve test compression corpus 13 | run: | 14 | git clone https://github.com/ariya/compression-corpus.git 15 | cd compression-corpus 16 | cd enwik 17 | unzip enwik8.zip 18 | - name: Install tcc 19 | run: | 20 | echo "D73CF66CEC8C761DE38C7A3D16C9EB0D tcc.zip" > checksum.md5 21 | dos2unix checksum.md5 22 | curl -L -o tcc.zip https://archive.org/download/tinyccompiler/tcc-0.9.27-win64-bin.zip 23 | md5sum -c checksum.md5 && unzip -q tcc.zip 24 | - run: tcc\i386-win32-tcc.exe -v 25 | - run: cd tests && make roundtrip CC=..\tcc\i386-win32-tcc.exe 26 | name: Perform round-trip tests 27 | - name: 'Build examples: 6pack and 6unpack' 28 | run: cd examples && make CC=..\tcc\i386-win32-tcc.exe 29 | - name: 'Run examples: 6pack and 6unpack' 30 | run: | 31 | cd examples 32 | ./6pack -v 33 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 34 | ./6unpack -v 35 | ./6unpack archive.6pk 36 | -------------------------------------------------------------------------------- /.github/workflows/i686_windows_vs2019.yml: -------------------------------------------------------------------------------- 1 | name: i686_windows_vs2019 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | i686_windows_vs2019: 8 | runs-on: windows-2019 9 | timeout-minutes: 10 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Retrieve test compression corpus 13 | run: | 14 | git clone https://github.com/ariya/compression-corpus.git 15 | cd compression-corpus 16 | cd enwik 17 | unzip enwik8.zip 18 | - uses: ilammy/msvc-dev-cmd@v1 19 | with: 20 | arch: x86 21 | - run: cl 22 | - run: cd tests && mingw32-make -f Makefile.win roundtrip 23 | name: Perform round-trip tests 24 | - name: 'Build examples: 6pack and 6unpack' 25 | run: cd examples && make -f Makefile.win 26 | - name: 'Run examples: 6pack and 6unpack' 27 | run: | 28 | cd examples 29 | ./6pack -v 30 | ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 31 | ./6unpack -v 32 | ./6unpack archive.6pk 33 | -------------------------------------------------------------------------------- /.github/workflows/mips64_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: mips64_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | mips64_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/mips64-linux-musl-cross/bin/mips64-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/mips64-linux-musl-cross.tgz 25 | tar xzf mips64-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/mips64-linux-musl-cross/bin/mips64-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-mips64 ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-mips64 ./6pack -v 39 | qemu-mips64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-mips64 ./6unpack -v 41 | qemu-mips64 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/mips64el_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: mips64el_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | mips64el_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/mips64el-linux-musl-cross/bin/mips64el-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/mips64el-linux-musl-cross.tgz 25 | tar xzf mips64el-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/mips64el-linux-musl-cross/bin/mips64el-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-mips64el ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-mips64el ./6pack -v 39 | qemu-mips64el ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-mips64el ./6unpack -v 41 | qemu-mips64el ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/mips_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: mips_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | mips_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/mips-linux-musl-cross/bin/mips-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/mips-linux-musl-cross.tgz 25 | tar xzf mips-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/mips-linux-musl-cross/bin/mips-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-mips ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-mips ./6pack -v 39 | qemu-mips ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-mips ./6unpack -v 41 | qemu-mips ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/mipsel_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: mipsel_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | mipsel_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/mipsel-linux-musl-cross/bin/mipsel-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/mipsel-linux-musl-cross.tgz 25 | tar xzf mipsel-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/mipsel-linux-musl-cross/bin/mipsel-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-mipsel ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-mipsel ./6pack -v 39 | qemu-mipsel ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-mipsel ./6unpack -v 41 | qemu-mipsel ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/powerpc_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: powerpc_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | powerpc_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/powerpc-linux-musl-cross/bin/powerpc-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/powerpc-linux-musl-cross.tgz 25 | tar xzf powerpc-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/powerpc-linux-musl-cross/bin/powerpc-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-ppc ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-ppc ./6pack -v 39 | qemu-ppc ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-ppc ./6unpack -v 41 | qemu-ppc ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/ppc64_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: ppc64_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | ppc64_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/powerpc64-linux-musl-cross/bin/powerpc64-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/powerpc64-linux-musl-cross.tgz 25 | tar xzf powerpc64-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/powerpc64-linux-musl-cross/bin/powerpc64-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-ppc64 ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-ppc64 ./6pack -v 39 | qemu-ppc64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-ppc64 ./6unpack -v 41 | qemu-ppc64 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/ppc64le_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: ppc64le_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | ppc64le_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/powerpc64le-linux-musl-cross/bin/powerpc64le-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/powerpc64le-linux-musl-cross.tgz 25 | tar xzf powerpc64le-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/powerpc64le-linux-musl-cross/bin/powerpc64le-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-ppc64le ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-ppc64le ./6pack -v 39 | qemu-ppc64le ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-ppc64le ./6unpack -v 41 | qemu-ppc64le ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/riscv64_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: riscv64_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | riscv64_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/riscv64-linux-musl-cross/bin/riscv64-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/riscv64-linux-musl-cross.tgz 25 | tar xzf riscv64-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/riscv64-linux-musl-cross/bin/riscv64-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-riscv64 ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-riscv64 ./6pack -v 39 | qemu-riscv64 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-riscv64 ./6unpack -v 41 | qemu-riscv64 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/riscv_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: riscv_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | riscv_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/riscv32-linux-musl-cross/bin/riscv32-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/riscv32-linux-musl-cross.tgz 25 | tar xzf riscv32-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/riscv32-linux-musl-cross/bin/riscv32-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-riscv32 ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-riscv32 ./6pack -v 39 | qemu-riscv32 ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-riscv32 ./6unpack -v 41 | qemu-riscv32 ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.github/workflows/s390x_linux_gcc.yml: -------------------------------------------------------------------------------- 1 | name: s390x_linux_gcc 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | s390x_linux_gcc: 8 | runs-on: ubuntu-20.04 9 | timeout-minutes: 10 10 | env: 11 | CC: /opt/s390x-linux-musl-cross/bin/s390x-linux-musl-gcc 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Retrieve test compression corpus 15 | run: | 16 | git clone https://github.com/ariya/compression-corpus.git 17 | cd compression-corpus 18 | cd enwik 19 | unzip enwik8.zip 20 | - run: sudo apt-get -y -qq update 21 | - run: sudo apt install -y qemu-user 22 | - name: Prepare cross-compiler 23 | run: | 24 | curl -O http://musl.cc/s390x-linux-musl-cross.tgz 25 | tar xzf s390x-linux-musl-cross.tgz -C /opt 26 | - name: Verify compiler version 27 | run: /opt/s390x-linux-musl-cross/bin/s390x-linux-musl-gcc --version 28 | - name: Perform round-trip tests 29 | run: | 30 | cd tests 31 | make test_roundtrip CFLAGS=-static 32 | file ./test_roundtrip 33 | qemu-s390x ./test_roundtrip 34 | - name: 'Build and run example: 6pack and 6unpack' 35 | run: | 36 | cd examples 37 | make CFLAGS=-static 38 | qemu-s390x ./6pack -v 39 | qemu-s390x ./6pack ../compression-corpus/enwik/enwik8.txt archive.6pk 40 | qemu-s390x ./6unpack -v 41 | qemu-s390x ./6unpack archive.6pk 42 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "compression-corpus"] 2 | path = compression-corpus 3 | url = https://github.com/ariya/compression-corpus.git 4 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2020-02-02: Version 0.5.0 2 | 3 | Minor speed improvement on the decompressor. 4 | Prevent memory violation when decompressing corrupted input. 5 | 6 | 2020-01-10: Version 0.4.0 7 | 8 | Only code & infrastructure clean-up, no new functionality. 9 | -------------------------------------------------------------------------------- /LICENSE.MIT: -------------------------------------------------------------------------------- 1 | FastLZ - Byte-aligned LZ77 compression library 2 | Copyright (C) 2005-2020 Ariya Hidayat 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) 2 | [![Code style](https://github.com/ariya/fastlz/workflows/Code%20style/badge.svg)](https://github.com/ariya/fastlz/actions) 3 | [![Address Sanitizer](https://github.com/ariya/fastlz/workflows/Address%20Sanitizer/badge.svg)](https://github.com/ariya/fastlz/actions) 4 | 5 | ## Overview 6 | 7 | FastLZ (MIT license) is an ANSI C/C90 implementation of [Lempel-Ziv 77 algorithm](https://en.wikipedia.org/wiki/LZ77_and_LZ78#LZ77) (LZ77) of lossless data compression. It is suitable to compress series of text/paragraphs, sequences of raw pixel data, or any other blocks of data with lots of repetition. It is not intended to be used on images, videos, and other formats of data typically already in an optimal compressed form. 8 | 9 | The focus for FastLZ is a very fast compression and decompression, doing that at the cost of the compression ratio. As an illustration, the comparison with zlib when compressing [enwik8](http://www.mattmahoney.net/dc/textdata.html) (also in [more details](https://github.com/inikep/lzbench)): 10 | 11 | ||Ratio|Compression|Decompression 12 | |--|--|--|--| 13 | |FastLZ |54.2%|159 MB/s|305 MB/s| 14 | |zlib -1|42.3%|50 MB/s|184 MB/s| 15 | |zlib -9|36.5%|11 MB/s|185 MB/s| 16 | 17 | FastLZ is used by many software products, from a number of games (such as [Death Stranding](https://en.wikipedia.org/wiki/Death_Stranding)) to various open-source projects ([Godot Engine](https://godotengine.org/), [Facebook HHVM](https://hhvm.com/), [Apache Traffic Server](https://trafficserver.apache.org/), [Calligra Office](https://www.calligra.org/), [OSv](http://osv.io/), [Netty](https://netty.io/), etc). It even serves as the basis for other compression projects like [BLOSC](https://blosc.org/). 18 | 19 | For other implementations of byte-aligned LZ77, take a look at [LZ4](https://lz4.github.io/lz4/), [Snappy](http://google.github.io/snappy/), [Density](https://github.com/centaurean/density), [LZO](http://www.oberhumer.com/opensource/lzo/), [LZF](http://oldhome.schmorp.de/marc/liblzf.html), [LZJB](https://en.wikipedia.org/wiki/LZJB), [LZRW](http://www.ross.net/compression/lzrw1.html), etc. 20 | 21 | ## Usage 22 | 23 | FastLZ can be used directly in any C/C++ applications. For other programming languages/environments, use the corresponding binding: 24 | 25 | * [Rust](https://crates.io/crates/fastlz), available on Crates: `cargo install fastlz` 26 | * [Python](https://pypi.org/project/fastlz/), available on PyPi: `pip install fastlz` 27 | * [JavaScript](https://www.npmjs.com/package/fastlz), available on npm: `npm install fastlz` 28 | * [Ruby](https://rubygems.org/gems/fastlz), available on Rubygems: `gem install fastlz` 29 | * Lua via [github.com/oneoo/lua-fastlz](https://github.com/oneoo/lua-fastlz) 30 | 31 | FastLZ consists of only two files: `fastlz.h` and `fastlz.c`. Just add these files to your project in order to use FastLZ. For the detailed information on the API to perform compression and decompression, see `fastlz.h`. 32 | 33 | For [Vcpkg](https://github.com/microsoft/vcpkg) users, FastLZ is [already available](https://github.com/microsoft/vcpkg): `vcpkg install fastlz`. 34 | 35 | A simple file compressor called `6pack` is included as an example on how to use FastLZ. The corresponding decompressor is `6unpack`. 36 | 37 | FastLZ supports any standard-conforming ANSI C/C90 compiler, including the popular ones such as [GCC](https://gcc.gnu.org/), [Clang](https://clang.llvm.org/), [Visual Studio](https://visualstudio.microsoft.com/vs/features/cplusplus/), and even [Tiny CC](https://bellard.org/tcc/). FastLZ works well on a number of architectures (32-bit and 64-bit, big endian and little endian), from Intel/AMD, PowerPC, System z, ARM, MIPS, and RISC-V. 38 | 39 | The continuous integration system runs an extensive set of compression-decompression round trips on the following systems: 40 | 41 | 42 | For more details, check the corresponding [GitHub Actions build logs](https://github.com/ariya/FastLZ/actions). 43 | 44 | | | | | | 45 | |----------------------|--------------------------------------------------------------------------------------------------------:|--------------------------------------------------------------------------------------------------:|--------------------------------------------------------------------------------------------:| 46 | | **amd64** | **Linux** | **Windows** | **macOS** | 47 | | GCC | ![amd64_linux_gcc](https://github.com/ariya/FastLZ/workflows/amd64_linux_gcc/badge.svg) | ![amd64_windows_gcc](https://github.com/ariya/FastLZ/workflows/amd64_windows_gcc/badge.svg) | ![amd64_macos_gcc](https://github.com/ariya/FastLZ/workflows/amd64_macos_gcc/badge.svg) | 48 | | Clang | ![amd64_linux_clang](https://github.com/ariya/FastLZ/workflows/amd64_linux_clang/badge.svg) | ![amd64_windows_clang](https://github.com/ariya/FastLZ/workflows/amd64_windows_clang/badge.svg) | ![amd64_macos_clang](https://github.com/ariya/FastLZ/workflows/amd64_macos_clang/badge.svg) | 49 | | TinyCC | ![amd64_linux_tcc](https://github.com/ariya/FastLZ/workflows/amd64_linux_tcc/badge.svg) | ![amd64_windows_tcc](https://github.com/ariya/FastLZ/workflows/amd64_windows_tcc/badge.svg) | | 50 | | VS 2019 | | ![amd64_windows_vs2019](https://github.com/ariya/FastLZ/workflows/amd64_windows_vs2019/badge.svg) | | 51 | | **i686** | **Linux** | **Windows** | **macOS** | 52 | | GCC | ![i686_linux_gcc](https://github.com/ariya/FastLZ/workflows/i686_linux_gcc/badge.svg) | | | 53 | | Clang | ![i686_linux_clang](https://github.com/ariya/FastLZ/workflows/i686_linux_clang/badge.svg) | | | 54 | | TinyCC | | ![i686_windows_tcc](https://github.com/ariya/FastLZ/workflows/i686_windows_tcc/badge.svg) | | 55 | | VS 2019 | | ![i686_windows_vs2019](https://github.com/ariya/FastLZ/workflows/i686_windows_vs2019/badge.svg) | | 56 | | **i586** | **Linux** | **DOS** | | 57 | | GCC | | ![i586_dos_gcc_cross](https://github.com/ariya/FastLZ/workflows/i586_dos_gcc_cross/badge.svg) | | 58 | | | **Linux** | | | 59 | | **powerpc** | | | | 60 | | GCC | ![powerpc_linux_gcc](https://github.com/ariya/FastLZ/workflows/powerpc_linux_gcc/badge.svg) | | | 61 | | **ppc64(le)** | | | | 62 | | GCC | ![ppc64_linux_gcc](https://github.com/ariya/FastLZ/workflows/ppc64_linux_gcc/badge.svg) | | | 63 | | GCC | ![ppc64le_linux_gcc](https://github.com/ariya/FastLZ/workflows/ppc64le_linux_gcc/badge.svg) | | | 64 | | **s390x** | | | | 65 | | GCC | ![s390x_linux_gcc](https://github.com/ariya/FastLZ/workflows/s390x_linux_gcc/badge.svg) | | | 66 | | **armhf** | | | | 67 | | GCC | ![armhf_linux_gcc](https://github.com/ariya/FastLZ/workflows/armhf_linux_gcc/badge.svg) | | | 68 | | **arm64** | | | | 69 | | GCC | ![arm64_linux_gcc](https://github.com/ariya/FastLZ/workflows/arm64_linux_gcc/badge.svg) | | | 70 | | **mips(el)** | | | | 71 | | GCC | ![mipsel_linux_gcc](https://github.com/ariya/FastLZ/workflows/mipsel_linux_gcc/badge.svg) | | | 72 | | GCC | ![mips_linux_gcc](https://github.com/ariya/FastLZ/workflows/mips_linux_gcc/badge.svg) | | | 73 | | **mips64(el)** | | | | 74 | | GCC | ![mips64el_linux_gcc](https://github.com/ariya/FastLZ/workflows/mips64el_linux_gcc/badge.svg) | | | 75 | | GCC | ![mips64_linux_gcc](https://github.com/ariya/FastLZ/workflows/mips64_linux_gcc/badge.svg) | | | 76 | | **riscv** | | | | 77 | | GCC | ![riscv_linux_gcc](https://github.com/ariya/FastLZ/workflows/riscv_linux_gcc/badge.svg) | | | 78 | | **riscv64** | | | | 79 | | GCC | ![riscv64_linux_gcc](https://github.com/ariya/FastLZ/workflows/riscv64_linux_gcc/badge.svg) | | | 80 | 81 | 82 | 83 | ## Block Format 84 | 85 | Let us assume that FastLZ compresses an array of bytes, called the _uncompressed block_, into another array of bytes, called the _compressed block_. To understand what will be stored in the compressed block, it is illustrative to demonstrate how FastLZ will _decompress_ the block to retrieve the original uncompressed block. 86 | 87 | The first 3-bit of the block, i.e. the 3 most-significant bits of the first byte, is the **block tag**. Currently the block tag determines the compression level used to produce the compressed block. 88 | 89 | |Block tag|Compression level| 90 | |---------|-----------------| 91 | | 0 | Level 1 | 92 | | 1 | Level 2 | 93 | 94 | The content of the block will vary depending on the compression level. 95 | 96 | ### Block Format for Level 1 97 | 98 | FastLZ Level 1 implements LZ77 compression algorithm with 8 KB sliding window and up to 264 bytes of match length. 99 | 100 | The compressed block consists of one or more **instructions**. 101 | Each instruction starts with a 1-byte opcode, 2-byte opcode, or 3-byte opcode. 102 | 103 | | Instruction type | Opcode[0] | Opcode[1] | Opcode[2] 104 | |-----------|------------------|--------------------|--| 105 | | Literal run | `000`, L₄-L₀ | -|- | 106 | | Short match | M₂-M₀, R₁₂-R₈ | R₇-R₀ | - | 107 | | Long match | `111`, R₁₂-R₈ | M₇-M₀ | R₇-R₀ | 108 | 109 | Note that the _very first_ instruction in a compressed block is always a literal run. 110 | 111 | #### Literal run instruction 112 | 113 | For the literal run instruction, there is one or more bytes following the code. This is called the literal run. 114 | 115 | The 5 least-significant bits of `opcode[0]`, _L_, determines the **number of literals** following the opcode. The value of 0 indicates a 1-byte literal run, 1 indicates a 2-byte literal run, and so on. The minimum literal run is 1 and the maximum literal run is 32. 116 | 117 | The decompressor copies (_L + 1_) bytes of literal run, starting from the first one right after opcode. 118 | 119 | _Example_: If the compressed block is a 4-byte array of `[0x02, 0x41, 0x42, 0x43]`, then the opcode is `0x02` and that means a literal run of 3 bytes. The decompressor will then copy the subsequent 3 bytes, `[0x41, 0x42, 0x43]`, to the output buffer. The output buffer now represents the (original) uncompressed block, `[0x41, 0x42, 0x43]`. 120 | 121 | #### Short match instruction 122 | 123 | The 3 most-significant bits of `opcode[0]`, _M_, determines the **match length**. The value of 1 indicates a 3-byte match, 2 indicates a 4-byte match and so on. The minimum match length is 3 and the maximum match length is 8. 124 | 125 | The 5 least-significant bits of `opcode[0]` combined with the 8 bits of the `opcode[1]`, _R_, determines the **reference offset**. Since the offset is encoded in 13 bits, the minimum is 0 and the maximum is 8191. 126 | 127 | The following C code retrieves the match length and reference offset: 128 | 129 | ```c 130 | M = opcode[0] >> 5; 131 | R = 256 * (opcode[0] << 5) + opcode[1]; 132 | ``` 133 | 134 | The decompressor copies _(M+2)_ bytes, starting from the location offsetted by _R_ in the output buffer. Note that _R_ is a *back reference*, i.e. the value of 0 corresponds the last byte in the output buffer, 1 is the second to last byte, and so forth. 135 | 136 | _Example 1_: If the compressed block is a 7-byte array of `[0x03, 0x41, 0x42, 0x43, 0x44, 0x20, 0x02]`, then there are two instructions in the there. The first instruction is the literal run of 4 bytes (due to _L = 3_). Thus, the decompressor copies 4 bytes to the output buffer, resulting in `[0x41, 0x42, 0x43, 0x44]`. The second instruction is the short match of 3 bytes (from _M = 1_, i.e `0x20 >> 5`) and the offset of 2. Therefore, the compressor goes back 2 bytes from the last position, copies 3 bytes (`[0x42, 0x43, 0x44]`), and appends them to the output buffer. The output buffer now represents the complete uncompressed data, `[0x41, 0x42, 0x43, 0x44, 0x42, 0x43, 0x44]`. 137 | 138 | _Example 2_: If the compressed block is a 4-byte array of `[0x00, 0x61, 0x40, 0x00]`, then there are two instructions in there. The first instruction is the literal run of just 1 byte (_L = 0_). Thus, the decompressor copies the byte (`0x61`) to the output buffer. The output buffer now becomes `[0x61]`. The second instruction is the short match of 4 bytes (from _M = 2_, i.e. `0x40 >> 5`) and the offset of 0. Therefore, the decompressor copies 4 bytes starting using the back reference of 0 (i.e. the position of `0x61`). The output buffer now represents the complete uncompressed data, `[0x61, 0x61, 0x61, 0x61, 0x61]`. 139 | 140 | #### Long match instruction 141 | 142 | The value of `opcode[1]`, _M_, determines the **match length**. The value of 0 indicates a 9-byte match, 1 indicates a 10-byte match and so on. The minimum match length is 9 and the maximum match length is 264. 143 | 144 | The 5 least-significant bits of `opcode[0]` combined with the 8 bits of `opcode[2]`, _R_, determines the **reference offset**. Since the offset is encoded in 13 bits, the minimum is 0 and the maximum is 8191. 145 | 146 | The following C code retrieves the match length and reference offset: 147 | 148 | ```c 149 | M = opcode[1]; 150 | R = 256 * (opcode[0] << 5) + opcode[2]; 151 | ``` 152 | The decompressor copies _(M+9)_ bytes, starting from the location offsetted by _R_ in the output buffer. Note that _R_ is a *back reference*, i.e. the value of 0 corresponds to the last byte in the output buffer, 1 is for the second to last byte, and so forth. 153 | 154 | _Example_: If the compressed block is a 4-byte array of `[0x01, 0x44, 0x45, 0xE0, 0x01, 0x01]`, then there are two instructions in there. The first instruction is the literal run with the length of 2 (due to _L = 1_). Thus, the decompressor copies the 2-byte literal run (`[0x44, 0x45]`) to the output buffer. The second instruction is the long match with the match length of 10 (from _M = 1_) and the offset of 1. Therefore, the decompressor copies 10 bytes starting using the back reference of 1 (i.e. the position of `0x44`). The output buffer now represents the complete uncompressed data, `[0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45, 0x44, 0x45]`. 155 | 156 | #### Decompressor Reference Implementation 157 | 158 | The following 40-line C function implements a fully-functional decompressor for the above block format. Note that it is intended to be educational, e.g. no bound check is implemented, and therefore it is absolutely **unsafe** for production. 159 | 160 | ```c 161 | void fastlz_level1_decompress(const uint8_t* input, int length, uint8_t* output) { 162 | int src = 0; 163 | int dest = 0; 164 | while (src < length) { 165 | int type = input[src] >> 5; 166 | if (type == 0) { 167 | /* literal run */ 168 | int run = 1 + input[src]; 169 | src = src + 1; 170 | while (run > 0) { 171 | output[dest] = input[src]; 172 | src = src + 1; 173 | dest = dest + 1; 174 | run = run - 1; 175 | } 176 | } else if (type < 7) { 177 | /* short match */ 178 | int ofs = 256 * (input[src] & 31) + input[src + 1]; 179 | int len = 2 + (input[src] >> 5); 180 | src = src + 2; 181 | int ref = dest - ofs - 1; 182 | while (len > 0) { 183 | output[dest] = output[ref]; 184 | ref = ref + 1; 185 | dest = dest + 1; 186 | len = len - 1; 187 | } 188 | } else { 189 | /* long match */ 190 | int ofs = 256 * (input[src] & 31) + input[src + 2]; 191 | int len = 9 + input[src + 1]; 192 | src = src + 3; 193 | int ref = dest - ofs - 1; 194 | while (len > 0) { 195 | output[dest] = output[ref]; 196 | ref = ref + 1; 197 | dest = dest + 1; 198 | len = len - 1; 199 | } 200 | } 201 | } 202 | } 203 | ``` 204 | 205 | ### Block Format for Level 2 206 | 207 | (To be written) 208 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-tactile -------------------------------------------------------------------------------- /examples/6pack.c: -------------------------------------------------------------------------------- 1 | /* 2 | 6PACK - file compressor using FastLZ (lightning-fast compression library) 3 | Copyright (C) 2007-2020 Ariya Hidayat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #define SIXPACK_VERSION_MAJOR 0 29 | #define SIXPACK_VERSION_MINOR 1 30 | #define SIXPACK_VERSION_REVISION 0 31 | #define SIXPACK_VERSION_STRING "snapshot 20070615" 32 | 33 | #include "fastlz.h" 34 | 35 | #undef PATH_SEPARATOR 36 | 37 | #if defined(MSDOS) || defined(__MSDOS__) || defined(MSDOS) 38 | #define PATH_SEPARATOR '\\' 39 | #endif 40 | 41 | #if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__) 42 | #define PATH_SEPARATOR '\\' 43 | #endif 44 | 45 | #ifndef PATH_SEPARATOR 46 | #define PATH_SEPARATOR '/' 47 | #endif 48 | 49 | #undef SIXPACK_BENCHMARK_WIN32 50 | #if defined(WIN32) || defined(__NT__) || defined(_WIN32) || defined(__WIN32__) 51 | #if defined(_MSC_VER) || defined(__GNUC__) 52 | #define SIXPACK_BENCHMARK_WIN32 53 | #include 54 | #endif 55 | #endif 56 | 57 | /* magic identifier for 6pack file */ 58 | static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10}; 59 | 60 | #define BLOCK_SIZE (2 * 64 * 1024) 61 | 62 | /* prototypes */ 63 | static unsigned long update_adler32(unsigned long checksum, const void* buf, int len); 64 | void usage(void); 65 | int detect_magic(FILE* f); 66 | void write_magic(FILE* f); 67 | void write_chunk_header(FILE* f, int id, int options, unsigned long size, unsigned long checksum, unsigned long extra); 68 | unsigned long block_compress(const unsigned char* input, unsigned long length, unsigned char* output); 69 | int pack_file_compressed(const char* input_file, int method, int level, FILE* f); 70 | int pack_file(int compress_level, const char* input_file, const char* output_file); 71 | 72 | /* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */ 73 | #define ADLER32_BASE 65521 74 | static unsigned long update_adler32(unsigned long checksum, const void* buf, int len) { 75 | const unsigned char* ptr = (const unsigned char*)buf; 76 | unsigned long s1 = checksum & 0xffff; 77 | unsigned long s2 = (checksum >> 16) & 0xffff; 78 | 79 | while (len > 0) { 80 | unsigned k = len < 5552 ? len : 5552; 81 | len -= k; 82 | 83 | while (k >= 8) { 84 | s1 += *ptr++; 85 | s2 += s1; 86 | s1 += *ptr++; 87 | s2 += s1; 88 | s1 += *ptr++; 89 | s2 += s1; 90 | s1 += *ptr++; 91 | s2 += s1; 92 | s1 += *ptr++; 93 | s2 += s1; 94 | s1 += *ptr++; 95 | s2 += s1; 96 | s1 += *ptr++; 97 | s2 += s1; 98 | s1 += *ptr++; 99 | s2 += s1; 100 | k -= 8; 101 | } 102 | 103 | while (k-- > 0) { 104 | s1 += *ptr++; 105 | s2 += s1; 106 | } 107 | s1 = s1 % ADLER32_BASE; 108 | s2 = s2 % ADLER32_BASE; 109 | } 110 | return (s2 << 16) + s1; 111 | } 112 | 113 | void usage(void) { 114 | printf("6pack: high-speed file compression tool\n"); 115 | printf("Copyright (C) Ariya Hidayat\n"); 116 | printf("\n"); 117 | printf("Usage: 6pack [options] input-file output-file\n"); 118 | printf("\n"); 119 | printf("Options:\n"); 120 | printf(" -1 compress faster\n"); 121 | printf(" -2 compress better\n"); 122 | printf(" -v show program version\n"); 123 | #ifdef SIXPACK_BENCHMARK_WIN32 124 | printf(" -mem check in-memory compression speed\n"); 125 | #endif 126 | printf("\n"); 127 | } 128 | 129 | /* return non-zero if magic sequence is detected */ 130 | /* warning: reset the read pointer to the beginning of the file */ 131 | int detect_magic(FILE* f) { 132 | unsigned char buffer[8]; 133 | size_t bytes_read; 134 | int c; 135 | 136 | fseek(f, SEEK_SET, 0); 137 | bytes_read = fread(buffer, 1, 8, f); 138 | fseek(f, SEEK_SET, 0); 139 | if (bytes_read < 8) return 0; 140 | 141 | for (c = 0; c < 8; c++) 142 | if (buffer[c] != sixpack_magic[c]) return 0; 143 | 144 | return -1; 145 | } 146 | 147 | void write_magic(FILE* f) { fwrite(sixpack_magic, 8, 1, f); } 148 | 149 | void write_chunk_header(FILE* f, int id, int options, unsigned long size, unsigned long checksum, unsigned long extra) { 150 | unsigned char buffer[16]; 151 | 152 | buffer[0] = id & 255; 153 | buffer[1] = id >> 8; 154 | buffer[2] = options & 255; 155 | buffer[3] = options >> 8; 156 | buffer[4] = size & 255; 157 | buffer[5] = (size >> 8) & 255; 158 | buffer[6] = (size >> 16) & 255; 159 | buffer[7] = (size >> 24) & 255; 160 | buffer[8] = checksum & 255; 161 | buffer[9] = (checksum >> 8) & 255; 162 | buffer[10] = (checksum >> 16) & 255; 163 | buffer[11] = (checksum >> 24) & 255; 164 | buffer[12] = extra & 255; 165 | buffer[13] = (extra >> 8) & 255; 166 | buffer[14] = (extra >> 16) & 255; 167 | buffer[15] = (extra >> 24) & 255; 168 | 169 | fwrite(buffer, 16, 1, f); 170 | } 171 | 172 | int pack_file_compressed(const char* input_file, int method, int level, FILE* f) { 173 | FILE* in; 174 | unsigned long fsize; 175 | unsigned long checksum; 176 | const char* shown_name; 177 | unsigned char buffer[BLOCK_SIZE]; 178 | unsigned char result[BLOCK_SIZE * 2]; /* FIXME twice is too large */ 179 | unsigned char progress[20]; 180 | int c; 181 | unsigned long percent; 182 | unsigned long total_read; 183 | unsigned long total_compressed; 184 | int chunk_size; 185 | 186 | /* sanity check */ 187 | in = fopen(input_file, "rb"); 188 | if (!in) { 189 | printf("Error: could not open %s\n", input_file); 190 | return -1; 191 | } 192 | 193 | /* find size of the file */ 194 | fseek(in, 0, SEEK_END); 195 | fsize = ftell(in); 196 | fseek(in, 0, SEEK_SET); 197 | 198 | /* already a 6pack archive? */ 199 | if (detect_magic(in)) { 200 | printf("Error: file %s is already a 6pack archive!\n", input_file); 201 | fclose(in); 202 | return -1; 203 | } 204 | 205 | /* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */ 206 | shown_name = input_file + strlen(input_file) - 1; 207 | while (shown_name > input_file) 208 | if (*(shown_name - 1) == PATH_SEPARATOR) 209 | break; 210 | else 211 | shown_name--; 212 | 213 | /* chunk for File Entry */ 214 | buffer[0] = fsize & 255; 215 | buffer[1] = (fsize >> 8) & 255; 216 | buffer[2] = (fsize >> 16) & 255; 217 | buffer[3] = (fsize >> 24) & 255; 218 | #if 0 219 | buffer[4] = (fsize >> 32) & 255; 220 | buffer[5] = (fsize >> 40) & 255; 221 | buffer[6] = (fsize >> 48) & 255; 222 | buffer[7] = (fsize >> 56) & 255; 223 | #else 224 | /* because fsize is only 32-bit */ 225 | buffer[4] = 0; 226 | buffer[5] = 0; 227 | buffer[6] = 0; 228 | buffer[7] = 0; 229 | #endif 230 | buffer[8] = (strlen(shown_name) + 1) & 255; 231 | buffer[9] = (strlen(shown_name) + 1) >> 8; 232 | checksum = 1L; 233 | checksum = update_adler32(checksum, buffer, 10); 234 | checksum = update_adler32(checksum, shown_name, strlen(shown_name) + 1); 235 | write_chunk_header(f, 1, 0, 10 + strlen(shown_name) + 1, checksum, 0); 236 | fwrite(buffer, 10, 1, f); 237 | fwrite(shown_name, strlen(shown_name) + 1, 1, f); 238 | total_compressed = 16 + 10 + strlen(shown_name) + 1; 239 | 240 | /* for progress status */ 241 | memset(progress, ' ', 20); 242 | if (strlen(shown_name) < 16) 243 | for (c = 0; c < (int)strlen(shown_name); c++) progress[c] = shown_name[c]; 244 | else { 245 | for (c = 0; c < 13; c++) progress[c] = shown_name[c]; 246 | progress[13] = '.'; 247 | progress[14] = '.'; 248 | progress[15] = ' '; 249 | } 250 | progress[16] = '['; 251 | progress[17] = 0; 252 | printf("%s", progress); 253 | for (c = 0; c < 50; c++) printf("."); 254 | printf("]\r"); 255 | printf("%s", progress); 256 | 257 | /* read file and place in archive */ 258 | total_read = 0; 259 | percent = 0; 260 | for (;;) { 261 | int compress_method = method; 262 | int last_percent = (int)percent; 263 | size_t bytes_read = fread(buffer, 1, BLOCK_SIZE, in); 264 | if (bytes_read == 0) break; 265 | total_read += bytes_read; 266 | 267 | /* for progress */ 268 | if (fsize < (1 << 24)) 269 | percent = total_read * 100 / fsize; 270 | else 271 | percent = total_read / 256 * 100 / (fsize >> 8); 272 | percent >>= 1; 273 | while (last_percent < (int)percent) { 274 | printf("#"); 275 | last_percent++; 276 | } 277 | 278 | /* too small, don't bother to compress */ 279 | if (bytes_read < 32) compress_method = 0; 280 | 281 | /* write to output */ 282 | switch (compress_method) { 283 | /* FastLZ */ 284 | case 1: 285 | chunk_size = fastlz_compress_level(level, buffer, bytes_read, result); 286 | checksum = update_adler32(1L, result, chunk_size); 287 | write_chunk_header(f, 17, 1, chunk_size, checksum, bytes_read); 288 | fwrite(result, 1, chunk_size, f); 289 | total_compressed += 16; 290 | total_compressed += chunk_size; 291 | break; 292 | 293 | /* uncompressed, also fallback method */ 294 | case 0: 295 | default: 296 | checksum = 1L; 297 | checksum = update_adler32(checksum, buffer, bytes_read); 298 | write_chunk_header(f, 17, 0, bytes_read, checksum, bytes_read); 299 | fwrite(buffer, 1, bytes_read, f); 300 | total_compressed += 16; 301 | total_compressed += bytes_read; 302 | break; 303 | } 304 | } 305 | 306 | fclose(in); 307 | if (total_read != fsize) { 308 | printf("\n"); 309 | printf("Error: reading %s failed!\n", input_file); 310 | return -1; 311 | } else { 312 | printf("] "); 313 | if (total_compressed < fsize) { 314 | if (fsize < (1 << 20)) 315 | percent = total_compressed * 1000 / fsize; 316 | else 317 | percent = total_compressed / 256 * 1000 / (fsize >> 8); 318 | percent = 1000 - percent; 319 | printf("%2d.%d%% saved", (int)percent / 10, (int)percent % 10); 320 | } 321 | printf("\n"); 322 | } 323 | 324 | return 0; 325 | } 326 | 327 | int pack_file(int compress_level, const char* input_file, const char* output_file) { 328 | FILE* f; 329 | int result; 330 | 331 | f = fopen(output_file, "rb"); 332 | if (f) { 333 | fclose(f); 334 | printf("Error: file %s already exists. Aborted.\n\n", output_file); 335 | return -1; 336 | } 337 | 338 | f = fopen(output_file, "wb"); 339 | if (!f) { 340 | printf("Error: could not create %s. Aborted.\n\n", output_file); 341 | return -1; 342 | } 343 | 344 | write_magic(f); 345 | 346 | result = pack_file_compressed(input_file, 1, compress_level, f); 347 | fclose(f); 348 | 349 | return result; 350 | } 351 | 352 | #ifdef SIXPACK_BENCHMARK_WIN32 353 | int benchmark_speed(int compress_level, const char* input_file); 354 | 355 | int benchmark_speed(int compress_level, const char* input_file) { 356 | FILE* in; 357 | unsigned long fsize; 358 | unsigned long maxout; 359 | const char* shown_name; 360 | unsigned char* buffer; 361 | unsigned char* result; 362 | size_t bytes_read; 363 | 364 | /* sanity check */ 365 | in = fopen(input_file, "rb"); 366 | if (!in) { 367 | printf("Error: could not open %s\n", input_file); 368 | return -1; 369 | } 370 | 371 | /* find size of the file */ 372 | fseek(in, 0, SEEK_END); 373 | fsize = ftell(in); 374 | fseek(in, 0, SEEK_SET); 375 | 376 | /* already a 6pack archive? */ 377 | if (detect_magic(in)) { 378 | printf("Error: no benchmark for 6pack archive!\n"); 379 | fclose(in); 380 | return -1; 381 | } 382 | 383 | /* truncate directory prefix, e.g. "foo/bar/FILE.txt" becomes "FILE.txt" */ 384 | shown_name = input_file + strlen(input_file) - 1; 385 | while (shown_name > input_file) 386 | if (*(shown_name - 1) == PATH_SEPARATOR) 387 | break; 388 | else 389 | shown_name--; 390 | 391 | maxout = 1.05 * fsize; 392 | maxout = (maxout < 66) ? 66 : maxout; 393 | buffer = (unsigned char*)malloc(fsize); 394 | result = (unsigned char*)malloc(maxout); 395 | if (!buffer || !result) { 396 | printf("Error: not enough memory!\n"); 397 | free(buffer); 398 | free(result); 399 | fclose(in); 400 | return -1; 401 | } 402 | 403 | printf("Reading source file....\n"); 404 | bytes_read = fread(buffer, 1, fsize, in); 405 | if (bytes_read != fsize) { 406 | printf("Error reading file %s!\n", shown_name); 407 | printf("Read %d bytes, expecting %d bytes\n", bytes_read, fsize); 408 | free(buffer); 409 | free(result); 410 | fclose(in); 411 | return -1; 412 | } 413 | 414 | /* shamelessly copied from QuickLZ 1.20 test program */ 415 | { 416 | unsigned int j, y; 417 | size_t i, u = 0; 418 | double mbs, fastest; 419 | unsigned long compressed_size; 420 | 421 | printf("Setting HIGH_PRIORITY_CLASS...\n"); 422 | SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS); 423 | 424 | printf("Benchmarking FastLZ Level %d, please wait...\n", compress_level); 425 | 426 | i = bytes_read; 427 | fastest = 0.0; 428 | for (j = 0; j < 3; j++) { 429 | y = 0; 430 | mbs = GetTickCount(); 431 | while (GetTickCount() == mbs) 432 | ; 433 | mbs = GetTickCount(); 434 | while (GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */ 435 | { 436 | u = fastlz_compress_level(compress_level, buffer, bytes_read, result); 437 | y++; 438 | } 439 | 440 | mbs = ((double)i * (double)y) / ((double)(GetTickCount() - mbs) / 1000.) / 1000000.; 441 | /*printf(" %.1f Mbyte/s ", mbs);*/ 442 | if (fastest < mbs) fastest = mbs; 443 | } 444 | 445 | printf("\nCompressed %d bytes into %d bytes (%.1f%%) at %.1f Mbyte/s.\n", (unsigned int)i, (unsigned int)u, 446 | (double)u / (double)i * 100., fastest); 447 | 448 | #if 1 449 | fastest = 0.0; 450 | compressed_size = u; 451 | for (j = 0; j < 3; j++) { 452 | y = 0; 453 | mbs = GetTickCount(); 454 | while (GetTickCount() == mbs) 455 | ; 456 | mbs = GetTickCount(); 457 | while (GetTickCount() - mbs < 3000) /* 1% accuracy with 18.2 timer */ 458 | { 459 | u = fastlz_decompress(result, compressed_size, buffer, bytes_read); 460 | y++; 461 | } 462 | 463 | mbs = ((double)i * (double)y) / ((double)(GetTickCount() - mbs) / 1000.) / 1000000.; 464 | /*printf(" %.1f Mbyte/s ", mbs);*/ 465 | if (fastest < mbs) fastest = mbs; 466 | } 467 | 468 | printf("\nDecompressed at %.1f Mbyte/s.\n\n(1 MB = 1000000 byte)\n", fastest); 469 | #endif 470 | } 471 | 472 | fclose(in); 473 | return 0; 474 | } 475 | #endif /* SIXPACK_BENCHMARK_WIN32 */ 476 | 477 | int main(int argc, char** argv) { 478 | int i; 479 | int compress_level; 480 | int benchmark; 481 | char* input_file; 482 | char* output_file; 483 | 484 | /* show help with no argument at all*/ 485 | if (argc == 1) { 486 | usage(); 487 | return 0; 488 | } 489 | 490 | /* default compression level, not the fastest */ 491 | compress_level = 2; 492 | 493 | /* do benchmark only when explicitly specified */ 494 | benchmark = 0; 495 | 496 | /* no file is specified */ 497 | input_file = 0; 498 | output_file = 0; 499 | 500 | for (i = 1; i <= argc; i++) { 501 | char* argument = argv[i]; 502 | 503 | if (!argument) continue; 504 | 505 | /* display help on usage */ 506 | if (!strcmp(argument, "-h") || !strcmp(argument, "--help")) { 507 | usage(); 508 | return 0; 509 | } 510 | 511 | /* check for version information */ 512 | if (!strcmp(argument, "-v") || !strcmp(argument, "--version")) { 513 | printf("6pack: high-speed file compression tool\n"); 514 | printf("Version %s (using FastLZ %s)\n", SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING); 515 | printf("Copyright (C) Ariya Hidayat\n"); 516 | printf("\n"); 517 | return 0; 518 | } 519 | 520 | /* test compression speed? */ 521 | if (!strcmp(argument, "-mem")) { 522 | benchmark = 1; 523 | continue; 524 | } 525 | 526 | /* compression level */ 527 | if (!strcmp(argument, "-1") || !strcmp(argument, "--fastest")) { 528 | compress_level = 1; 529 | continue; 530 | } 531 | if (!strcmp(argument, "-2")) { 532 | compress_level = 2; 533 | continue; 534 | } 535 | 536 | /* unknown option */ 537 | if (argument[0] == '-') { 538 | printf("Error: unknown option %s\n\n", argument); 539 | printf("To get help on usage:\n"); 540 | printf(" 6pack --help\n\n"); 541 | return -1; 542 | } 543 | 544 | /* first specified file is input */ 545 | if (!input_file) { 546 | input_file = argument; 547 | continue; 548 | } 549 | 550 | /* next specified file is output */ 551 | if (!output_file) { 552 | output_file = argument; 553 | continue; 554 | } 555 | 556 | /* files are already specified */ 557 | printf("Error: unknown option %s\n\n", argument); 558 | printf("To get help on usage:\n"); 559 | printf(" 6pack --help\n\n"); 560 | return -1; 561 | } 562 | 563 | if (!input_file) { 564 | printf("Error: input file is not specified.\n\n"); 565 | printf("To get help on usage:\n"); 566 | printf(" 6pack --help\n\n"); 567 | return -1; 568 | } 569 | 570 | if (!output_file && !benchmark) { 571 | printf("Error: output file is not specified.\n\n"); 572 | printf("To get help on usage:\n"); 573 | printf(" 6pack --help\n\n"); 574 | return -1; 575 | } 576 | 577 | #ifdef SIXPACK_BENCHMARK_WIN32 578 | if (benchmark) 579 | return benchmark_speed(compress_level, input_file); 580 | else 581 | #endif 582 | return pack_file(compress_level, input_file, output_file); 583 | 584 | /* unreachable */ 585 | return 0; 586 | } 587 | -------------------------------------------------------------------------------- /examples/6unpack.c: -------------------------------------------------------------------------------- 1 | /* 2 | 6PACK - file compressor using FastLZ (lightning-fast compression library) 3 | Copyright (C) 2007-2020 Ariya Hidayat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #define SIXPACK_VERSION_MAJOR 0 29 | #define SIXPACK_VERSION_MINOR 1 30 | #define SIXPACK_VERSION_REVISION 0 31 | #define SIXPACK_VERSION_STRING "0.1.0" 32 | 33 | #include "fastlz.h" 34 | 35 | /* magic identifier for 6pack file */ 36 | static unsigned char sixpack_magic[8] = {137, '6', 'P', 'K', 13, 10, 26, 10}; 37 | 38 | #define BLOCK_SIZE 65536 39 | 40 | /* prototypes */ 41 | static unsigned long update_adler32(unsigned long checksum, const void* buf, int len); 42 | void usage(void); 43 | int detect_magic(FILE* f); 44 | static unsigned long readU16(const unsigned char* ptr); 45 | static unsigned long readU32(const unsigned char* ptr); 46 | void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size, unsigned long* checksum, 47 | unsigned long* extra); 48 | int unpack_file(const char* archive_file); 49 | 50 | /* for Adler-32 checksum algorithm, see RFC 1950 Section 8.2 */ 51 | #define ADLER32_BASE 65521 52 | static unsigned long update_adler32(unsigned long checksum, const void* buf, int len) { 53 | const unsigned char* ptr = (const unsigned char*)buf; 54 | unsigned long s1 = checksum & 0xffff; 55 | unsigned long s2 = (checksum >> 16) & 0xffff; 56 | 57 | while (len > 0) { 58 | unsigned k = len < 5552 ? len : 5552; 59 | len -= k; 60 | 61 | while (k >= 8) { 62 | s1 += *ptr++; 63 | s2 += s1; 64 | s1 += *ptr++; 65 | s2 += s1; 66 | s1 += *ptr++; 67 | s2 += s1; 68 | s1 += *ptr++; 69 | s2 += s1; 70 | s1 += *ptr++; 71 | s2 += s1; 72 | s1 += *ptr++; 73 | s2 += s1; 74 | s1 += *ptr++; 75 | s2 += s1; 76 | s1 += *ptr++; 77 | s2 += s1; 78 | k -= 8; 79 | } 80 | 81 | while (k-- > 0) { 82 | s1 += *ptr++; 83 | s2 += s1; 84 | } 85 | s1 = s1 % ADLER32_BASE; 86 | s2 = s2 % ADLER32_BASE; 87 | } 88 | return (s2 << 16) + s1; 89 | } 90 | 91 | void usage(void) { 92 | printf("6unpack: uncompress 6pack archive\n"); 93 | printf("Copyright (C) Ariya Hidayat\n"); 94 | printf("\n"); 95 | printf("Usage: 6unpack archive-file\n"); 96 | printf("\n"); 97 | } 98 | 99 | /* return non-zero if magic sequence is detected */ 100 | /* warning: reset the read pointer to the beginning of the file */ 101 | int detect_magic(FILE* f) { 102 | unsigned char buffer[8]; 103 | size_t bytes_read; 104 | int c; 105 | 106 | fseek(f, SEEK_SET, 0); 107 | bytes_read = fread(buffer, 1, 8, f); 108 | fseek(f, SEEK_SET, 0); 109 | if (bytes_read < 8) return 0; 110 | 111 | for (c = 0; c < 8; c++) 112 | if (buffer[c] != sixpack_magic[c]) return 0; 113 | 114 | return -1; 115 | } 116 | 117 | static unsigned long readU16(const unsigned char* ptr) { return ptr[0] + (ptr[1] << 8); } 118 | 119 | static unsigned long readU32(const unsigned char* ptr) { 120 | return ptr[0] + (ptr[1] << 8) + (ptr[2] << 16) + (ptr[3] << 24); 121 | } 122 | 123 | void read_chunk_header(FILE* f, int* id, int* options, unsigned long* size, unsigned long* checksum, 124 | unsigned long* extra) { 125 | unsigned char buffer[16]; 126 | fread(buffer, 1, 16, f); 127 | 128 | *id = readU16(buffer) & 0xffff; 129 | *options = readU16(buffer + 2) & 0xffff; 130 | *size = readU32(buffer + 4) & 0xffffffff; 131 | *checksum = readU32(buffer + 8) & 0xffffffff; 132 | *extra = readU32(buffer + 12) & 0xffffffff; 133 | } 134 | 135 | int unpack_file(const char* input_file) { 136 | FILE* in; 137 | unsigned long fsize; 138 | int c; 139 | unsigned long percent; 140 | unsigned char progress[20]; 141 | int chunk_id; 142 | int chunk_options; 143 | unsigned long chunk_size; 144 | unsigned long chunk_checksum; 145 | unsigned long chunk_extra; 146 | unsigned char buffer[BLOCK_SIZE]; 147 | unsigned long checksum; 148 | 149 | unsigned long decompressed_size; 150 | unsigned long total_extracted; 151 | int name_length; 152 | char* output_file; 153 | FILE* f; 154 | 155 | unsigned char* compressed_buffer; 156 | unsigned char* decompressed_buffer; 157 | unsigned long compressed_bufsize; 158 | unsigned long decompressed_bufsize; 159 | 160 | /* sanity check */ 161 | in = fopen(input_file, "rb"); 162 | if (!in) { 163 | printf("Error: could not open %s\n", input_file); 164 | return -1; 165 | } 166 | 167 | /* find size of the file */ 168 | fseek(in, 0, SEEK_END); 169 | fsize = ftell(in); 170 | fseek(in, 0, SEEK_SET); 171 | 172 | /* not a 6pack archive? */ 173 | if (!detect_magic(in)) { 174 | fclose(in); 175 | printf("Error: file %s is not a 6pack archive!\n", input_file); 176 | return -1; 177 | } 178 | 179 | printf("Archive: %s", input_file); 180 | 181 | /* position of first chunk */ 182 | fseek(in, 8, SEEK_SET); 183 | 184 | /* initialize */ 185 | output_file = 0; 186 | f = 0; 187 | total_extracted = 0; 188 | decompressed_size = 0; 189 | percent = 0; 190 | compressed_buffer = 0; 191 | decompressed_buffer = 0; 192 | compressed_bufsize = 0; 193 | decompressed_bufsize = 0; 194 | 195 | /* main loop */ 196 | for (;;) { 197 | /* end of file? */ 198 | size_t pos = ftell(in); 199 | if (pos >= fsize) break; 200 | 201 | read_chunk_header(in, &chunk_id, &chunk_options, &chunk_size, &chunk_checksum, &chunk_extra); 202 | 203 | if ((chunk_id == 1) && (chunk_size > 10) && (chunk_size < BLOCK_SIZE)) { 204 | /* close current file, if any */ 205 | printf("\n"); 206 | free(output_file); 207 | output_file = 0; 208 | if (f) fclose(f); 209 | 210 | /* file entry */ 211 | fread(buffer, 1, chunk_size, in); 212 | checksum = update_adler32(1L, buffer, chunk_size); 213 | if (checksum != chunk_checksum) { 214 | free(output_file); 215 | output_file = 0; 216 | fclose(in); 217 | printf("\nError: checksum mismatch!\n"); 218 | printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum); 219 | return -1; 220 | } 221 | 222 | decompressed_size = readU32(buffer); 223 | total_extracted = 0; 224 | percent = 0; 225 | 226 | /* get file to extract */ 227 | name_length = (int)readU16(buffer + 8); 228 | if (name_length > (int)chunk_size - 10) name_length = chunk_size - 10; 229 | output_file = (char*)malloc(name_length + 1); 230 | memset(output_file, 0, name_length + 1); 231 | for (c = 0; c < name_length; c++) output_file[c] = buffer[10 + c]; 232 | 233 | /* check if already exists */ 234 | f = fopen(output_file, "rb"); 235 | if (f) { 236 | fclose(f); 237 | printf("File %s already exists. Skipped.\n", output_file); 238 | free(output_file); 239 | output_file = 0; 240 | f = 0; 241 | } else { 242 | /* create the file */ 243 | f = fopen(output_file, "wb"); 244 | if (!f) { 245 | printf("Can't create file %s. Skipped.\n", output_file); 246 | free(output_file); 247 | output_file = 0; 248 | f = 0; 249 | } else { 250 | /* for progress status */ 251 | printf("\n"); 252 | memset(progress, ' ', 20); 253 | if (strlen(output_file) < 16) 254 | for (c = 0; c < (int)strlen(output_file); c++) progress[c] = output_file[c]; 255 | else { 256 | for (c = 0; c < 13; c++) progress[c] = output_file[c]; 257 | progress[13] = '.'; 258 | progress[14] = '.'; 259 | progress[15] = ' '; 260 | } 261 | progress[16] = '['; 262 | progress[17] = 0; 263 | printf("%s", progress); 264 | for (c = 0; c < 50; c++) printf("."); 265 | printf("]\r"); 266 | printf("%s", progress); 267 | } 268 | } 269 | } 270 | 271 | if ((chunk_id == 17) && f && output_file && decompressed_size) { 272 | unsigned long remaining; 273 | 274 | /* uncompressed */ 275 | switch (chunk_options) { 276 | /* stored, simply copy to output */ 277 | case 0: 278 | /* read one block at at time, write and update checksum */ 279 | total_extracted += chunk_size; 280 | remaining = chunk_size; 281 | checksum = 1L; 282 | for (;;) { 283 | unsigned long r = (BLOCK_SIZE < remaining) ? BLOCK_SIZE : remaining; 284 | size_t bytes_read = fread(buffer, 1, r, in); 285 | if (bytes_read == 0) break; 286 | fwrite(buffer, 1, bytes_read, f); 287 | checksum = update_adler32(checksum, buffer, bytes_read); 288 | remaining -= bytes_read; 289 | } 290 | 291 | /* verify everything is written correctly */ 292 | if (checksum != chunk_checksum) { 293 | fclose(f); 294 | f = 0; 295 | free(output_file); 296 | output_file = 0; 297 | printf("\nError: checksum mismatch. Aborted.\n"); 298 | printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum); 299 | } 300 | break; 301 | 302 | /* compressed using FastLZ */ 303 | case 1: 304 | /* enlarge input buffer if necessary */ 305 | if (chunk_size > compressed_bufsize) { 306 | compressed_bufsize = chunk_size; 307 | free(compressed_buffer); 308 | compressed_buffer = (unsigned char*)malloc(compressed_bufsize); 309 | } 310 | 311 | /* enlarge output buffer if necessary */ 312 | if (chunk_extra > decompressed_bufsize) { 313 | decompressed_bufsize = chunk_extra; 314 | free(decompressed_buffer); 315 | decompressed_buffer = (unsigned char*)malloc(decompressed_bufsize); 316 | } 317 | 318 | /* read and check checksum */ 319 | fread(compressed_buffer, 1, chunk_size, in); 320 | checksum = update_adler32(1L, compressed_buffer, chunk_size); 321 | total_extracted += chunk_extra; 322 | 323 | /* verify that the chunk data is correct */ 324 | if (checksum != chunk_checksum) { 325 | fclose(f); 326 | f = 0; 327 | free(output_file); 328 | output_file = 0; 329 | printf("\nError: checksum mismatch. Skipped.\n"); 330 | printf("Got %08lX Expecting %08lX\n", checksum, chunk_checksum); 331 | } else { 332 | /* decompress and verify */ 333 | remaining = fastlz_decompress(compressed_buffer, chunk_size, decompressed_buffer, chunk_extra); 334 | if (remaining != chunk_extra) { 335 | fclose(f); 336 | f = 0; 337 | free(output_file); 338 | output_file = 0; 339 | printf("\nError: decompression failed. Skipped.\n"); 340 | } else 341 | fwrite(decompressed_buffer, 1, chunk_extra, f); 342 | } 343 | break; 344 | 345 | default: 346 | printf("\nError: unknown compression method (%d)\n", chunk_options); 347 | fclose(f); 348 | f = 0; 349 | free(output_file); 350 | output_file = 0; 351 | break; 352 | } 353 | 354 | /* for progress, if everything is fine */ 355 | if (f) { 356 | int last_percent = (int)percent; 357 | if (decompressed_size < (1 << 24)) 358 | percent = total_extracted * 100 / decompressed_size; 359 | else 360 | percent = total_extracted / 256 * 100 / (decompressed_size >> 8); 361 | percent >>= 1; 362 | while (last_percent < (int)percent) { 363 | printf("#"); 364 | last_percent++; 365 | } 366 | } 367 | } 368 | 369 | /* position of next chunk */ 370 | fseek(in, pos + 16 + chunk_size, SEEK_SET); 371 | } 372 | printf("\n\n"); 373 | 374 | /* free allocated stuff */ 375 | free(compressed_buffer); 376 | free(decompressed_buffer); 377 | free(output_file); 378 | 379 | /* close working files */ 380 | if (f) fclose(f); 381 | fclose(in); 382 | 383 | /* so far so good */ 384 | return 0; 385 | } 386 | 387 | int main(int argc, char** argv) { 388 | int i; 389 | const char* archive_file; 390 | 391 | /* show help with no argument at all*/ 392 | if (argc == 1) { 393 | usage(); 394 | return 0; 395 | } 396 | 397 | /* check for help on usage */ 398 | for (i = 1; i <= argc; i++) 399 | if (argv[i]) 400 | if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { 401 | usage(); 402 | return 0; 403 | } 404 | 405 | /* check for version information */ 406 | for (i = 1; i <= argc; i++) 407 | if (argv[i]) 408 | if (!strcmp(argv[i], "-v") || !strcmp(argv[i], "--version")) { 409 | printf("6unpack: high-speed file compression tool\n"); 410 | printf("Version %s (using FastLZ %s)\n", SIXPACK_VERSION_STRING, FASTLZ_VERSION_STRING); 411 | printf("Copyright (C) Ariya Hidayat\n"); 412 | printf("\n"); 413 | return 0; 414 | } 415 | 416 | /* needs at least two arguments */ 417 | if (argc <= 1) { 418 | usage(); 419 | return 0; 420 | } 421 | 422 | archive_file = argv[1]; 423 | 424 | return unpack_file(archive_file); 425 | } 426 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS?=-Wall -std=c90 2 | 3 | all: 6pack 6unpack 4 | 5 | 6pack: 6pack.c ../fastlz.c 6 | $(CC) -o 6pack $(CFLAGS) -I.. 6pack.c ../fastlz.c 7 | 8 | 6unpack: 6unpack.c ../fastlz.c 9 | $(CC) -o 6unpack $(CFLAGS) -I.. 6unpack.c ../fastlz.c 10 | 11 | clean : 12 | $(RM) 6pack 6unpack *.o 13 | -------------------------------------------------------------------------------- /examples/Makefile.win: -------------------------------------------------------------------------------- 1 | CC=cl.exe 2 | CFLAGS=/Wall 3 | RM=del 4 | 5 | all: 6pack 6unpack 6 | 7 | 6pack: 6pack.c ../fastlz.c 8 | $(CC) -o 6pack $(CFLAGS) -I.. 6pack.c ../fastlz.c 9 | 10 | 6unpack: 6unpack.c ../fastlz.c 11 | $(CC) -o 6unpack $(CFLAGS) -I.. 6unpack.c ../fastlz.c 12 | 13 | clean : 14 | $(RM) 6pack.exe 6unpack.exe *.obj 15 | -------------------------------------------------------------------------------- /fastlz.c: -------------------------------------------------------------------------------- 1 | /* 2 | FastLZ - Byte-aligned LZ77 compression library 3 | Copyright (C) 2005-2020 Ariya Hidayat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | */ 23 | 24 | #include "fastlz.h" 25 | 26 | #include 27 | 28 | #pragma GCC diagnostic push 29 | #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" 30 | 31 | /* 32 | * Give hints to the compiler for branch prediction optimization. 33 | */ 34 | #if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2)) 35 | #define FASTLZ_LIKELY(c) (__builtin_expect(!!(c), 1)) 36 | #define FASTLZ_UNLIKELY(c) (__builtin_expect(!!(c), 0)) 37 | #else 38 | #define FASTLZ_LIKELY(c) (c) 39 | #define FASTLZ_UNLIKELY(c) (c) 40 | #endif 41 | 42 | /* 43 | * Specialize custom 64-bit implementation for speed improvements. 44 | */ 45 | #if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) 46 | #define FLZ_ARCH64 47 | #endif 48 | 49 | /* 50 | * Workaround for DJGPP to find uint8_t, uint16_t, etc. 51 | */ 52 | #if defined(__MSDOS__) && defined(__GNUC__) 53 | #include 54 | #endif 55 | 56 | #if defined(FASTLZ_USE_MEMMOVE) && (FASTLZ_USE_MEMMOVE == 0) 57 | 58 | static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { 59 | do { 60 | *dest++ = *src++; 61 | } while (--count); 62 | } 63 | 64 | static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { 65 | return fastlz_memmove(dest, src, count); 66 | } 67 | 68 | #else 69 | 70 | #include 71 | 72 | static void fastlz_memmove(uint8_t* dest, const uint8_t* src, uint32_t count) { 73 | if ((count > 4) && (dest >= src + count)) { 74 | memmove(dest, src, count); 75 | } else { 76 | switch (count) { 77 | default: 78 | do { 79 | *dest++ = *src++; 80 | } while (--count); 81 | break; 82 | case 3: 83 | *dest++ = *src++; 84 | case 2: 85 | *dest++ = *src++; 86 | case 1: 87 | *dest++ = *src++; 88 | case 0: 89 | break; 90 | } 91 | } 92 | } 93 | 94 | static void fastlz_memcpy(uint8_t* dest, const uint8_t* src, uint32_t count) { memcpy(dest, src, count); } 95 | 96 | #endif 97 | 98 | #if defined(FLZ_ARCH64) 99 | 100 | static uint32_t flz_readu32(const void* ptr) { return *(const uint32_t*)ptr; } 101 | 102 | static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) { 103 | const uint8_t* start = p; 104 | 105 | if (flz_readu32(p) == flz_readu32(q)) { 106 | p += 4; 107 | q += 4; 108 | } 109 | while (q < r) 110 | if (*p++ != *q++) break; 111 | return p - start; 112 | } 113 | 114 | #endif /* FLZ_ARCH64 */ 115 | 116 | #if !defined(FLZ_ARCH64) 117 | 118 | static uint32_t flz_readu32(const void* ptr) { 119 | const uint8_t* p = (const uint8_t*)ptr; 120 | return (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]; 121 | } 122 | 123 | static uint32_t flz_cmp(const uint8_t* p, const uint8_t* q, const uint8_t* r) { 124 | const uint8_t* start = p; 125 | while (q < r) 126 | if (*p++ != *q++) break; 127 | return p - start; 128 | } 129 | 130 | #endif /* !FLZ_ARCH64 */ 131 | 132 | #define MAX_COPY 32 133 | #define MAX_LEN 264 /* 256 + 8 */ 134 | #define MAX_L1_DISTANCE 8192 135 | #define MAX_L2_DISTANCE 8191 136 | #define MAX_FARDISTANCE (65535 + MAX_L2_DISTANCE - 1) 137 | 138 | #define HASH_LOG 13 139 | #define HASH_SIZE (1 << HASH_LOG) 140 | #define HASH_MASK (HASH_SIZE - 1) 141 | 142 | static uint16_t flz_hash(uint32_t v) { 143 | uint32_t h = (v * 2654435769LL) >> (32 - HASH_LOG); 144 | return h & HASH_MASK; 145 | } 146 | 147 | /* special case of memcpy: at most MAX_COPY bytes */ 148 | static void flz_smallcopy(uint8_t* dest, const uint8_t* src, uint32_t count) { 149 | #if defined(FLZ_ARCH64) 150 | if (count >= 4) { 151 | const uint32_t* p = (const uint32_t*)src; 152 | uint32_t* q = (uint32_t*)dest; 153 | while (count > 4) { 154 | *q++ = *p++; 155 | count -= 4; 156 | dest += 4; 157 | src += 4; 158 | } 159 | } 160 | #endif 161 | fastlz_memcpy(dest, src, count); 162 | } 163 | 164 | /* special case of memcpy: exactly MAX_COPY bytes */ 165 | static void flz_maxcopy(void* dest, const void* src) { 166 | #if defined(FLZ_ARCH64) 167 | const uint32_t* p = (const uint32_t*)src; 168 | uint32_t* q = (uint32_t*)dest; 169 | *q++ = *p++; 170 | *q++ = *p++; 171 | *q++ = *p++; 172 | *q++ = *p++; 173 | *q++ = *p++; 174 | *q++ = *p++; 175 | *q++ = *p++; 176 | *q++ = *p++; 177 | #else 178 | fastlz_memcpy(dest, src, MAX_COPY); 179 | #endif 180 | } 181 | 182 | static uint8_t* flz_literals(uint32_t runs, const uint8_t* src, uint8_t* dest) { 183 | while (runs >= MAX_COPY) { 184 | *dest++ = MAX_COPY - 1; 185 | flz_maxcopy(dest, src); 186 | src += MAX_COPY; 187 | dest += MAX_COPY; 188 | runs -= MAX_COPY; 189 | } 190 | if (runs > 0) { 191 | *dest++ = runs - 1; 192 | flz_smallcopy(dest, src, runs); 193 | dest += runs; 194 | } 195 | return dest; 196 | } 197 | 198 | static uint8_t* flz1_match(uint32_t len, uint32_t distance, uint8_t* op) { 199 | --distance; 200 | if (FASTLZ_UNLIKELY(len > MAX_LEN - 2)) 201 | while (len > MAX_LEN - 2) { 202 | *op++ = (7 << 5) + (distance >> 8); 203 | *op++ = MAX_LEN - 2 - 7 - 2; 204 | *op++ = (distance & 255); 205 | len -= MAX_LEN - 2; 206 | } 207 | if (len < 7) { 208 | *op++ = (len << 5) + (distance >> 8); 209 | *op++ = (distance & 255); 210 | } else { 211 | *op++ = (7 << 5) + (distance >> 8); 212 | *op++ = len - 7; 213 | *op++ = (distance & 255); 214 | } 215 | return op; 216 | } 217 | 218 | #define FASTLZ_BOUND_CHECK(cond) \ 219 | if (FASTLZ_UNLIKELY(!(cond))) return 0; 220 | 221 | static int fastlz1_compress(const void* input, int length, void* output) { 222 | const uint8_t* ip = (const uint8_t*)input; 223 | const uint8_t* ip_start = ip; 224 | const uint8_t* ip_bound = ip + length - 4; /* because readU32 */ 225 | const uint8_t* ip_limit = ip + length - 12 - 1; 226 | uint8_t* op = (uint8_t*)output; 227 | 228 | uint32_t htab[HASH_SIZE]; 229 | uint32_t seq, hash; 230 | 231 | /* initializes hash table */ 232 | for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0; 233 | 234 | /* we start with literal copy */ 235 | const uint8_t* anchor = ip; 236 | ip += 2; 237 | 238 | /* main loop */ 239 | while (FASTLZ_LIKELY(ip < ip_limit)) { 240 | const uint8_t* ref; 241 | uint32_t distance, cmp; 242 | 243 | /* find potential match */ 244 | do { 245 | seq = flz_readu32(ip) & 0xffffff; 246 | hash = flz_hash(seq); 247 | ref = ip_start + htab[hash]; 248 | htab[hash] = ip - ip_start; 249 | distance = ip - ref; 250 | cmp = FASTLZ_LIKELY(distance < MAX_L1_DISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000; 251 | if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; 252 | ++ip; 253 | } while (seq != cmp); 254 | 255 | if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; 256 | --ip; 257 | 258 | if (FASTLZ_LIKELY(ip > anchor)) { 259 | op = flz_literals(ip - anchor, anchor, op); 260 | } 261 | 262 | uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound); 263 | op = flz1_match(len, distance, op); 264 | 265 | /* update the hash at match boundary */ 266 | ip += len; 267 | seq = flz_readu32(ip); 268 | hash = flz_hash(seq & 0xffffff); 269 | htab[hash] = ip++ - ip_start; 270 | seq >>= 8; 271 | hash = flz_hash(seq); 272 | htab[hash] = ip++ - ip_start; 273 | 274 | anchor = ip; 275 | } 276 | 277 | uint32_t copy = (uint8_t*)input + length - anchor; 278 | op = flz_literals(copy, anchor, op); 279 | 280 | return op - (uint8_t*)output; 281 | } 282 | 283 | static int fastlz1_decompress(const void* input, int length, void* output, int maxout) { 284 | const uint8_t* ip = (const uint8_t*)input; 285 | const uint8_t* ip_limit = ip + length; 286 | const uint8_t* ip_bound = ip_limit - 2; 287 | uint8_t* op = (uint8_t*)output; 288 | uint8_t* op_limit = op + maxout; 289 | uint32_t ctrl = (*ip++) & 31; 290 | 291 | while (1) { 292 | if (ctrl >= 32) { 293 | uint32_t len = (ctrl >> 5) - 1; 294 | uint32_t ofs = (ctrl & 31) << 8; 295 | const uint8_t* ref = op - ofs - 1; 296 | if (len == 7 - 1) { 297 | FASTLZ_BOUND_CHECK(ip <= ip_bound); 298 | len += *ip++; 299 | } 300 | ref -= *ip++; 301 | len += 3; 302 | FASTLZ_BOUND_CHECK(op + len <= op_limit); 303 | FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); 304 | fastlz_memmove(op, ref, len); 305 | op += len; 306 | } else { 307 | ctrl++; 308 | FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); 309 | FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); 310 | fastlz_memcpy(op, ip, ctrl); 311 | ip += ctrl; 312 | op += ctrl; 313 | } 314 | 315 | if (FASTLZ_UNLIKELY(ip > ip_bound)) break; 316 | ctrl = *ip++; 317 | } 318 | 319 | return op - (uint8_t*)output; 320 | } 321 | 322 | static uint8_t* flz2_match(uint32_t len, uint32_t distance, uint8_t* op) { 323 | --distance; 324 | if (distance < MAX_L2_DISTANCE) { 325 | if (len < 7) { 326 | *op++ = (len << 5) + (distance >> 8); 327 | *op++ = (distance & 255); 328 | } else { 329 | *op++ = (7 << 5) + (distance >> 8); 330 | for (len -= 7; len >= 255; len -= 255) *op++ = 255; 331 | *op++ = len; 332 | *op++ = (distance & 255); 333 | } 334 | } else { 335 | /* far away, but not yet in the another galaxy... */ 336 | if (len < 7) { 337 | distance -= MAX_L2_DISTANCE; 338 | *op++ = (len << 5) + 31; 339 | *op++ = 255; 340 | *op++ = distance >> 8; 341 | *op++ = distance & 255; 342 | } else { 343 | distance -= MAX_L2_DISTANCE; 344 | *op++ = (7 << 5) + 31; 345 | for (len -= 7; len >= 255; len -= 255) *op++ = 255; 346 | *op++ = len; 347 | *op++ = 255; 348 | *op++ = distance >> 8; 349 | *op++ = distance & 255; 350 | } 351 | } 352 | return op; 353 | } 354 | 355 | static int fastlz2_compress(const void* input, int length, void* output) { 356 | const uint8_t* ip = (const uint8_t*)input; 357 | const uint8_t* ip_start = ip; 358 | const uint8_t* ip_bound = ip + length - 4; /* because readU32 */ 359 | const uint8_t* ip_limit = ip + length - 12 - 1; 360 | uint8_t* op = (uint8_t*)output; 361 | 362 | uint32_t htab[HASH_SIZE]; 363 | uint32_t seq, hash; 364 | 365 | /* initializes hash table */ 366 | for (hash = 0; hash < HASH_SIZE; ++hash) htab[hash] = 0; 367 | 368 | /* we start with literal copy */ 369 | const uint8_t* anchor = ip; 370 | ip += 2; 371 | 372 | /* main loop */ 373 | while (FASTLZ_LIKELY(ip < ip_limit)) { 374 | const uint8_t* ref; 375 | uint32_t distance, cmp; 376 | 377 | /* find potential match */ 378 | do { 379 | seq = flz_readu32(ip) & 0xffffff; 380 | hash = flz_hash(seq); 381 | ref = ip_start + htab[hash]; 382 | htab[hash] = ip - ip_start; 383 | distance = ip - ref; 384 | cmp = FASTLZ_LIKELY(distance < MAX_FARDISTANCE) ? flz_readu32(ref) & 0xffffff : 0x1000000; 385 | if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; 386 | ++ip; 387 | } while (seq != cmp); 388 | 389 | if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; 390 | 391 | --ip; 392 | 393 | /* far, needs at least 5-byte match */ 394 | if (distance >= MAX_L2_DISTANCE) { 395 | if (ref[3] != ip[3] || ref[4] != ip[4]) { 396 | ++ip; 397 | continue; 398 | } 399 | } 400 | 401 | if (FASTLZ_LIKELY(ip > anchor)) { 402 | op = flz_literals(ip - anchor, anchor, op); 403 | } 404 | 405 | uint32_t len = flz_cmp(ref + 3, ip + 3, ip_bound); 406 | op = flz2_match(len, distance, op); 407 | 408 | /* update the hash at match boundary */ 409 | ip += len; 410 | seq = flz_readu32(ip); 411 | hash = flz_hash(seq & 0xffffff); 412 | htab[hash] = ip++ - ip_start; 413 | seq >>= 8; 414 | hash = flz_hash(seq); 415 | htab[hash] = ip++ - ip_start; 416 | 417 | anchor = ip; 418 | } 419 | 420 | uint32_t copy = (uint8_t*)input + length - anchor; 421 | op = flz_literals(copy, anchor, op); 422 | 423 | /* marker for fastlz2 */ 424 | *(uint8_t*)output |= (1 << 5); 425 | 426 | return op - (uint8_t*)output; 427 | } 428 | 429 | static int fastlz2_decompress(const void* input, int length, void* output, int maxout) { 430 | const uint8_t* ip = (const uint8_t*)input; 431 | const uint8_t* ip_limit = ip + length; 432 | const uint8_t* ip_bound = ip_limit - 2; 433 | uint8_t* op = (uint8_t*)output; 434 | uint8_t* op_limit = op + maxout; 435 | uint32_t ctrl = (*ip++) & 31; 436 | 437 | while (1) { 438 | if (ctrl >= 32) { 439 | uint32_t len = (ctrl >> 5) - 1; 440 | uint32_t ofs = (ctrl & 31) << 8; 441 | const uint8_t* ref = op - ofs - 1; 442 | 443 | uint8_t code; 444 | if (len == 7 - 1) do { 445 | FASTLZ_BOUND_CHECK(ip <= ip_bound); 446 | code = *ip++; 447 | len += code; 448 | } while (code == 255); 449 | code = *ip++; 450 | ref -= code; 451 | len += 3; 452 | 453 | /* match from 16-bit distance */ 454 | if (FASTLZ_UNLIKELY(code == 255)) 455 | if (FASTLZ_LIKELY(ofs == (31 << 8))) { 456 | FASTLZ_BOUND_CHECK(ip < ip_bound); 457 | ofs = (*ip++) << 8; 458 | ofs += *ip++; 459 | ref = op - ofs - MAX_L2_DISTANCE - 1; 460 | } 461 | 462 | FASTLZ_BOUND_CHECK(op + len <= op_limit); 463 | FASTLZ_BOUND_CHECK(ref >= (uint8_t*)output); 464 | fastlz_memmove(op, ref, len); 465 | op += len; 466 | } else { 467 | ctrl++; 468 | FASTLZ_BOUND_CHECK(op + ctrl <= op_limit); 469 | FASTLZ_BOUND_CHECK(ip + ctrl <= ip_limit); 470 | fastlz_memcpy(op, ip, ctrl); 471 | ip += ctrl; 472 | op += ctrl; 473 | } 474 | 475 | if (FASTLZ_UNLIKELY(ip >= ip_limit)) break; 476 | ctrl = *ip++; 477 | } 478 | 479 | return op - (uint8_t*)output; 480 | } 481 | 482 | int fastlz_compress(const void* input, int length, void* output) { 483 | /* for short block, choose fastlz1 */ 484 | if (length < 65536) return fastlz1_compress(input, length, output); 485 | 486 | /* else... */ 487 | return fastlz2_compress(input, length, output); 488 | } 489 | 490 | int fastlz_decompress(const void* input, int length, void* output, int maxout) { 491 | /* magic identifier for compression level */ 492 | int level = ((*(const uint8_t*)input) >> 5) + 1; 493 | 494 | if (level == 1) return fastlz1_decompress(input, length, output, maxout); 495 | if (level == 2) return fastlz2_decompress(input, length, output, maxout); 496 | 497 | /* unknown level, trigger error */ 498 | return 0; 499 | } 500 | 501 | int fastlz_compress_level(int level, const void* input, int length, void* output) { 502 | if (level == 1) return fastlz1_compress(input, length, output); 503 | if (level == 2) return fastlz2_compress(input, length, output); 504 | 505 | return 0; 506 | } 507 | 508 | #pragma GCC diagnostic pop 509 | -------------------------------------------------------------------------------- /fastlz.h: -------------------------------------------------------------------------------- 1 | /* 2 | FastLZ - Byte-aligned LZ77 compression library 3 | Copyright (C) 2005-2020 Ariya Hidayat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | */ 23 | 24 | #ifndef FASTLZ_H 25 | #define FASTLZ_H 26 | 27 | #define FASTLZ_VERSION 0x000500 28 | 29 | #define FASTLZ_VERSION_MAJOR 0 30 | #define FASTLZ_VERSION_MINOR 5 31 | #define FASTLZ_VERSION_REVISION 0 32 | 33 | #define FASTLZ_VERSION_STRING "0.5.0" 34 | 35 | #if defined(__cplusplus) 36 | extern "C" { 37 | #endif 38 | 39 | /** 40 | Compress a block of data in the input buffer and returns the size of 41 | compressed block. The size of input buffer is specified by length. The 42 | minimum input buffer size is 16. 43 | 44 | The output buffer must be at least 5% larger than the input buffer 45 | and can not be smaller than 66 bytes. 46 | 47 | If the input is not compressible, the return value might be larger than 48 | length (input buffer size). 49 | 50 | The input buffer and the output buffer can not overlap. 51 | 52 | Compression level can be specified in parameter level. At the moment, 53 | only level 1 and level 2 are supported. 54 | Level 1 is the fastest compression and generally useful for short data. 55 | Level 2 is slightly slower but it gives better compression ratio. 56 | 57 | Note that the compressed data, regardless of the level, can always be 58 | decompressed using the function fastlz_decompress below. 59 | */ 60 | 61 | int fastlz_compress_level(int level, const void* input, int length, void* output); 62 | 63 | /** 64 | Decompress a block of compressed data and returns the size of the 65 | decompressed block. If error occurs, e.g. the compressed data is 66 | corrupted or the output buffer is not large enough, then 0 (zero) 67 | will be returned instead. 68 | 69 | The input buffer and the output buffer can not overlap. 70 | 71 | Decompression is memory safe and guaranteed not to write the output buffer 72 | more than what is specified in maxout. 73 | 74 | Note that the decompression will always work, regardless of the 75 | compression level specified in fastlz_compress_level above (when 76 | producing the compressed block). 77 | */ 78 | 79 | int fastlz_decompress(const void* input, int length, void* output, int maxout); 80 | 81 | /** 82 | DEPRECATED. 83 | 84 | This is similar to fastlz_compress_level above, but with the level 85 | automatically chosen. 86 | 87 | This function is deprecated and it will be completely removed in some future 88 | version. 89 | */ 90 | 91 | int fastlz_compress(const void* input, int length, void* output); 92 | 93 | #if defined(__cplusplus) 94 | } 95 | #endif 96 | 97 | #endif /* FASTLZ_H */ 98 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS?=-Wall -std=c90 2 | TEST_ROUNDTRIP?=./test_roundtrip 3 | 4 | all: roundtrip 5 | 6 | roundtrip: test_roundtrip 7 | $(TEST_ROUNDTRIP) 8 | 9 | test_roundtrip: test_roundtrip.c ../fastlz.c refimpl.c 10 | $(CC) -o $(TEST_ROUNDTRIP) $(CFLAGS) -I.. test_roundtrip.c ../fastlz.c refimpl.c 11 | 12 | clean : 13 | $(RM) $(TEST_ROUNDTRIP) *.o 14 | -------------------------------------------------------------------------------- /tests/Makefile.win: -------------------------------------------------------------------------------- 1 | CC=cl.exe 2 | CFLAGS=/Wall /Za 3 | RM=del 4 | 5 | TEST_ROUNDTRIP=test_roundtrip.exe 6 | 7 | all: roundtrip 8 | 9 | roundtrip: test_roundtrip.c ../fastlz.c refimpl.c 10 | $(CC) -o $(TEST_ROUNDTRIP) $(CFLAGS) -I.. test_roundtrip.c ../fastlz.c refimpl.c 11 | $(TEST_ROUNDTRIP) 12 | 13 | clean : 14 | $(RM) $(TEST_ROUNDTRIP) *.obj 15 | -------------------------------------------------------------------------------- /tests/refimpl.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | /* 4 | * Workaround for DJGPP to find uint8_t, uint16_t, etc. 5 | */ 6 | #if defined(__MSDOS__) && defined(__GNUC__) 7 | #include 8 | #endif 9 | 10 | void REF_Level1_decompress(const uint8_t* input, int length, uint8_t* output) { 11 | int src = 0; 12 | int dest = 0; 13 | while (src < length) { 14 | int type = input[src] >> 5; 15 | if (type == 0) { 16 | /* literal run */ 17 | int run = 1 + input[src]; 18 | src = src + 1; 19 | while (run > 0) { 20 | output[dest] = input[src]; 21 | src = src + 1; 22 | dest = dest + 1; 23 | run = run - 1; 24 | } 25 | } else if (type < 7) { 26 | /* short match */ 27 | int ofs = 256 * (input[src] & 31) + input[src + 1]; 28 | int len = 2 + (input[src] >> 5); 29 | src = src + 2; 30 | int ref = dest - ofs - 1; 31 | while (len > 0) { 32 | output[dest] = output[ref]; 33 | ref = ref + 1; 34 | dest = dest + 1; 35 | len = len - 1; 36 | } 37 | } else { 38 | /* long match */ 39 | int ofs = 256 * (input[src] & 31) + input[src + 2]; 40 | int len = 9 + input[src + 1]; 41 | src = src + 3; 42 | int ref = dest - ofs - 1; 43 | while (len > 0) { 44 | output[dest] = output[ref]; 45 | ref = ref + 1; 46 | dest = dest + 1; 47 | len = len - 1; 48 | } 49 | } 50 | } 51 | } 52 | 53 | void REF_Level2_decompress(const uint8_t* input, int length, uint8_t* output) { 54 | int src = 0; 55 | int dest = 0; 56 | while (src < length) { 57 | int type = input[src] >> 5; 58 | if (type == 0) { 59 | /* literal run */ 60 | int run = 1 + input[src]; 61 | src = src + 1; 62 | while (run > 0) { 63 | output[dest] = input[src]; 64 | src = src + 1; 65 | dest = dest + 1; 66 | run = run - 1; 67 | } 68 | } else { 69 | int next = 2; 70 | int len = 2 + (input[src] >> 5); 71 | if (len == 9) { 72 | /* long match */ 73 | next = next + 1; 74 | len = len + input[src + 1]; 75 | if (len == 9 + 255) { 76 | /* Gamma code for match length */ 77 | int nn = input[src + 1]; 78 | while (nn == 255) { 79 | nn = input[src + next - 1]; 80 | next = next + 1; 81 | len += nn; 82 | } 83 | } 84 | } 85 | 86 | int ofs = 256 * (input[src] & 31) + input[src + next - 1]; 87 | if (ofs == 8191) { 88 | /* match from 16-bit distance */ 89 | ofs += 256 * input[src + next] + input[src + next + 1]; 90 | next = next + 2; 91 | } 92 | src = src + next; 93 | 94 | int ref = dest - ofs - 1; 95 | while (len > 0) { 96 | output[dest] = output[ref]; 97 | ref = ref + 1; 98 | dest = dest + 1; 99 | len = len - 1; 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /tests/test_roundtrip.c: -------------------------------------------------------------------------------- 1 | /* 2 | FastLZ - Byte-aligned LZ77 compression library 3 | Copyright (C) 2005-2020 Ariya Hidayat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "fastlz.h" 30 | 31 | /* 32 | * Workaround for DJGPP to find uint8_t, uint16_t, etc. 33 | */ 34 | #if defined(__MSDOS__) && defined(__GNUC__) 35 | #include 36 | #endif 37 | 38 | #define LOG 39 | #undef LOG 40 | 41 | int compare(const char* name, const uint8_t* a, const uint8_t* b, int size) { 42 | int bad = 0; 43 | int i; 44 | for (i = 0; i < size; ++i) { 45 | if (a[i] != b[i]) { 46 | bad = 1; 47 | printf("Error on %s!\n", name); 48 | printf("Different at index %d: expecting %02x,actual %02x\n", i, a[i], b[i]); 49 | break; 50 | } 51 | } 52 | return bad; 53 | } 54 | 55 | #if !defined(__MSDOS__) 56 | #define MAX_FILE_SIZE (100 * 1024 * 1024) 57 | #else 58 | #define MAX_FILE_SIZE (32 * 1024 * 1024) 59 | #endif 60 | 61 | /* prototype, implemented in refimpl.c */ 62 | void REF_Level1_decompress(const uint8_t* input, int length, uint8_t* output); 63 | void REF_Level2_decompress(const uint8_t* input, int length, uint8_t* output); 64 | 65 | /* 66 | Same as test_roundtrip_level1 EXCEPT that the decompression is carried out 67 | using the highly-simplified, unoptimized vanilla reference decompressor. 68 | */ 69 | 70 | void test_ref_decompressor_level1(const char* name, const char* file_name) { 71 | #ifdef LOG 72 | printf("Processing %s...\n", name); 73 | #endif 74 | FILE* f = fopen(file_name, "rb"); 75 | if (!f) { 76 | printf("Error: can not open %s!\n", file_name); 77 | exit(1); 78 | } 79 | fseek(f, 0L, SEEK_END); 80 | long file_size = ftell(f); 81 | rewind(f); 82 | 83 | #ifdef LOG 84 | printf("Size is %ld bytes.\n", file_size); 85 | #endif 86 | if (file_size > MAX_FILE_SIZE) { 87 | fclose(f); 88 | printf("%25s %10ld [skipped, file too big]\n", name, file_size); 89 | return; 90 | } 91 | 92 | uint8_t* file_buffer = malloc(file_size); 93 | long read = fread(file_buffer, 1, file_size, f); 94 | fclose(f); 95 | if (read != file_size) { 96 | free(file_buffer); 97 | printf("Error: only read %ld bytes!\n", read); 98 | exit(1); 99 | } 100 | 101 | #ifdef LOG 102 | printf("Compressing. Please wait...\n"); 103 | #endif 104 | uint8_t* compressed_buffer = malloc(1.05 * file_size); 105 | int compressed_size = fastlz_compress_level(1, file_buffer, file_size, compressed_buffer); 106 | double ratio = (100.0 * compressed_size) / file_size; 107 | #ifdef LOG 108 | printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio); 109 | #endif 110 | 111 | #ifdef LOG 112 | printf("Decompressing. Please wait...\n"); 113 | #endif 114 | uint8_t* uncompressed_buffer = malloc(file_size); 115 | if (uncompressed_buffer == NULL) { 116 | printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio); 117 | return; 118 | } 119 | memset(uncompressed_buffer, '-', file_size); 120 | REF_Level1_decompress(compressed_buffer, compressed_size, uncompressed_buffer); 121 | #ifdef LOG 122 | printf("Comparing. Please wait...\n"); 123 | #endif 124 | int result = compare(file_name, file_buffer, uncompressed_buffer, file_size); 125 | if (result == 1) { 126 | free(uncompressed_buffer); 127 | exit(1); 128 | } 129 | 130 | free(file_buffer); 131 | free(compressed_buffer); 132 | free(uncompressed_buffer); 133 | #ifdef LOG 134 | printf("OK.\n"); 135 | #else 136 | printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio); 137 | #endif 138 | } 139 | 140 | /* 141 | Same as test_roundtrip_level2 EXCEPT that the decompression is carried out 142 | using the highly-simplified, unoptimized vanilla reference decompressor. 143 | */ 144 | 145 | void test_ref_decompressor_level2(const char* name, const char* file_name) { 146 | #ifdef LOG 147 | printf("Processing %s...\n", name); 148 | #endif 149 | FILE* f = fopen(file_name, "rb"); 150 | if (!f) { 151 | printf("Error: can not open %s!\n", file_name); 152 | exit(1); 153 | } 154 | fseek(f, 0L, SEEK_END); 155 | long file_size = ftell(f); 156 | rewind(f); 157 | 158 | #ifdef LOG 159 | printf("Size is %ld bytes.\n", file_size); 160 | #endif 161 | if (file_size > MAX_FILE_SIZE) { 162 | fclose(f); 163 | printf("%25s %10ld [skipped, file too big]\n", name, file_size); 164 | return; 165 | } 166 | 167 | uint8_t* file_buffer = malloc(file_size); 168 | long read = fread(file_buffer, 1, file_size, f); 169 | fclose(f); 170 | if (read != file_size) { 171 | free(file_buffer); 172 | printf("Error: only read %ld bytes!\n", read); 173 | exit(1); 174 | } 175 | 176 | #ifdef LOG 177 | printf("Compressing. Please wait...\n"); 178 | #endif 179 | uint8_t* compressed_buffer = malloc(1.05 * file_size); 180 | int compressed_size = fastlz_compress_level(2, file_buffer, file_size, compressed_buffer); 181 | double ratio = (100.0 * compressed_size) / file_size; 182 | #ifdef LOG 183 | printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio); 184 | #endif 185 | 186 | #ifdef LOG 187 | printf("Decompressing. Please wait...\n"); 188 | #endif 189 | uint8_t* uncompressed_buffer = malloc(file_size); 190 | if (uncompressed_buffer == NULL) { 191 | printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio); 192 | return; 193 | } 194 | memset(uncompressed_buffer, '-', file_size); 195 | 196 | /* intentionally mask out the block tag */ 197 | compressed_buffer[0] = compressed_buffer[0] & 31; 198 | 199 | REF_Level2_decompress(compressed_buffer, compressed_size, uncompressed_buffer); 200 | #ifdef LOG 201 | printf("Comparing. Please wait...\n"); 202 | #endif 203 | int result = compare(file_name, file_buffer, uncompressed_buffer, file_size); 204 | if (result == 1) { 205 | free(uncompressed_buffer); 206 | exit(1); 207 | } 208 | 209 | free(file_buffer); 210 | free(compressed_buffer); 211 | free(uncompressed_buffer); 212 | #ifdef LOG 213 | printf("OK.\n"); 214 | #else 215 | printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio); 216 | #endif 217 | } 218 | 219 | /* 220 | Read the content of the file. 221 | Compress it first using the Level 1 compressor. 222 | Decompress the output with Level 1 decompressor. 223 | Compare the result with the original file content. 224 | */ 225 | void test_roundtrip_level1(const char* name, const char* file_name) { 226 | #ifdef LOG 227 | printf("Processing %s...\n", name); 228 | #endif 229 | FILE* f = fopen(file_name, "rb"); 230 | if (!f) { 231 | printf("Error: can not open %s!\n", file_name); 232 | exit(1); 233 | } 234 | fseek(f, 0L, SEEK_END); 235 | long file_size = ftell(f); 236 | rewind(f); 237 | 238 | #ifdef LOG 239 | printf("Size is %ld bytes.\n", file_size); 240 | #endif 241 | if (file_size > MAX_FILE_SIZE) { 242 | fclose(f); 243 | printf("%25s %10ld [skipped, file too big]\n", name, file_size); 244 | return; 245 | } 246 | 247 | uint8_t* file_buffer = malloc(file_size); 248 | long read = fread(file_buffer, 1, file_size, f); 249 | fclose(f); 250 | if (read != file_size) { 251 | free(file_buffer); 252 | printf("Error: only read %ld bytes!\n", read); 253 | exit(1); 254 | } 255 | 256 | #ifdef LOG 257 | printf("Compressing. Please wait...\n"); 258 | #endif 259 | uint8_t* compressed_buffer = malloc(1.05 * file_size); 260 | int compressed_size = fastlz_compress_level(1, file_buffer, file_size, compressed_buffer); 261 | double ratio = (100.0 * compressed_size) / file_size; 262 | #ifdef LOG 263 | printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio); 264 | #endif 265 | 266 | #ifdef LOG 267 | printf("Decompressing. Please wait...\n"); 268 | #endif 269 | uint8_t* uncompressed_buffer = malloc(file_size); 270 | if (uncompressed_buffer == NULL) { 271 | printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress\n", name, file_size, compressed_size, ratio); 272 | return; 273 | } 274 | memset(uncompressed_buffer, '-', file_size); 275 | fastlz_decompress(compressed_buffer, compressed_size, uncompressed_buffer, file_size); 276 | #ifdef LOG 277 | printf("Comparing. Please wait...\n"); 278 | #endif 279 | int result = compare(file_name, file_buffer, uncompressed_buffer, file_size); 280 | if (result == 1) { 281 | free(uncompressed_buffer); 282 | exit(1); 283 | } 284 | 285 | free(file_buffer); 286 | free(compressed_buffer); 287 | free(uncompressed_buffer); 288 | #ifdef LOG 289 | printf("OK.\n"); 290 | #else 291 | printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio); 292 | #endif 293 | } 294 | 295 | /* 296 | Read the content of the file. 297 | Compress it first using the Level 2 compressor. 298 | Decompress the output with Level 2 decompressor. 299 | Compare the result with the original file content. 300 | */ 301 | void test_roundtrip_level2(const char* name, const char* file_name) { 302 | #ifdef LOG 303 | printf("Processing %s...\n", name); 304 | #endif 305 | FILE* f = fopen(file_name, "rb"); 306 | if (!f) { 307 | printf("Error: can not open %s!\n", file_name); 308 | exit(1); 309 | } 310 | fseek(f, 0L, SEEK_END); 311 | long file_size = ftell(f); 312 | rewind(f); 313 | 314 | #ifdef LOG 315 | printf("Size is %ld bytes.\n", file_size); 316 | #endif 317 | if (file_size > MAX_FILE_SIZE) { 318 | fclose(f); 319 | printf("%25s %10ld [skipped, file too big]\n", name, file_size); 320 | return; 321 | } 322 | 323 | uint8_t* file_buffer = malloc(file_size); 324 | long read = fread(file_buffer, 1, file_size, f); 325 | fclose(f); 326 | if (read != file_size) { 327 | free(file_buffer); 328 | printf("Error: only read %ld bytes!\n", read); 329 | exit(1); 330 | } 331 | 332 | #ifdef LOG 333 | printf("Compressing. Please wait...\n"); 334 | #endif 335 | uint8_t* compressed_buffer = malloc(1.05 * file_size); 336 | int compressed_size = fastlz_compress_level(2, file_buffer, file_size, compressed_buffer); 337 | double ratio = (100.0 * compressed_size) / file_size; 338 | #ifdef LOG 339 | printf("Compressing was completed: %ld -> %ld (%.2f%%)\n", file_size, compressed_size, ratio); 340 | #endif 341 | 342 | #ifdef LOG 343 | printf("Decompressing. Please wait...\n"); 344 | #endif 345 | uint8_t* uncompressed_buffer = malloc(file_size); 346 | if (uncompressed_buffer == NULL) { 347 | free(file_buffer); 348 | free(compressed_buffer); 349 | printf("%25s %10ld -> %10d (%.2f%%) skipped, can't decompress OOM\n", name, file_size, compressed_size, ratio); 350 | exit(1); 351 | return; 352 | } 353 | memset(uncompressed_buffer, '-', file_size); 354 | fastlz_decompress(compressed_buffer, compressed_size, uncompressed_buffer, file_size); 355 | #ifdef LOG 356 | printf("Comparing. Please wait...\n"); 357 | #endif 358 | int result = compare(file_name, file_buffer, uncompressed_buffer, file_size); 359 | if (result == 1) { 360 | free(uncompressed_buffer); 361 | exit(1); 362 | } 363 | 364 | free(file_buffer); 365 | free(compressed_buffer); 366 | free(uncompressed_buffer); 367 | #ifdef LOG 368 | printf("OK.\n"); 369 | #else 370 | printf("%25s %10ld -> %10d (%.2f%%)\n", name, file_size, compressed_size, ratio); 371 | #endif 372 | } 373 | 374 | int main(int argc, char** argv) { 375 | const char* default_prefix = "../compression-corpus/"; 376 | const char* names[] = {"canterbury/alice29.txt", 377 | "canterbury/asyoulik.txt", 378 | "canterbury/cp.html", 379 | "canterbury/fields.c", 380 | "canterbury/grammar.lsp", 381 | "canterbury/kennedy.xls", 382 | "canterbury/lcet10.txt", 383 | "canterbury/plrabn12.txt", 384 | "canterbury/ptt5", 385 | "canterbury/sum", 386 | "canterbury/xargs.1", 387 | "silesia/dickens", 388 | "silesia/mozilla", 389 | "silesia/mr", 390 | "silesia/nci", 391 | "silesia/ooffice", 392 | "silesia/osdb", 393 | "silesia/reymont", 394 | "silesia/samba", 395 | "silesia/sao", 396 | "silesia/webster", 397 | "silesia/x-ray", 398 | "silesia/xml", 399 | "enwik/enwik8.txt"}; 400 | 401 | const char* prefix = (argc == 2) ? argv[1] : default_prefix; 402 | 403 | const int count = sizeof(names) / sizeof(names[0]); 404 | int i; 405 | 406 | printf("Test reference decompressor for Level 1\n\n"); 407 | for (i = 0; i < count; ++i) { 408 | const char* name = names[i]; 409 | char* filename = malloc(strlen(prefix) + strlen(name) + 1); 410 | strcpy(filename, prefix); 411 | strcat(filename, name); 412 | test_ref_decompressor_level1(name, filename); 413 | free(filename); 414 | } 415 | printf("\n"); 416 | 417 | printf("Test reference decompressor for Level 2\n\n"); 418 | for (i = 0; i < count; ++i) { 419 | const char* name = names[i]; 420 | char* filename = malloc(strlen(prefix) + strlen(name) + 1); 421 | strcpy(filename, prefix); 422 | strcat(filename, name); 423 | test_ref_decompressor_level2(name, filename); 424 | free(filename); 425 | } 426 | printf("\n"); 427 | 428 | printf("Test round-trip for Level 1\n\n"); 429 | for (i = 0; i < count; ++i) { 430 | const char* name = names[i]; 431 | char* filename = malloc(strlen(prefix) + strlen(name) + 1); 432 | strcpy(filename, prefix); 433 | strcat(filename, name); 434 | test_roundtrip_level1(name, filename); 435 | free(filename); 436 | } 437 | printf("\n"); 438 | 439 | printf("Test round-trip for Level 2\n\n"); 440 | for (i = 0; i < count; ++i) { 441 | const char* name = names[i]; 442 | char* filename = malloc(strlen(prefix) + strlen(name) + 1); 443 | strcpy(filename, prefix); 444 | strcat(filename, name); 445 | test_roundtrip_level2(name, filename); 446 | free(filename); 447 | } 448 | printf("\n"); 449 | 450 | return 0; 451 | } 452 | -------------------------------------------------------------------------------- /tools/format-code.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cwd=$(pwd) 4 | clang-format-6.0 -i --style='{BasedOnStyle: "google", ColumnLimit: 120}' $cwd/*.h $cwd/*.c $cwd/tests/*.c $cwd/examples/*.c 5 | --------------------------------------------------------------------------------