├── README.md ├── doc.go ├── Makefile ├── .gitignore ├── .github └── workflows │ └── semgrep.yml ├── lz4_hc.go ├── LICENSE ├── lz4.go ├── lz4_test.go ├── lz4_hc_test.go ├── sample.txt └── src ├── lz4hc.h ├── lz4.h ├── lz4hc.c └── lz4.c /README.md: -------------------------------------------------------------------------------- 1 | golz4 2 | ===== 3 | 4 | Golang interface to LZ4 compression 5 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package lz4 implements compression using lz4.c and lz4hc.c 2 | // 3 | // Copyright (c) 2013 CloudFlare, Inc. 4 | package lz4 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GCFLAGS := 2 | LDFLAGS := 3 | 4 | .PHONY: install 5 | install: 6 | @go install -v . 7 | 8 | .PHONY: test 9 | test: 10 | @go test -gcflags='$(GCFLAGS)' -ldflags='$(LDFLAGS)' . 11 | 12 | .PHONY: bench 13 | bench: 14 | @go test -gcflags='$(GCFLAGS)' -ldflags='$(LDFLAGS)' -bench . 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /.github/workflows/semgrep.yml: -------------------------------------------------------------------------------- 1 | 2 | on: 3 | pull_request: {} 4 | workflow_dispatch: {} 5 | push: 6 | branches: 7 | - main 8 | - master 9 | name: Semgrep config 10 | jobs: 11 | semgrep: 12 | name: semgrep/ci 13 | runs-on: ubuntu-20.04 14 | env: 15 | SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} 16 | SEMGREP_URL: https://cloudflare.semgrep.dev 17 | SEMGREP_APP_URL: https://cloudflare.semgrep.dev 18 | SEMGREP_VERSION_CHECK_URL: https://cloudflare.semgrep.dev/api/check-version 19 | container: 20 | image: returntocorp/semgrep 21 | steps: 22 | - uses: actions/checkout@v3 23 | - run: semgrep ci 24 | -------------------------------------------------------------------------------- /lz4_hc.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | // #cgo CFLAGS: -O3 4 | // #include "src/lz4hc.h" 5 | // #include "src/lz4hc.c" 6 | import "C" 7 | 8 | import ( 9 | "fmt" 10 | ) 11 | 12 | // CompressHC compresses in and puts the content in out. len(out) 13 | // should have enough space for the compressed data (use CompressBound 14 | // to calculate). Returns the number of bytes in the out slice. Determines 15 | // the compression level automatically. 16 | func CompressHC(in, out []byte) (int, error) { 17 | // 0 automatically sets the compression level. 18 | return CompressHCLevel(in, out, 0) 19 | } 20 | 21 | // CompressHCLevel compresses in at the given compression level and puts the 22 | // content in out. len(out) should have enough space for the compressed data 23 | // (use CompressBound to calculate). Returns the number of bytes in the out 24 | // slice. To automatically choose the compression level, use 0. Otherwise, use 25 | // any value in the inclusive range 1 (worst) through 16 (best). Most 26 | // applications will prefer CompressHC. 27 | func CompressHCLevel(in, out []byte, level int) (outSize int, err error) { 28 | // LZ4HC does not handle empty buffers. Pass through to Compress. 29 | if len(in) == 0 || len(out) == 0 { 30 | return Compress(in, out) 31 | } 32 | 33 | outSize = int(C.LZ4_compressHC2_limitedOutput(p(in), p(out), clen(in), clen(out), C.int(level))) 34 | if outSize == 0 { 35 | err = fmt.Errorf("insufficient space for compression") 36 | } 37 | return 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 CloudFlare, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | * Neither the name of the CloudFlare, Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /lz4.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | // #cgo CFLAGS: -O3 4 | // #include "src/lz4.h" 5 | // #include "src/lz4.c" 6 | import "C" 7 | 8 | import ( 9 | "errors" 10 | "fmt" 11 | "unsafe" 12 | ) 13 | 14 | // p gets a char pointer to the first byte of a []byte slice 15 | func p(in []byte) *C.char { 16 | if len(in) == 0 { 17 | return (*C.char)(unsafe.Pointer(nil)) 18 | } 19 | return (*C.char)(unsafe.Pointer(&in[0])) 20 | } 21 | 22 | // clen gets the length of a []byte slice as a char * 23 | func clen(s []byte) C.int { 24 | return C.int(len(s)) 25 | } 26 | 27 | // Uncompress with a known output size. len(out) should be equal to 28 | // the length of the uncompressed out. 29 | func Uncompress(in, out []byte) (error) { 30 | if int(C.LZ4_decompress_safe(p(in), p(out), clen(in), clen(out))) < 0 { 31 | return errors.New("Malformed compression stream") 32 | } 33 | 34 | return nil 35 | } 36 | 37 | // CompressBound calculates the size of the output buffer needed by 38 | // Compress. This is based on the following macro: 39 | // 40 | // #define LZ4_COMPRESSBOUND(isize) 41 | // ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 42 | func CompressBound(in []byte) int { 43 | return len(in) + ((len(in) / 255) + 16) 44 | } 45 | 46 | // Compress compresses in and puts the content in out. len(out) 47 | // should have enough space for the compressed data (use CompressBound 48 | // to calculate). Returns the number of bytes in the out slice. 49 | func Compress(in, out []byte) (outSize int, err error) { 50 | outSize = int(C.LZ4_compress_limitedOutput(p(in), p(out), clen(in), clen(out))) 51 | if outSize == 0 { 52 | err = fmt.Errorf("insufficient space for compression") 53 | } 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /lz4_test.go: -------------------------------------------------------------------------------- 1 | // Package lz4 implements compression using lz4.c. This is its test 2 | // suite. 3 | // 4 | // Copyright (c) 2013 CloudFlare, Inc. 5 | 6 | package lz4 7 | 8 | import ( 9 | "io/ioutil" 10 | "strings" 11 | "testing" 12 | "testing/quick" 13 | ) 14 | 15 | func TestCompressionRatio(t *testing.T) { 16 | input, err := ioutil.ReadFile("sample.txt") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | output := make([]byte, CompressBound(input)) 21 | outSize, err := Compress(input, output) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | 26 | if want := 4573; want != outSize { 27 | t.Fatalf("Compressed output length != expected: %d != %d", want, outSize) 28 | } 29 | } 30 | 31 | func TestCompression(t *testing.T) { 32 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 33 | output := make([]byte, CompressBound(input)) 34 | outSize, err := Compress(input, output) 35 | if err != nil { 36 | t.Fatalf("Compression failed: %v", err) 37 | } 38 | if outSize == 0 { 39 | t.Fatal("Output buffer is empty.") 40 | } 41 | output = output[:outSize] 42 | decompressed := make([]byte, len(input)) 43 | err = Uncompress(output, decompressed) 44 | if err != nil { 45 | t.Fatalf("Decompression failed: %v", err) 46 | } 47 | if string(decompressed) != string(input) { 48 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 49 | } 50 | } 51 | 52 | func TestEmptyCompression(t *testing.T) { 53 | input := []byte("") 54 | output := make([]byte, CompressBound(input)) 55 | outSize, err := Compress(input, output) 56 | if err != nil { 57 | t.Fatalf("Compression failed: %v", err) 58 | } 59 | if outSize == 0 { 60 | t.Fatal("Output buffer is empty.") 61 | } 62 | output = output[:outSize] 63 | decompressed := make([]byte, len(input)) 64 | err = Uncompress(output, decompressed) 65 | if err != nil { 66 | t.Fatalf("Decompression failed: %v", err) 67 | } 68 | if string(decompressed) != string(input) { 69 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 70 | } 71 | } 72 | 73 | func TestNoCompression(t *testing.T) { 74 | input := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ") 75 | output := make([]byte, CompressBound(input)) 76 | outSize, err := Compress(input, output) 77 | if err != nil { 78 | t.Fatalf("Compression failed: %v", err) 79 | } 80 | if outSize == 0 { 81 | t.Fatal("Output buffer is empty.") 82 | } 83 | output = output[:outSize] 84 | decompressed := make([]byte, len(input)) 85 | err = Uncompress(output, decompressed) 86 | if err != nil { 87 | t.Fatalf("Decompression failed: %v", err) 88 | } 89 | if string(decompressed) != string(input) { 90 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 91 | } 92 | } 93 | 94 | func TestCompressionError(t *testing.T) { 95 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 96 | output := make([]byte, 1) 97 | _, err := Compress(input, output) 98 | if err == nil { 99 | t.Fatalf("Compression should have failed but didn't") 100 | } 101 | 102 | output = make([]byte, 0) 103 | _, err = Compress(input, output) 104 | if err == nil { 105 | t.Fatalf("Compression should have failed but didn't") 106 | } 107 | } 108 | 109 | func TestDecompressionError(t *testing.T) { 110 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 111 | output := make([]byte, CompressBound(input)) 112 | outSize, err := Compress(input, output) 113 | if err != nil { 114 | t.Fatalf("Compression failed: %v", err) 115 | } 116 | if outSize == 0 { 117 | t.Fatal("Output buffer is empty.") 118 | } 119 | output = output[:outSize] 120 | decompressed := make([]byte, len(input)-1) 121 | err = Uncompress(output, decompressed) 122 | if err == nil { 123 | t.Fatalf("Decompression should have failed") 124 | } 125 | 126 | decompressed = make([]byte, 1) 127 | err = Uncompress(output, decompressed) 128 | if err == nil { 129 | t.Fatalf("Decompression should have failed") 130 | } 131 | 132 | decompressed = make([]byte, 0) 133 | err = Uncompress(output, decompressed) 134 | if err == nil { 135 | t.Fatalf("Decompression should have failed") 136 | } 137 | } 138 | 139 | func assert(t *testing.T, b bool) { 140 | if !b { 141 | t.Fatalf("assert failed") 142 | } 143 | } 144 | 145 | func TestCompressBound(t *testing.T) { 146 | input := make([]byte, 0) 147 | assert(t, CompressBound(input) == 16) 148 | 149 | input = make([]byte, 1) 150 | assert(t, CompressBound(input) == 17) 151 | 152 | input = make([]byte, 254) 153 | assert(t, CompressBound(input) == 270) 154 | 155 | input = make([]byte, 255) 156 | assert(t, CompressBound(input) == 272) 157 | 158 | input = make([]byte, 510) 159 | assert(t, CompressBound(input) == 528) 160 | } 161 | 162 | func TestFuzz(t *testing.T) { 163 | f := func(input []byte) bool { 164 | output := make([]byte, CompressBound(input)) 165 | outSize, err := Compress(input, output) 166 | if err != nil { 167 | t.Fatalf("Compression failed: %v", err) 168 | } 169 | if outSize == 0 { 170 | t.Fatal("Output buffer is empty.") 171 | } 172 | output = output[:outSize] 173 | decompressed := make([]byte, len(input)) 174 | err = Uncompress(output, decompressed) 175 | if err != nil { 176 | t.Fatalf("Decompression failed: %v", err) 177 | } 178 | if string(decompressed) != string(input) { 179 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 180 | } 181 | 182 | return true 183 | } 184 | 185 | conf := &quick.Config{MaxCount: 20000} 186 | if testing.Short() { 187 | conf.MaxCount = 1000 188 | } 189 | if err := quick.Check(f, conf); err != nil { 190 | t.Fatal(err) 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /lz4_hc_test.go: -------------------------------------------------------------------------------- 1 | // Package lz4 implements compression using lz4.c. This is its test 2 | // suite. 3 | // 4 | // Copyright (c) 2013 CloudFlare, Inc. 5 | 6 | package lz4 7 | 8 | import ( 9 | "io/ioutil" 10 | "strings" 11 | "testing" 12 | "testing/quick" 13 | ) 14 | 15 | func TestCompressionHCRatio(t *testing.T) { 16 | input, err := ioutil.ReadFile("sample.txt") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | output := make([]byte, CompressBound(input)) 21 | outSize, err := CompressHC(input, output) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | 26 | if want := 4317; want != outSize { 27 | t.Fatalf("HC Compressed output length != expected: %d != %d", want, outSize) 28 | } 29 | } 30 | 31 | func TestCompressionHCLevels(t *testing.T) { 32 | input, err := ioutil.ReadFile("sample.txt") 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | cases := []struct { 38 | Level int 39 | Outsize int 40 | }{ 41 | {0, 4317}, 42 | {1, 4415}, 43 | {2, 4359}, 44 | {3, 4339}, 45 | {4, 4321}, 46 | {5, 4317}, 47 | {6, 4317}, 48 | {7, 4317}, 49 | {8, 4317}, 50 | {9, 4317}, 51 | {10, 4317}, 52 | {11, 4317}, 53 | {12, 4317}, 54 | {13, 4317}, 55 | {14, 4317}, 56 | {15, 4317}, 57 | {16, 4317}, 58 | } 59 | 60 | for _, tt := range cases { 61 | output := make([]byte, CompressBound(input)) 62 | outSize, err := CompressHCLevel(input, output, tt.Level) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | 67 | if want := tt.Outsize; want != outSize { 68 | t.Errorf("HC level %d length != expected: %d != %d", 69 | tt.Level, want, outSize) 70 | } 71 | } 72 | } 73 | 74 | func TestCompressionHC(t *testing.T) { 75 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 76 | output := make([]byte, CompressBound(input)) 77 | outSize, err := CompressHC(input, output) 78 | if err != nil { 79 | t.Fatalf("Compression failed: %v", err) 80 | } 81 | if outSize == 0 { 82 | t.Fatal("Output buffer is empty.") 83 | } 84 | output = output[:outSize] 85 | decompressed := make([]byte, len(input)) 86 | err = Uncompress(output, decompressed) 87 | if err != nil { 88 | t.Fatalf("Decompression failed: %v", err) 89 | } 90 | if string(decompressed) != string(input) { 91 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 92 | } 93 | } 94 | 95 | func TestEmptyCompressionHC(t *testing.T) { 96 | input := []byte("") 97 | output := make([]byte, CompressBound(input)) 98 | 99 | outSize, err := CompressHC(input, output) 100 | if err != nil { 101 | t.Fatalf("Compression failed: %v", err) 102 | } 103 | if outSize == 0 { 104 | t.Fatal("Output buffer is empty.") 105 | } 106 | output = output[:outSize] 107 | decompressed := make([]byte, len(input)) 108 | err = Uncompress(output, decompressed) 109 | if err != nil { 110 | t.Fatalf("Decompression failed: %v", err) 111 | } 112 | if string(decompressed) != string(input) { 113 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 114 | } 115 | } 116 | 117 | func TestNoCompressionHC(t *testing.T) { 118 | input := []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ") 119 | output := make([]byte, CompressBound(input)) 120 | outSize, err := CompressHC(input, output) 121 | if err != nil { 122 | t.Fatalf("Compression failed: %v", err) 123 | } 124 | if outSize == 0 { 125 | t.Fatal("Output buffer is empty.") 126 | } 127 | output = output[:outSize] 128 | decompressed := make([]byte, len(input)) 129 | err = Uncompress(output, decompressed) 130 | if err != nil { 131 | t.Fatalf("Decompression failed: %v", err) 132 | } 133 | if string(decompressed) != string(input) { 134 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 135 | } 136 | } 137 | 138 | func TestCompressionErrorHC(t *testing.T) { 139 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 140 | output := make([]byte, 0) 141 | outSize, err := CompressHC(input, output) 142 | 143 | if outSize != 0 { 144 | t.Fatalf("%d", outSize) 145 | } 146 | 147 | if err == nil { 148 | t.Fatalf("Compression should have failed but didn't") 149 | } 150 | 151 | output = make([]byte, 1) 152 | _, err = CompressHC(input, output) 153 | if err == nil { 154 | t.Fatalf("Compression should have failed but didn't") 155 | } 156 | } 157 | 158 | func TestDecompressionErrorHC(t *testing.T) { 159 | input := []byte(strings.Repeat("Hello world, this is quite something", 10)) 160 | output := make([]byte, CompressBound(input)) 161 | outSize, err := CompressHC(input, output) 162 | if err != nil { 163 | t.Fatalf("Compression failed: %v", err) 164 | } 165 | if outSize == 0 { 166 | t.Fatal("Output buffer is empty.") 167 | } 168 | output = output[:outSize] 169 | decompressed := make([]byte, len(input)-1) 170 | err = Uncompress(output, decompressed) 171 | if err == nil { 172 | t.Fatalf("Decompression should have failed") 173 | } 174 | 175 | decompressed = make([]byte, 1) 176 | err = Uncompress(output, decompressed) 177 | if err == nil { 178 | t.Fatalf("Decompression should have failed") 179 | } 180 | 181 | decompressed = make([]byte, 0) 182 | err = Uncompress(output, decompressed) 183 | if err == nil { 184 | t.Fatalf("Decompression should have failed") 185 | } 186 | } 187 | 188 | func TestFuzzHC(t *testing.T) { 189 | f := func(input []byte) bool { 190 | output := make([]byte, CompressBound(input)) 191 | outSize, err := CompressHC(input, output) 192 | if err != nil { 193 | t.Fatalf("Compression failed: %v", err) 194 | } 195 | if outSize == 0 { 196 | t.Fatal("Output buffer is empty.") 197 | } 198 | output = output[:outSize] 199 | decompressed := make([]byte, len(input)) 200 | err = Uncompress(output, decompressed) 201 | if err != nil { 202 | t.Fatalf("Decompression failed: %v", err) 203 | } 204 | if string(decompressed) != string(input) { 205 | t.Fatalf("Decompressed output != input: %q != %q", decompressed, input) 206 | } 207 | 208 | return true 209 | } 210 | 211 | conf := &quick.Config{MaxCount: 20000} 212 | if testing.Short() { 213 | conf.MaxCount = 1000 214 | } 215 | if err := quick.Check(f, conf); err != nil { 216 | t.Fatal(err) 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /sample.txt: -------------------------------------------------------------------------------- 1 | CANTO I 2 | 3 | 4 | IN the midway of this our mortal life, 5 | I found me in a gloomy wood, astray 6 | Gone from the path direct: and e'en to tell 7 | It were no easy task, how savage wild 8 | That forest, how robust and rough its growth, 9 | Which to remember only, my dismay 10 | Renews, in bitterness not far from death. 11 | Yet to discourse of what there good befell, 12 | All else will I relate discover'd there. 13 | How first I enter'd it I scarce can say, 14 | Such sleepy dullness in that instant weigh'd 15 | My senses down, when the true path I left, 16 | But when a mountain's foot I reach'd, where clos'd 17 | The valley, that had pierc'd my heart with dread, 18 | I look'd aloft, and saw his shoulders broad 19 | Already vested with that planet's beam, 20 | Who leads all wanderers safe through every way. 21 | 22 | Then was a little respite to the fear, 23 | That in my heart's recesses deep had lain, 24 | All of that night, so pitifully pass'd: 25 | And as a man, with difficult short breath, 26 | Forespent with toiling, 'scap'd from sea to shore, 27 | Turns to the perilous wide waste, and stands 28 | At gaze; e'en so my spirit, that yet fail'd 29 | Struggling with terror, turn'd to view the straits, 30 | That none hath pass'd and liv'd. My weary frame 31 | After short pause recomforted, again 32 | I journey'd on over that lonely steep, 33 | 34 | The hinder foot still firmer. Scarce the ascent 35 | Began, when, lo! a panther, nimble, light, 36 | And cover'd with a speckled skin, appear'd, 37 | Nor, when it saw me, vanish'd, rather strove 38 | To check my onward going; that ofttimes 39 | With purpose to retrace my steps I turn'd. 40 | 41 | The hour was morning's prime, and on his way 42 | Aloft the sun ascended with those stars, 43 | That with him rose, when Love divine first mov'd 44 | Those its fair works: so that with joyous hope 45 | All things conspir'd to fill me, the gay skin 46 | Of that swift animal, the matin dawn 47 | And the sweet season. Soon that joy was chas'd, 48 | And by new dread succeeded, when in view 49 | A lion came, 'gainst me, as it appear'd, 50 | 51 | With his head held aloft and hunger-mad, 52 | That e'en the air was fear-struck. A she-wolf 53 | Was at his heels, who in her leanness seem'd 54 | Full of all wants, and many a land hath made 55 | Disconsolate ere now. She with such fear 56 | O'erwhelmed me, at the sight of her appall'd, 57 | That of the height all hope I lost. As one, 58 | Who with his gain elated, sees the time 59 | When all unwares is gone, he inwardly 60 | Mourns with heart-griping anguish; such was I, 61 | Haunted by that fell beast, never at peace, 62 | Who coming o'er against me, by degrees 63 | Impell'd me where the sun in silence rests. 64 | 65 | While to the lower space with backward step 66 | I fell, my ken discern'd the form one of one, 67 | Whose voice seem'd faint through long disuse of speech. 68 | When him in that great desert I espied, 69 | "Have mercy on me!" cried I out aloud, 70 | "Spirit! or living man! what e'er thou be!" 71 | 72 | He answer'd: "Now not man, man once I was, 73 | And born of Lombard parents, Mantuana both 74 | By country, when the power of Julius yet 75 | Was scarcely firm. At Rome my life was past 76 | Beneath the mild Augustus, in the time 77 | Of fabled deities and false. A bard 78 | Was I, and made Anchises' upright son 79 | The subject of my song, who came from Troy, 80 | When the flames prey'd on Ilium's haughty towers. 81 | But thou, say wherefore to such perils past 82 | Return'st thou? wherefore not this pleasant mount 83 | Ascendest, cause and source of all delight?" 84 | "And art thou then that Virgil, that well-spring, 85 | From which such copious floods of eloquence 86 | Have issued?" I with front abash'd replied. 87 | "Glory and light of all the tuneful train! 88 | May it avail me that I long with zeal 89 | Have sought thy volume, and with love immense 90 | Have conn'd it o'er. My master thou and guide! 91 | Thou he from whom alone I have deriv'd 92 | That style, which for its beauty into fame 93 | Exalts me. See the beast, from whom I fled. 94 | O save me from her, thou illustrious sage!" 95 | 96 | "For every vein and pulse throughout my frame 97 | She hath made tremble." He, soon as he saw 98 | That I was weeping, answer'd, "Thou must needs 99 | Another way pursue, if thou wouldst 'scape 100 | From out that savage wilderness. This beast, 101 | At whom thou criest, her way will suffer none 102 | To pass, and no less hindrance makes than death: 103 | So bad and so accursed in her kind, 104 | That never sated is her ravenous will, 105 | Still after food more craving than before. 106 | To many an animal in wedlock vile 107 | She fastens, and shall yet to many more, 108 | Until that greyhound come, who shall destroy 109 | Her with sharp pain. He will not life support 110 | By earth nor its base metals, but by love, 111 | Wisdom, and virtue, and his land shall be 112 | The land 'twixt either Feltro. In his might 113 | Shall safety to Italia's plains arise, 114 | For whose fair realm, Camilla, virgin pure, 115 | Nisus, Euryalus, and Turnus fell. 116 | He with incessant chase through every town 117 | Shall worry, until he to hell at length 118 | Restore her, thence by envy first let loose. 119 | I for thy profit pond'ring now devise, 120 | That thou mayst follow me, and I thy guide 121 | Will lead thee hence through an eternal space, 122 | Where thou shalt hear despairing shrieks, and see 123 | Spirits of old tormented, who invoke 124 | A second death; and those next view, who dwell 125 | Content in fire, for that they hope to come, 126 | Whene'er the time may be, among the blest, 127 | Into whose regions if thou then desire 128 | T' ascend, a spirit worthier then I 129 | Must lead thee, in whose charge, when I depart, 130 | Thou shalt be left: for that Almighty King, 131 | Who reigns above, a rebel to his law, 132 | Adjudges me, and therefore hath decreed, 133 | That to his city none through me should come. 134 | He in all parts hath sway; there rules, there holds 135 | His citadel and throne. O happy those, 136 | Whom there he chooses!" I to him in few: 137 | "Bard! by that God, whom thou didst not adore, 138 | I do beseech thee (that this ill and worse 139 | I may escape) to lead me, where thou saidst, 140 | That I Saint Peter's gate may view, and those 141 | Who as thou tell'st, are in such dismal plight." 142 | 143 | Onward he mov'd, I close his steps pursu'd. 144 | -------------------------------------------------------------------------------- /src/lz4hc.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 HC - High Compression Mode of LZ4 3 | Header File 4 | Copyright (C) 2011-2014, Yann Collet. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html 32 | - LZ4 source repository : http://code.google.com/p/lz4/ 33 | */ 34 | #pragma once 35 | 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | 42 | int LZ4_compressHC (const char* source, char* dest, int inputSize); 43 | /* 44 | LZ4_compressHC : 45 | return : the number of bytes in compressed buffer dest 46 | or 0 if compression fails. 47 | note : destination buffer must be already allocated. 48 | To avoid any problem, size it to handle worst cases situations (input data not compressible) 49 | Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h") 50 | */ 51 | 52 | int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); 53 | /* 54 | LZ4_compress_limitedOutput() : 55 | Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. 56 | If it cannot achieve it, compression will stop, and result of the function will be zero. 57 | This function never writes outside of provided output buffer. 58 | 59 | inputSize : Max supported value is 1 GB 60 | maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated) 61 | return : the number of output bytes written in buffer 'dest' 62 | or 0 if compression fails. 63 | */ 64 | 65 | 66 | int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); 67 | int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 68 | /* 69 | Same functions as above, but with programmable 'compressionLevel'. 70 | Recommended values are between 4 and 9, although any value between 0 and 16 will work. 71 | 'compressionLevel'==0 means use default 'compressionLevel' value. 72 | Values above 16 behave the same as 16. 73 | Equivalent variants exist for all other compression functions below. 74 | */ 75 | 76 | /* Note : 77 | Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) 78 | */ 79 | 80 | 81 | /************************************** 82 | Using an external allocation 83 | **************************************/ 84 | int LZ4_sizeofStateHC(void); 85 | int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); 86 | int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 87 | 88 | int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); 89 | int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 90 | 91 | /* 92 | These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods. 93 | To know how much memory must be allocated for the compression tables, use : 94 | int LZ4_sizeofStateHC(); 95 | 96 | Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0). 97 | 98 | The allocated memory can be provided to the compression functions using 'void* state' parameter. 99 | LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions. 100 | They just use the externally allocated memory for state instead of allocating their own (on stack, or on heap). 101 | */ 102 | 103 | 104 | 105 | /************************************** 106 | Experimental Streaming Functions 107 | **************************************/ 108 | #define LZ4_STREAMHCSIZE_U64 32774 109 | #define LZ4_STREAMHCSIZE (LZ4_STREAMHCSIZE_U64 * sizeof(unsigned long long)) 110 | typedef struct { unsigned long long table[LZ4_STREAMHCSIZE_U64]; } LZ4_streamHC_t; 111 | /* 112 | LZ4_streamHC_t 113 | This structure allows static allocation of LZ4 HC streaming state. 114 | State must then be initialized using LZ4_resetStreamHC() before first use. 115 | 116 | Static allocation should only be used with statically linked library. 117 | If you want to use LZ4 as a DLL, please use construction functions below, which are more future-proof. 118 | */ 119 | 120 | 121 | LZ4_streamHC_t* LZ4_createStreamHC(void); 122 | int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr); 123 | /* 124 | These functions create and release memory for LZ4 HC streaming state. 125 | Newly created states are already initialized. 126 | Existing state space can be re-used anytime using LZ4_resetStreamHC(). 127 | If you use LZ4 as a DLL, please use these functions instead of direct struct allocation, 128 | to avoid size mismatch between different versions. 129 | */ 130 | 131 | void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); 132 | int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize); 133 | 134 | int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); 135 | int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 136 | 137 | int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int maxDictSize); 138 | 139 | /* 140 | These functions compress data in successive blocks of any size, using previous blocks as dictionary. 141 | One key assumption is that each previous block will remain read-accessible while compressing next block. 142 | 143 | Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). 144 | A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). 145 | 146 | Then, use LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue() to compress each successive block. 147 | They work like usual LZ4_compressHC() or LZ4_compressHC_limitedOutput(), but use previous memory blocks to improve compression. 148 | Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. 149 | 150 | If, for any reason, previous data block can't be preserved in memory during next compression block, 151 | you must save it to a safer memory space, 152 | using LZ4_saveDictHC(). 153 | */ 154 | 155 | 156 | 157 | /************************************** 158 | * Deprecated Streaming Functions 159 | * ************************************/ 160 | /* Note : these streaming functions follows the older model, and should no longer be used */ 161 | void* LZ4_createHC (const char* inputBuffer); 162 | char* LZ4_slideInputBufferHC (void* LZ4HC_Data); 163 | int LZ4_freeHC (void* LZ4HC_Data); 164 | 165 | int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); 166 | int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 167 | 168 | int LZ4_sizeofStreamStateHC(void); 169 | int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); 170 | 171 | 172 | #if defined (__cplusplus) 173 | } 174 | #endif 175 | -------------------------------------------------------------------------------- /src/lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Header File 4 | Copyright (C) 2011-2014, Yann Collet. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : http://code.google.com/p/lz4/ 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | #pragma once 35 | 36 | #if defined (__cplusplus) 37 | extern "C" { 38 | #endif 39 | 40 | /* 41 | * lz4.h provides raw compression format functions, for optimal performance and integration into programs. 42 | * If you need to generate data using an inter-operable format (respecting the framing specification), 43 | * please use lz4frame.h instead. 44 | */ 45 | 46 | /************************************** 47 | Version 48 | **************************************/ 49 | #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 50 | #define LZ4_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */ 51 | #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ 52 | #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) 53 | int LZ4_versionNumber (void); 54 | 55 | /************************************** 56 | Tuning parameter 57 | **************************************/ 58 | /* 59 | * LZ4_MEMORY_USAGE : 60 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 61 | * Increasing memory usage improves compression ratio 62 | * Reduced memory usage can improve speed, due to cache effect 63 | * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache 64 | */ 65 | #define LZ4_MEMORY_USAGE 14 66 | 67 | 68 | /************************************** 69 | Simple Functions 70 | **************************************/ 71 | 72 | int LZ4_compress (const char* source, char* dest, int sourceSize); 73 | int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); 74 | 75 | /* 76 | LZ4_compress() : 77 | Compresses 'sourceSize' bytes from 'source' into 'dest'. 78 | Destination buffer must be already allocated, 79 | and must be sized to handle worst cases situations (input data not compressible) 80 | Worst case size evaluation is provided by function LZ4_compressBound() 81 | inputSize : Max supported value is LZ4_MAX_INPUT_SIZE 82 | return : the number of bytes written in buffer dest 83 | or 0 if the compression fails 84 | 85 | LZ4_decompress_safe() : 86 | compressedSize : is obviously the source size 87 | maxDecompressedSize : is the size of the destination buffer, which must be already allocated. 88 | return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize) 89 | If the destination buffer is not large enough, decoding will stop and output an error code (<0). 90 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 91 | This function is protected against buffer overflow exploits, 92 | and never writes outside of output buffer, nor reads outside of input buffer. 93 | It is also protected against malicious data packets. 94 | */ 95 | 96 | 97 | /************************************** 98 | Advanced Functions 99 | **************************************/ 100 | #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ 101 | #define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 102 | 103 | /* 104 | LZ4_compressBound() : 105 | Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) 106 | This function is primarily useful for memory allocation purposes (output buffer size). 107 | Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). 108 | 109 | isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE 110 | return : maximum output size in a "worst case" scenario 111 | or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) 112 | */ 113 | int LZ4_compressBound(int isize); 114 | 115 | 116 | /* 117 | LZ4_compress_limitedOutput() : 118 | Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. 119 | If it cannot achieve it, compression will stop, and result of the function will be zero. 120 | This saves time and memory on detecting non-compressible (or barely compressible) data. 121 | This function never writes outside of provided output buffer. 122 | 123 | sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE 124 | maxOutputSize : is the size of the destination buffer (which must be already allocated) 125 | return : the number of bytes written in buffer 'dest' 126 | or 0 if compression fails 127 | */ 128 | int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 129 | 130 | 131 | /* 132 | LZ4_compress_withState() : 133 | Same compression functions, but using an externally allocated memory space to store compression state. 134 | Use LZ4_sizeofState() to know how much memory must be allocated, 135 | and then, provide it as 'void* state' to compression functions. 136 | */ 137 | int LZ4_sizeofState(void); 138 | int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 139 | int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 140 | 141 | 142 | /* 143 | LZ4_decompress_fast() : 144 | originalSize : is the original and therefore uncompressed size 145 | return : the number of bytes read from the source buffer (in other words, the compressed size) 146 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 147 | Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. 148 | note : This function fully respect memory boundaries for properly formed compressed data. 149 | It is a bit faster than LZ4_decompress_safe(). 150 | However, it does not provide any protection against intentionally modified data stream (malicious input). 151 | Use this function in trusted environment only (data to decode comes from a trusted source). 152 | */ 153 | int LZ4_decompress_fast (const char* source, char* dest, int originalSize); 154 | 155 | 156 | /* 157 | LZ4_decompress_safe_partial() : 158 | This function decompress a compressed block of size 'compressedSize' at position 'source' 159 | into destination buffer 'dest' of size 'maxDecompressedSize'. 160 | The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, 161 | reducing decompression time. 162 | return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) 163 | Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. 164 | Always control how many bytes were decoded. 165 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 166 | This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets 167 | */ 168 | int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); 169 | 170 | 171 | /*********************************************** 172 | Streaming Compression Functions 173 | ***********************************************/ 174 | 175 | #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) 176 | #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) 177 | /* 178 | * LZ4_stream_t 179 | * information structure to track an LZ4 stream. 180 | * important : init this structure content before first use ! 181 | * note : only allocated directly the structure if you are statically linking LZ4 182 | * If you are using liblz4 as a DLL, please use below construction methods instead. 183 | */ 184 | typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; 185 | 186 | /* 187 | * LZ4_resetStream 188 | * Use this function to init an allocated LZ4_stream_t structure 189 | */ 190 | void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr); 191 | 192 | /* 193 | * LZ4_createStream will allocate and initialize an LZ4_stream_t structure 194 | * LZ4_freeStream releases its memory. 195 | * In the context of a DLL (liblz4), please use these methods rather than the static struct. 196 | * They are more future proof, in case of a change of LZ4_stream_t size. 197 | */ 198 | LZ4_stream_t* LZ4_createStream(void); 199 | int LZ4_freeStream (LZ4_stream_t* LZ4_streamPtr); 200 | 201 | /* 202 | * LZ4_loadDict 203 | * Use this function to load a static dictionary into LZ4_stream. 204 | * Any previous data will be forgotten, only 'dictionary' will remain in memory. 205 | * Loading a size of 0 is allowed. 206 | * Return : dictionary size, in bytes (necessarily <= 64 KB) 207 | */ 208 | int LZ4_loadDict (LZ4_stream_t* LZ4_streamPtr, const char* dictionary, int dictSize); 209 | 210 | /* 211 | * LZ4_compress_continue 212 | * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio 213 | * Previous data blocks are assumed to still be present at their previous location. 214 | */ 215 | int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 216 | 217 | /* 218 | * LZ4_compress_limitedOutput_continue 219 | * Same as before, but also specify a maximum target compressed size (maxOutputSize) 220 | * If objective cannot be met, compression exits, and returns a zero. 221 | */ 222 | int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 223 | 224 | /* 225 | * LZ4_saveDict 226 | * If previously compressed data block is not guaranteed to remain available at its memory location 227 | * save it into a safer place (char* safeBuffer) 228 | * Note : you don't need to call LZ4_loadDict() afterwards, 229 | * dictionary is immediately usable, you can therefore call again LZ4_compress_continue() 230 | * Return : dictionary size in bytes, or 0 if error 231 | * Note : any dictSize > 64 KB will be interpreted as 64KB. 232 | */ 233 | int LZ4_saveDict (LZ4_stream_t* LZ4_streamPtr, char* safeBuffer, int dictSize); 234 | 235 | 236 | /************************************************ 237 | Streaming Decompression Functions 238 | ************************************************/ 239 | 240 | #define LZ4_STREAMDECODESIZE_U64 4 241 | #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) 242 | typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; 243 | /* 244 | * LZ4_streamDecode_t 245 | * information structure to track an LZ4 stream. 246 | * init this structure content using LZ4_setStreamDecode or memset() before first use ! 247 | * 248 | * In the context of a DLL (liblz4) please prefer usage of construction methods below. 249 | * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. 250 | * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure 251 | * LZ4_freeStreamDecode releases its memory. 252 | */ 253 | LZ4_streamDecode_t* LZ4_createStreamDecode(void); 254 | int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); 255 | 256 | /* 257 | * LZ4_setStreamDecode 258 | * Use this function to instruct where to find the dictionary. 259 | * Setting a size of 0 is allowed (same effect as reset). 260 | * Return : 1 if OK, 0 if error 261 | */ 262 | int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); 263 | 264 | /* 265 | *_continue() : 266 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 267 | Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) 268 | If this condition is not possible, save the relevant part of decoded data into a safe buffer, 269 | and indicate where is its new address using LZ4_setStreamDecode() 270 | */ 271 | int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); 272 | int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); 273 | 274 | 275 | /* 276 | Advanced decoding functions : 277 | *_usingDict() : 278 | These decoding functions work the same as 279 | a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue() 280 | They are stand-alone and don't use nor update an LZ4_streamDecode_t structure. 281 | */ 282 | int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); 283 | int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); 284 | 285 | 286 | 287 | /************************************** 288 | Obsolete Functions 289 | **************************************/ 290 | /* 291 | Obsolete decompression functions 292 | These function names are deprecated and should no longer be used. 293 | They are only provided here for compatibility with older user programs. 294 | - LZ4_uncompress is the same as LZ4_decompress_fast 295 | - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe 296 | These function prototypes are now disabled; uncomment them if you really need them. 297 | It is highly recommended to stop using these functions and migrate to newer ones */ 298 | /* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ 299 | /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ 300 | 301 | 302 | /* Obsolete streaming functions; use new streaming interface whenever possible */ 303 | void* LZ4_create (const char* inputBuffer); 304 | int LZ4_sizeofStreamState(void); 305 | int LZ4_resetStreamState(void* state, const char* inputBuffer); 306 | char* LZ4_slideInputBuffer (void* state); 307 | 308 | /* Obsolete streaming decoding functions */ 309 | int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize); 310 | int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize); 311 | 312 | 313 | #if defined (__cplusplus) 314 | } 315 | #endif 316 | -------------------------------------------------------------------------------- /src/lz4hc.c: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 HC - High Compression Mode of LZ4 3 | Copyright (C) 2011-2014, Yann Collet. 4 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | You can contact the author at : 30 | - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html 31 | - LZ4 source repository : http://code.google.com/p/lz4/ 32 | */ 33 | 34 | 35 | 36 | /************************************** 37 | Tuning Parameter 38 | **************************************/ 39 | static const int LZ4HC_compressionLevel_default = 8; 40 | 41 | 42 | /************************************** 43 | Includes 44 | **************************************/ 45 | #include "lz4hc.h" 46 | 47 | 48 | /************************************** 49 | Local Compiler Options 50 | **************************************/ 51 | #if defined(__GNUC__) 52 | # pragma GCC diagnostic ignored "-Wunused-function" 53 | #endif 54 | 55 | #if defined (__clang__) 56 | # pragma clang diagnostic ignored "-Wunused-function" 57 | #endif 58 | 59 | 60 | /************************************** 61 | Common LZ4 definition 62 | **************************************/ 63 | #define LZ4_COMMONDEFS_ONLY 64 | #include "lz4.c" 65 | 66 | 67 | /************************************** 68 | Local Constants 69 | **************************************/ 70 | #define DICTIONARY_LOGSIZE 16 71 | #define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) 105 | #define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] 106 | #define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) 107 | 108 | static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } 109 | 110 | 111 | 112 | /************************************** 113 | HC Compression 114 | **************************************/ 115 | static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) 116 | { 117 | MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); 118 | MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); 119 | hc4->nextToUpdate = 64 KB; 120 | hc4->base = start - 64 KB; 121 | hc4->inputBuffer = start; 122 | hc4->end = start; 123 | hc4->dictBase = start - 64 KB; 124 | hc4->dictLimit = 64 KB; 125 | hc4->lowLimit = 64 KB; 126 | } 127 | 128 | 129 | /* Update chains up to ip (excluded) */ 130 | FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) 131 | { 132 | U16* chainTable = hc4->chainTable; 133 | U32* HashTable = hc4->hashTable; 134 | const BYTE* const base = hc4->base; 135 | const U32 target = (U32)(ip - base); 136 | U32 idx = hc4->nextToUpdate; 137 | 138 | while(idx < target) 139 | { 140 | U32 h = LZ4HC_hashPtr(base+idx); 141 | size_t delta = idx - HashTable[h]; 142 | if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; 143 | chainTable[idx & 0xFFFF] = (U16)delta; 144 | HashTable[h] = idx; 145 | idx++; 146 | } 147 | 148 | hc4->nextToUpdate = target; 149 | } 150 | 151 | 152 | FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */ 153 | const BYTE* ip, const BYTE* const iLimit, 154 | const BYTE** matchpos, 155 | const int maxNbAttempts) 156 | { 157 | U16* const chainTable = hc4->chainTable; 158 | U32* const HashTable = hc4->hashTable; 159 | const BYTE* const base = hc4->base; 160 | const BYTE* const dictBase = hc4->dictBase; 161 | const U32 dictLimit = hc4->dictLimit; 162 | const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); 163 | U32 matchIndex; 164 | const BYTE* match; 165 | int nbAttempts=maxNbAttempts; 166 | size_t ml=0; 167 | 168 | /* HC4 match finder */ 169 | LZ4HC_Insert(hc4, ip); 170 | matchIndex = HashTable[LZ4HC_hashPtr(ip)]; 171 | 172 | while ((matchIndex>=lowLimit) && (nbAttempts)) 173 | { 174 | nbAttempts--; 175 | if (matchIndex >= dictLimit) 176 | { 177 | match = base + matchIndex; 178 | if (*(match+ml) == *(ip+ml) 179 | && (LZ4_read32(match) == LZ4_read32(ip))) 180 | { 181 | size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; 182 | if (mlt > ml) { ml = mlt; *matchpos = match; } 183 | } 184 | } 185 | else 186 | { 187 | match = dictBase + matchIndex; 188 | if (LZ4_read32(match) == LZ4_read32(ip)) 189 | { 190 | size_t mlt; 191 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 192 | if (vLimit > iLimit) vLimit = iLimit; 193 | mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; 194 | if ((ip+mlt == vLimit) && (vLimit < iLimit)) 195 | mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); 196 | if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ 197 | } 198 | } 199 | matchIndex -= chainTable[matchIndex & 0xFFFF]; 200 | } 201 | 202 | return (int)ml; 203 | } 204 | 205 | 206 | FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( 207 | LZ4HC_Data_Structure* hc4, 208 | const BYTE* ip, 209 | const BYTE* iLowLimit, 210 | const BYTE* iHighLimit, 211 | int longest, 212 | const BYTE** matchpos, 213 | const BYTE** startpos, 214 | const int maxNbAttempts) 215 | { 216 | U16* const chainTable = hc4->chainTable; 217 | U32* const HashTable = hc4->hashTable; 218 | const BYTE* const base = hc4->base; 219 | const U32 dictLimit = hc4->dictLimit; 220 | const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); 221 | const BYTE* const dictBase = hc4->dictBase; 222 | const BYTE* match; 223 | U32 matchIndex; 224 | int nbAttempts = maxNbAttempts; 225 | int delta = (int)(ip-iLowLimit); 226 | 227 | 228 | /* First Match */ 229 | LZ4HC_Insert(hc4, ip); 230 | matchIndex = HashTable[LZ4HC_hashPtr(ip)]; 231 | 232 | while ((matchIndex>=lowLimit) && (nbAttempts)) 233 | { 234 | nbAttempts--; 235 | if (matchIndex >= dictLimit) 236 | { 237 | match = base + matchIndex; 238 | if (*(iLowLimit + longest) == *(match - delta + longest)) 239 | if (LZ4_read32(match) == LZ4_read32(ip)) 240 | { 241 | const BYTE* startt = ip; 242 | const BYTE* tmpMatch = match; 243 | const BYTE* const matchEnd = ip + MINMATCH + LZ4_count(ip+MINMATCH, match+MINMATCH, iHighLimit); 244 | 245 | while ((startt>iLowLimit) && (tmpMatch > iLowLimit) && (startt[-1] == tmpMatch[-1])) {startt--; tmpMatch--;} 246 | 247 | if ((matchEnd-startt) > longest) 248 | { 249 | longest = (int)(matchEnd-startt); 250 | *matchpos = tmpMatch; 251 | *startpos = startt; 252 | } 253 | } 254 | } 255 | else 256 | { 257 | match = dictBase + matchIndex; 258 | if (LZ4_read32(match) == LZ4_read32(ip)) 259 | { 260 | size_t mlt; 261 | int back=0; 262 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 263 | if (vLimit > iHighLimit) vLimit = iHighLimit; 264 | mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; 265 | if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) 266 | mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); 267 | while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; 268 | mlt -= back; 269 | if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } 270 | } 271 | } 272 | matchIndex -= chainTable[matchIndex & 0xFFFF]; 273 | } 274 | 275 | return longest; 276 | } 277 | 278 | 279 | typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; 280 | 281 | #define LZ4HC_DEBUG 0 282 | #if LZ4HC_DEBUG 283 | static unsigned debug = 0; 284 | #endif 285 | 286 | FORCE_INLINE int LZ4HC_encodeSequence ( 287 | const BYTE** ip, 288 | BYTE** op, 289 | const BYTE** anchor, 290 | int matchLength, 291 | const BYTE* const match, 292 | limitedOutput_directive limitedOutputBuffer, 293 | BYTE* oend) 294 | { 295 | int length; 296 | BYTE* token; 297 | 298 | #if LZ4HC_DEBUG 299 | if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); 300 | #endif 301 | 302 | /* Encode Literal length */ 303 | length = (int)(*ip - *anchor); 304 | token = (*op)++; 305 | if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ 306 | if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } 307 | else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ 319 | if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } 320 | else *token += (BYTE)(length); 321 | 322 | /* Prepare next loop */ 323 | *ip += matchLength; 324 | *anchor = *ip; 325 | 326 | return 0; 327 | } 328 | 329 | 330 | static int LZ4HC_compress_generic ( 331 | void* ctxvoid, 332 | const char* source, 333 | char* dest, 334 | int inputSize, 335 | int maxOutputSize, 336 | int compressionLevel, 337 | limitedOutput_directive limit 338 | ) 339 | { 340 | LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; 341 | const BYTE* ip = (const BYTE*) source; 342 | const BYTE* anchor = ip; 343 | const BYTE* const iend = ip + inputSize; 344 | const BYTE* const mflimit = iend - MFLIMIT; 345 | const BYTE* const matchlimit = (iend - LASTLITERALS); 346 | 347 | BYTE* op = (BYTE*) dest; 348 | BYTE* const oend = op + maxOutputSize; 349 | 350 | unsigned maxNbAttempts; 351 | int ml, ml2, ml3, ml0; 352 | const BYTE* ref=NULL; 353 | const BYTE* start2=NULL; 354 | const BYTE* ref2=NULL; 355 | const BYTE* start3=NULL; 356 | const BYTE* ref3=NULL; 357 | const BYTE* start0; 358 | const BYTE* ref0; 359 | 360 | 361 | /* init */ 362 | if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; 363 | if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default; 364 | maxNbAttempts = 1 << (compressionLevel-1); 365 | ctx->end += inputSize; 366 | 367 | ip++; 368 | 369 | /* Main Loop */ 370 | while (ip < mflimit) 371 | { 372 | ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); 373 | if (!ml) { ip++; continue; } 374 | 375 | /* saved, in case we would skip too much */ 376 | start0 = ip; 377 | ref0 = ref; 378 | ml0 = ml; 379 | 380 | _Search2: 381 | if (ip+ml < mflimit) 382 | ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); 383 | else ml2 = ml; 384 | 385 | if (ml2 == ml) /* No better match */ 386 | { 387 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 388 | continue; 389 | } 390 | 391 | if (start0 < ip) 392 | { 393 | if (start2 < ip + ml0) /* empirical */ 394 | { 395 | ip = start0; 396 | ref = ref0; 397 | ml = ml0; 398 | } 399 | } 400 | 401 | /* Here, start0==ip */ 402 | if ((start2 - ip) < 3) /* First Match too small : removed */ 403 | { 404 | ml = ml2; 405 | ip = start2; 406 | ref =ref2; 407 | goto _Search2; 408 | } 409 | 410 | _Search3: 411 | /* 412 | * Currently we have : 413 | * ml2 > ml1, and 414 | * ip1+3 <= ip2 (usually < ip1+ml1) 415 | */ 416 | if ((start2 - ip) < OPTIMAL_ML) 417 | { 418 | int correction; 419 | int new_ml = ml; 420 | if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; 421 | if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; 422 | correction = new_ml - (int)(start2 - ip); 423 | if (correction > 0) 424 | { 425 | start2 += correction; 426 | ref2 += correction; 427 | ml2 -= correction; 428 | } 429 | } 430 | /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ 431 | 432 | if (start2 + ml2 < mflimit) 433 | ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); 434 | else ml3 = ml2; 435 | 436 | if (ml3 == ml2) /* No better match : 2 sequences to encode */ 437 | { 438 | /* ip & ref are known; Now for ml */ 439 | if (start2 < ip+ml) ml = (int)(start2 - ip); 440 | /* Now, encode 2 sequences */ 441 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 442 | ip = start2; 443 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; 444 | continue; 445 | } 446 | 447 | if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ 448 | { 449 | if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ 450 | { 451 | if (start2 < ip+ml) 452 | { 453 | int correction = (int)(ip+ml - start2); 454 | start2 += correction; 455 | ref2 += correction; 456 | ml2 -= correction; 457 | if (ml2 < MINMATCH) 458 | { 459 | start2 = start3; 460 | ref2 = ref3; 461 | ml2 = ml3; 462 | } 463 | } 464 | 465 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 466 | ip = start3; 467 | ref = ref3; 468 | ml = ml3; 469 | 470 | start0 = start2; 471 | ref0 = ref2; 472 | ml0 = ml2; 473 | goto _Search2; 474 | } 475 | 476 | start2 = start3; 477 | ref2 = ref3; 478 | ml2 = ml3; 479 | goto _Search3; 480 | } 481 | 482 | /* 483 | * OK, now we have 3 ascending matches; let's write at least the first one 484 | * ip & ref are known; Now for ml 485 | */ 486 | if (start2 < ip+ml) 487 | { 488 | if ((start2 - ip) < (int)ML_MASK) 489 | { 490 | int correction; 491 | if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; 492 | if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; 493 | correction = ml - (int)(start2 - ip); 494 | if (correction > 0) 495 | { 496 | start2 += correction; 497 | ref2 += correction; 498 | ml2 -= correction; 499 | } 500 | } 501 | else 502 | { 503 | ml = (int)(start2 - ip); 504 | } 505 | } 506 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 507 | 508 | ip = start2; 509 | ref = ref2; 510 | ml = ml2; 511 | 512 | start2 = start3; 513 | ref2 = ref3; 514 | ml2 = ml3; 515 | 516 | goto _Search3; 517 | } 518 | 519 | /* Encode Last Literals */ 520 | { 521 | int lastRun = (int)(iend - anchor); 522 | if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ 523 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 524 | else *op++ = (BYTE)(lastRun<base = NULL; 598 | ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; 599 | } 600 | 601 | int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) 602 | { 603 | LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; 604 | if (dictSize > 64 KB) 605 | { 606 | dictionary += dictSize - 64 KB; 607 | dictSize = 64 KB; 608 | } 609 | LZ4HC_init (ctxPtr, (const BYTE*)dictionary); 610 | if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); 611 | ctxPtr->end = (const BYTE*)dictionary + dictSize; 612 | return dictSize; 613 | } 614 | 615 | 616 | /* compression */ 617 | 618 | static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) 619 | { 620 | if (ctxPtr->end >= ctxPtr->base + 4) 621 | LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ 622 | /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ 623 | ctxPtr->lowLimit = ctxPtr->dictLimit; 624 | ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); 625 | ctxPtr->dictBase = ctxPtr->base; 626 | ctxPtr->base = newBlock - ctxPtr->dictLimit; 627 | ctxPtr->end = newBlock; 628 | ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ 629 | } 630 | 631 | static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, 632 | const char* source, char* dest, 633 | int inputSize, int maxOutputSize, limitedOutput_directive limit) 634 | { 635 | /* auto-init if forgotten */ 636 | if (ctxPtr->base == NULL) 637 | LZ4HC_init (ctxPtr, (const BYTE*) source); 638 | 639 | /* Check overflow */ 640 | if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) 641 | { 642 | size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; 643 | if (dictSize > 64 KB) dictSize = 64 KB; 644 | 645 | LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); 646 | } 647 | 648 | /* Check if blocks follow each other */ 649 | if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); 650 | 651 | /* Check overlapping input/dictionary space */ 652 | { 653 | const BYTE* sourceEnd = (const BYTE*) source + inputSize; 654 | const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; 655 | const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; 656 | if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd)) 657 | { 658 | if (sourceEnd > dictEnd) sourceEnd = dictEnd; 659 | ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); 660 | if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; 661 | } 662 | } 663 | 664 | return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); 665 | } 666 | 667 | int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize) 668 | { 669 | return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit); 670 | } 671 | 672 | int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) 673 | { 674 | return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); 675 | } 676 | 677 | 678 | /* dictionary saving */ 679 | 680 | int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) 681 | { 682 | LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; 683 | int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); 684 | if (dictSize > 64 KB) dictSize = 64 KB; 685 | if (dictSize < 4) dictSize = 0; 686 | if (dictSize > prefixSize) dictSize = prefixSize; 687 | memcpy(safeBuffer, streamPtr->end - dictSize, dictSize); 688 | { 689 | U32 endIndex = (U32)(streamPtr->end - streamPtr->base); 690 | streamPtr->end = (const BYTE*)safeBuffer + dictSize; 691 | streamPtr->base = streamPtr->end - endIndex; 692 | streamPtr->dictLimit = endIndex - dictSize; 693 | streamPtr->lowLimit = endIndex - dictSize; 694 | if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; 695 | } 696 | return dictSize; 697 | } 698 | 699 | 700 | /*********************************** 701 | * Deprecated Functions 702 | ***********************************/ 703 | int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } 704 | 705 | int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) 706 | { 707 | if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ 708 | LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); 709 | return 0; 710 | } 711 | 712 | void* LZ4_createHC (const char* inputBuffer) 713 | { 714 | void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); 715 | LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); 716 | return hc4; 717 | } 718 | 719 | int LZ4_freeHC (void* LZ4HC_Data) 720 | { 721 | FREEMEM(LZ4HC_Data); 722 | return (0); 723 | } 724 | 725 | /* 726 | int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize) 727 | { 728 | return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit); 729 | } 730 | int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize) 731 | { 732 | return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput); 733 | } 734 | */ 735 | 736 | int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) 737 | { 738 | return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); 739 | } 740 | 741 | int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) 742 | { 743 | return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); 744 | } 745 | 746 | char* LZ4_slideInputBufferHC(void* LZ4HC_Data) 747 | { 748 | LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; 749 | int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); 750 | return (char*)(hc4->inputBuffer + dictSize); 751 | } 752 | -------------------------------------------------------------------------------- /src/lz4.c: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Copyright (C) 2011-2015, Yann Collet. 4 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | You can contact the author at : 30 | - LZ4 source repository : http://code.google.com/p/lz4 31 | - LZ4 source mirror : https://github.com/Cyan4973/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | 35 | 36 | /************************************** 37 | Tuning parameters 38 | **************************************/ 39 | /* 40 | * HEAPMODE : 41 | * Select how default compression functions will allocate memory for their hash table, 42 | * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). 43 | */ 44 | #define HEAPMODE 0 45 | 46 | /* 47 | * CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS : 48 | * By default, the source code expects the compiler to correctly optimize 49 | * 4-bytes and 8-bytes read on architectures able to handle it efficiently. 50 | * This is not always the case. In some circumstances (ARM notably), 51 | * the compiler will issue cautious code even when target is able to correctly handle unaligned memory accesses. 52 | * 53 | * You can force the compiler to use unaligned memory access by uncommenting the line below. 54 | * One of the below scenarios will happen : 55 | * 1 - Your target CPU correctly handle unaligned access, and was not well optimized by compiler (good case). 56 | * You will witness large performance improvements (+50% and up). 57 | * Keep the line uncommented and send a word to upstream (https://groups.google.com/forum/#!forum/lz4c) 58 | * The goal is to automatically detect such situations by adding your target CPU within an exception list. 59 | * 2 - Your target CPU correctly handle unaligned access, and was already already optimized by compiler 60 | * No change will be experienced. 61 | * 3 - Your target CPU inefficiently handle unaligned access. 62 | * You will experience a performance loss. Comment back the line. 63 | * 4 - Your target CPU does not handle unaligned access. 64 | * Program will crash. 65 | * If uncommenting results in better performance (case 1) 66 | * please report your configuration to upstream (https://groups.google.com/forum/#!forum/lz4c) 67 | * An automatic detection macro will be added to match your case within future versions of the library. 68 | */ 69 | /* #define CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS 1 */ 70 | 71 | 72 | /************************************** 73 | CPU Feature Detection 74 | **************************************/ 75 | /* 76 | * Automated efficient unaligned memory access detection 77 | * Based on known hardware architectures 78 | * This list will be updated thanks to feedbacks 79 | */ 80 | #if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ 81 | || defined(__ARM_FEATURE_UNALIGNED) \ 82 | || defined(__i386__) || defined(__x86_64__) \ 83 | || defined(_M_IX86) || defined(_M_X64) \ 84 | || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ 85 | || (defined(_M_ARM) && (_M_ARM >= 7)) 86 | # define LZ4_UNALIGNED_ACCESS 1 87 | #else 88 | # define LZ4_UNALIGNED_ACCESS 0 89 | #endif 90 | 91 | /* 92 | * LZ4_FORCE_SW_BITCOUNT 93 | * Define this parameter if your target system or compiler does not support hardware bit count 94 | */ 95 | #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ 96 | # define LZ4_FORCE_SW_BITCOUNT 97 | #endif 98 | 99 | 100 | /************************************** 101 | Compiler Options 102 | **************************************/ 103 | #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ 104 | /* "restrict" is a known keyword */ 105 | #else 106 | # define restrict /* Disable restrict */ 107 | #endif 108 | 109 | #ifdef _MSC_VER /* Visual Studio */ 110 | # define FORCE_INLINE static __forceinline 111 | # include 112 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 113 | # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ 114 | #else 115 | # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ 116 | # ifdef __GNUC__ 117 | # define FORCE_INLINE static inline __attribute__((always_inline)) 118 | # else 119 | # define FORCE_INLINE static inline 120 | # endif 121 | # else 122 | # define FORCE_INLINE static 123 | # endif /* __STDC_VERSION__ */ 124 | #endif /* _MSC_VER */ 125 | 126 | #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 127 | 128 | #if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) 129 | # define expect(expr,value) (__builtin_expect ((expr),(value)) ) 130 | #else 131 | # define expect(expr,value) (expr) 132 | #endif 133 | 134 | #define likely(expr) expect((expr) != 0, 1) 135 | #define unlikely(expr) expect((expr) != 0, 0) 136 | 137 | 138 | /************************************** 139 | Memory routines 140 | **************************************/ 141 | #include /* malloc, calloc, free */ 142 | #define ALLOCATOR(n,s) calloc(n,s) 143 | #define FREEMEM free 144 | #include /* memset, memcpy */ 145 | #define MEM_INIT memset 146 | 147 | 148 | /************************************** 149 | Includes 150 | **************************************/ 151 | #include "lz4.h" 152 | 153 | 154 | /************************************** 155 | Basic Types 156 | **************************************/ 157 | #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ 158 | # include 159 | typedef uint8_t BYTE; 160 | typedef uint16_t U16; 161 | typedef uint32_t U32; 162 | typedef int32_t S32; 163 | typedef uint64_t U64; 164 | #else 165 | typedef unsigned char BYTE; 166 | typedef unsigned short U16; 167 | typedef unsigned int U32; 168 | typedef signed int S32; 169 | typedef unsigned long long U64; 170 | #endif 171 | 172 | 173 | /************************************** 174 | Reading and writing into memory 175 | **************************************/ 176 | #define STEPSIZE sizeof(size_t) 177 | 178 | static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } 179 | 180 | static unsigned LZ4_isLittleEndian(void) 181 | { 182 | const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 183 | return one.c[0]; 184 | } 185 | 186 | 187 | static U16 LZ4_readLE16(const void* memPtr) 188 | { 189 | if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) 190 | return *(U16*)memPtr; 191 | else 192 | { 193 | const BYTE* p = memPtr; 194 | return (U16)((U16)p[0] + (p[1]<<8)); 195 | } 196 | } 197 | 198 | static void LZ4_writeLE16(void* memPtr, U16 value) 199 | { 200 | if ((LZ4_UNALIGNED_ACCESS) && (LZ4_isLittleEndian())) 201 | { 202 | *(U16*)memPtr = value; 203 | return; 204 | } 205 | else 206 | { 207 | BYTE* p = memPtr; 208 | p[0] = (BYTE) value; 209 | p[1] = (BYTE)(value>>8); 210 | } 211 | } 212 | 213 | 214 | static U16 LZ4_read16(const void* memPtr) 215 | { 216 | if (LZ4_UNALIGNED_ACCESS) 217 | return *(U16*)memPtr; 218 | else 219 | { 220 | U16 val16; 221 | memcpy(&val16, memPtr, 2); 222 | return val16; 223 | } 224 | } 225 | 226 | static U32 LZ4_read32(const void* memPtr) 227 | { 228 | if (LZ4_UNALIGNED_ACCESS) 229 | return *(U32*)memPtr; 230 | else 231 | { 232 | U32 val32; 233 | memcpy(&val32, memPtr, 4); 234 | return val32; 235 | } 236 | } 237 | 238 | static U64 LZ4_read64(const void* memPtr) 239 | { 240 | if (LZ4_UNALIGNED_ACCESS) 241 | return *(U64*)memPtr; 242 | else 243 | { 244 | U64 val64; 245 | memcpy(&val64, memPtr, 8); 246 | return val64; 247 | } 248 | } 249 | 250 | static size_t LZ4_read_ARCH(const void* p) 251 | { 252 | if (LZ4_64bits()) 253 | return (size_t)LZ4_read64(p); 254 | else 255 | return (size_t)LZ4_read32(p); 256 | } 257 | 258 | 259 | static void LZ4_copy4(void* dstPtr, const void* srcPtr) 260 | { 261 | if (LZ4_UNALIGNED_ACCESS) 262 | { 263 | *(U32*)dstPtr = *(U32*)srcPtr; 264 | return; 265 | } 266 | memcpy(dstPtr, srcPtr, 4); 267 | } 268 | 269 | static void LZ4_copy8(void* dstPtr, const void* srcPtr) 270 | { 271 | #if GCC_VERSION!=409 /* disabled on GCC 4.9, as it generates invalid opcode (crash) */ 272 | if (LZ4_UNALIGNED_ACCESS) 273 | { 274 | if (LZ4_64bits()) 275 | *(U64*)dstPtr = *(U64*)srcPtr; 276 | else 277 | ((U32*)dstPtr)[0] = ((U32*)srcPtr)[0], 278 | ((U32*)dstPtr)[1] = ((U32*)srcPtr)[1]; 279 | return; 280 | } 281 | #endif 282 | memcpy(dstPtr, srcPtr, 8); 283 | } 284 | 285 | /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ 286 | static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) 287 | { 288 | BYTE* d = dstPtr; 289 | const BYTE* s = srcPtr; 290 | BYTE* e = dstEnd; 291 | do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); 337 | # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) 338 | return (__builtin_ctzll((U64)val) >> 3); 339 | # else 340 | static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; 341 | return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; 342 | # endif 343 | } 344 | else /* 32 bits */ 345 | { 346 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 347 | unsigned long r; 348 | _BitScanForward( &r, (U32)val ); 349 | return (int)(r>>3); 350 | # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) 351 | return (__builtin_ctz((U32)val) >> 3); 352 | # else 353 | static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; 354 | return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; 355 | # endif 356 | } 357 | } 358 | else /* Big Endian CPU */ 359 | { 360 | if (LZ4_64bits()) 361 | { 362 | # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) 363 | unsigned long r = 0; 364 | _BitScanReverse64( &r, val ); 365 | return (unsigned)(r>>3); 366 | # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) 367 | return (__builtin_clzll(val) >> 3); 368 | # else 369 | unsigned r; 370 | if (!(val>>32)) { r=4; } else { r=0; val>>=32; } 371 | if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } 372 | r += (!val); 373 | return r; 374 | # endif 375 | } 376 | else /* 32 bits */ 377 | { 378 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 379 | unsigned long r = 0; 380 | _BitScanReverse( &r, (unsigned long)val ); 381 | return (unsigned)(r>>3); 382 | # elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) 383 | return (__builtin_clz(val) >> 3); 384 | # else 385 | unsigned r; 386 | if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } 387 | r += (!val); 388 | return r; 389 | # endif 390 | } 391 | } 392 | } 393 | 394 | static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) 395 | { 396 | const BYTE* const pStart = pIn; 397 | 398 | while (likely(pIn compression run slower on incompressible data */ 423 | 424 | 425 | /************************************** 426 | Local Utils 427 | **************************************/ 428 | int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } 429 | int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } 430 | 431 | 432 | /************************************** 433 | Local Structures and types 434 | **************************************/ 435 | typedef struct { 436 | U32 hashTable[HASH_SIZE_U32]; 437 | U32 currentOffset; 438 | U32 initCheck; 439 | const BYTE* dictionary; 440 | const BYTE* bufferStart; 441 | U32 dictSize; 442 | } LZ4_stream_t_internal; 443 | 444 | typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; 445 | typedef enum { byPtr, byU32, byU16 } tableType_t; 446 | 447 | typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; 448 | typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; 449 | 450 | typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; 451 | typedef enum { full = 0, partial = 1 } earlyEnd_directive; 452 | 453 | 454 | 455 | /******************************** 456 | Compression functions 457 | ********************************/ 458 | 459 | static U32 LZ4_hashSequence(U32 sequence, tableType_t tableType) 460 | { 461 | if (tableType == byU16) 462 | return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); 463 | else 464 | return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); 465 | } 466 | 467 | static U32 LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(LZ4_read32(p), tableType); } 468 | 469 | static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) 470 | { 471 | switch (tableType) 472 | { 473 | case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } 474 | case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } 475 | case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } 476 | } 477 | } 478 | 479 | static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 480 | { 481 | U32 h = LZ4_hashPosition(p, tableType); 482 | LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); 483 | } 484 | 485 | static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) 486 | { 487 | if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } 488 | if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } 489 | { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ 490 | } 491 | 492 | static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 493 | { 494 | U32 h = LZ4_hashPosition(p, tableType); 495 | return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); 496 | } 497 | 498 | static int LZ4_compress_generic( 499 | void* ctx, 500 | const char* source, 501 | char* dest, 502 | int inputSize, 503 | int maxOutputSize, 504 | limitedOutput_directive outputLimited, 505 | tableType_t tableType, 506 | dict_directive dict, 507 | dictIssue_directive dictIssue) 508 | { 509 | LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; 510 | 511 | const BYTE* ip = (const BYTE*) source; 512 | const BYTE* base; 513 | const BYTE* lowLimit; 514 | const BYTE* const lowRefLimit = ip - dictPtr->dictSize; 515 | const BYTE* const dictionary = dictPtr->dictionary; 516 | const BYTE* const dictEnd = dictionary + dictPtr->dictSize; 517 | const size_t dictDelta = dictEnd - (const BYTE*)source; 518 | const BYTE* anchor = (const BYTE*) source; 519 | const BYTE* const iend = ip + inputSize; 520 | const BYTE* const mflimit = iend - MFLIMIT; 521 | const BYTE* const matchlimit = iend - LASTLITERALS; 522 | 523 | BYTE* op = (BYTE*) dest; 524 | BYTE* const olimit = op + maxOutputSize; 525 | 526 | U32 forwardH; 527 | size_t refDelta=0; 528 | 529 | /* Init conditions */ 530 | if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ 531 | switch(dict) 532 | { 533 | case noDict: 534 | default: 535 | base = (const BYTE*)source; 536 | lowLimit = (const BYTE*)source; 537 | break; 538 | case withPrefix64k: 539 | base = (const BYTE*)source - dictPtr->currentOffset; 540 | lowLimit = (const BYTE*)source - dictPtr->dictSize; 541 | break; 542 | case usingExtDict: 543 | base = (const BYTE*)source - dictPtr->currentOffset; 544 | lowLimit = (const BYTE*)source; 545 | break; 546 | } 547 | if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 548 | if (inputSize> LZ4_skipTrigger; 570 | 571 | if (unlikely(forwardIp > mflimit)) goto _last_literals; 572 | 573 | match = LZ4_getPositionOnHash(h, ctx, tableType, base); 574 | if (dict==usingExtDict) 575 | { 576 | if (match<(const BYTE*)source) 577 | { 578 | refDelta = dictDelta; 579 | lowLimit = dictionary; 580 | } 581 | else 582 | { 583 | refDelta = 0; 584 | lowLimit = (const BYTE*)source; 585 | } 586 | } 587 | forwardH = LZ4_hashPosition(forwardIp, tableType); 588 | LZ4_putPositionOnHash(ip, h, ctx, tableType, base); 589 | 590 | } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) 591 | || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) 592 | || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); 593 | } 594 | 595 | /* Catch up */ 596 | while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } 597 | 598 | { 599 | /* Encode Literal length */ 600 | unsigned litLength = (unsigned)(ip - anchor); 601 | token = op++; 602 | if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) 603 | return 0; /* Check output limit */ 604 | if (litLength>=RUN_MASK) 605 | { 606 | int len = (int)litLength-RUN_MASK; 607 | *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; 609 | *op++ = (BYTE)len; 610 | } 611 | else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; 632 | matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); 633 | ip += MINMATCH + matchLength; 634 | if (ip==limit) 635 | { 636 | unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); 637 | matchLength += more; 638 | ip += more; 639 | } 640 | } 641 | else 642 | { 643 | matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); 644 | ip += MINMATCH + matchLength; 645 | } 646 | 647 | if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) 648 | return 0; /* Check output limit */ 649 | if (matchLength>=ML_MASK) 650 | { 651 | *token += ML_MASK; 652 | matchLength -= ML_MASK; 653 | for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } 654 | if (matchLength >= 255) { matchLength-=255; *op++ = 255; } 655 | *op++ = (BYTE)matchLength; 656 | } 657 | else *token += (BYTE)(matchLength); 658 | } 659 | 660 | anchor = ip; 661 | 662 | /* Test end of chunk */ 663 | if (ip > mflimit) break; 664 | 665 | /* Fill table */ 666 | LZ4_putPosition(ip-2, ctx, tableType, base); 667 | 668 | /* Test next position */ 669 | match = LZ4_getPosition(ip, ctx, tableType, base); 670 | if (dict==usingExtDict) 671 | { 672 | if (match<(const BYTE*)source) 673 | { 674 | refDelta = dictDelta; 675 | lowLimit = dictionary; 676 | } 677 | else 678 | { 679 | refDelta = 0; 680 | lowLimit = (const BYTE*)source; 681 | } 682 | } 683 | LZ4_putPosition(ip, ctx, tableType, base); 684 | if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) 685 | && (match+MAX_DISTANCE>=ip) 686 | && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) 687 | { token=op++; *token=0; goto _next_match; } 688 | 689 | /* Prepare next loop */ 690 | forwardH = LZ4_hashPosition(++ip, tableType); 691 | } 692 | 693 | _last_literals: 694 | /* Encode Last Literals */ 695 | { 696 | int lastRun = (int)(iend - anchor); 697 | if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) 698 | return 0; /* Check output limit */ 699 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 700 | else *op++ = (BYTE)(lastRun<= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ 769 | LZ4_resetStream(lz4s); 770 | return lz4s; 771 | } 772 | 773 | int LZ4_freeStream (LZ4_stream_t* LZ4_stream) 774 | { 775 | FREEMEM(LZ4_stream); 776 | return (0); 777 | } 778 | 779 | 780 | int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) 781 | { 782 | LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; 783 | const BYTE* p = (const BYTE*)dictionary; 784 | const BYTE* const dictEnd = p + dictSize; 785 | const BYTE* base; 786 | 787 | if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */ 788 | 789 | if (dictSize < MINMATCH) 790 | { 791 | dict->dictionary = NULL; 792 | dict->dictSize = 0; 793 | return 0; 794 | } 795 | 796 | if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB; 797 | base = p - dict->currentOffset; 798 | dict->dictionary = p; 799 | dict->dictSize = (U32)(dictEnd - p); 800 | dict->currentOffset += dict->dictSize; 801 | 802 | while (p <= dictEnd-MINMATCH) 803 | { 804 | LZ4_putPosition(p, dict, byU32, base); 805 | p+=3; 806 | } 807 | 808 | return dict->dictSize; 809 | } 810 | 811 | 812 | static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) 813 | { 814 | if ((LZ4_dict->currentOffset > 0x80000000) || 815 | ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ 816 | { 817 | /* rescale hash table */ 818 | U32 delta = LZ4_dict->currentOffset - 64 KB; 819 | const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; 820 | int i; 821 | for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; 824 | else LZ4_dict->hashTable[i] -= delta; 825 | } 826 | LZ4_dict->currentOffset = 64 KB; 827 | if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; 828 | LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; 829 | } 830 | } 831 | 832 | 833 | FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize, 834 | int maxOutputSize, limitedOutput_directive limit) 835 | { 836 | LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; 837 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 838 | 839 | const BYTE* smallest = (const BYTE*) source; 840 | if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ 841 | if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; 842 | LZ4_renormDictT(streamPtr, smallest); 843 | 844 | /* Check overlapping input/dictionary space */ 845 | { 846 | const BYTE* sourceEnd = (const BYTE*) source + inputSize; 847 | if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) 848 | { 849 | streamPtr->dictSize = (U32)(dictEnd - sourceEnd); 850 | if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; 851 | if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; 852 | streamPtr->dictionary = dictEnd - streamPtr->dictSize; 853 | } 854 | } 855 | 856 | /* prefix mode : source data follows dictionary */ 857 | if (dictEnd == (const BYTE*)source) 858 | { 859 | int result; 860 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 861 | result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall); 862 | else 863 | result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue); 864 | streamPtr->dictSize += (U32)inputSize; 865 | streamPtr->currentOffset += (U32)inputSize; 866 | return result; 867 | } 868 | 869 | /* external dictionary mode */ 870 | { 871 | int result; 872 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 873 | result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall); 874 | else 875 | result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue); 876 | streamPtr->dictionary = (const BYTE*)source; 877 | streamPtr->dictSize = (U32)inputSize; 878 | streamPtr->currentOffset += (U32)inputSize; 879 | return result; 880 | } 881 | } 882 | 883 | int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) 884 | { 885 | return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited); 886 | } 887 | 888 | int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize) 889 | { 890 | return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput); 891 | } 892 | 893 | 894 | /* Hidden debug function, to force separate dictionary mode */ 895 | int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) 896 | { 897 | LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; 898 | int result; 899 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 900 | 901 | const BYTE* smallest = dictEnd; 902 | if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; 903 | LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); 904 | 905 | result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue); 906 | 907 | streamPtr->dictionary = (const BYTE*)source; 908 | streamPtr->dictSize = (U32)inputSize; 909 | streamPtr->currentOffset += (U32)inputSize; 910 | 911 | return result; 912 | } 913 | 914 | 915 | int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) 916 | { 917 | LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; 918 | const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; 919 | 920 | if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ 921 | if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; 922 | 923 | memmove(safeBuffer, previousDictEnd - dictSize, dictSize); 924 | 925 | dict->dictionary = (const BYTE*)safeBuffer; 926 | dict->dictSize = (U32)dictSize; 927 | 928 | return dictSize; 929 | } 930 | 931 | 932 | 933 | /**************************** 934 | Decompression functions 935 | ****************************/ 936 | /* 937 | * This generic decompression function cover all use cases. 938 | * It shall be instantiated several times, using different sets of directives 939 | * Note that it is essential this generic function is really inlined, 940 | * in order to remove useless branches during compilation optimization. 941 | */ 942 | FORCE_INLINE int LZ4_decompress_generic( 943 | const char* const source, 944 | char* const dest, 945 | int inputSize, 946 | int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ 947 | 948 | int endOnInput, /* endOnOutputSize, endOnInputSize */ 949 | int partialDecoding, /* full, partial */ 950 | int targetOutputSize, /* only used if partialDecoding==partial */ 951 | int dict, /* noDict, withPrefix64k, usingExtDict */ 952 | const BYTE* const lowPrefix, /* == dest if dict == noDict */ 953 | const BYTE* const dictStart, /* only if dict==usingExtDict */ 954 | const size_t dictSize /* note : = 0 if noDict */ 955 | ) 956 | { 957 | /* Local Variables */ 958 | const BYTE* restrict ip = (const BYTE*) source; 959 | const BYTE* const iend = ip + inputSize; 960 | 961 | BYTE* op = (BYTE*) dest; 962 | BYTE* const oend = op + outputSize; 963 | BYTE* cpy; 964 | BYTE* oexit = op + targetOutputSize; 965 | const BYTE* const lowLimit = lowPrefix - dictSize; 966 | 967 | const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; 968 | const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; 969 | const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; 970 | 971 | const int safeDecode = (endOnInput==endOnInputSize); 972 | const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); 973 | 974 | 975 | /* Special cases */ 976 | if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ 977 | if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ 978 | if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); 979 | 980 | 981 | /* Main Loop */ 982 | while (1) 983 | { 984 | unsigned token; 985 | size_t length; 986 | const BYTE* match; 987 | 988 | /* get literal length */ 989 | token = *ip++; 990 | if ((length=(token>>ML_BITS)) == RUN_MASK) 991 | { 992 | unsigned s; 993 | do 994 | { 995 | s = *ip++; 996 | length += s; 997 | } 998 | while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) 1006 | || ((!endOnInput) && (cpy>oend-COPYLENGTH))) 1007 | { 1008 | if (partialDecoding) 1009 | { 1010 | if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ 1011 | if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ 1012 | } 1013 | else 1014 | { 1015 | if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ 1016 | if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ 1017 | } 1018 | memcpy(op, ip, length); 1019 | ip += length; 1020 | op += length; 1021 | break; /* Necessarily EOF, due to parsing restrictions */ 1022 | } 1023 | LZ4_wildCopy(op, ip, cpy); 1024 | ip += length; op = cpy; 1025 | 1026 | /* get offset */ 1027 | match = cpy - LZ4_readLE16(ip); ip+=2; 1028 | if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ 1029 | 1030 | /* get matchlength */ 1031 | length = token & ML_MASK; 1032 | if (length == ML_MASK) 1033 | { 1034 | unsigned s; 1035 | do 1036 | { 1037 | if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; 1038 | s = *ip++; 1039 | length += s; 1040 | } while (s==255); 1041 | if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ 1042 | } 1043 | length += MINMATCH; 1044 | 1045 | /* check external dictionary */ 1046 | if ((dict==usingExtDict) && (match < lowPrefix)) 1047 | { 1048 | if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ 1049 | 1050 | if (length <= (size_t)(lowPrefix-match)) 1051 | { 1052 | /* match can be copied as a single segment from external dictionary */ 1053 | match = dictEnd - (lowPrefix-match); 1054 | memcpy(op, match, length); 1055 | op += length; 1056 | } 1057 | else 1058 | { 1059 | /* match encompass external dictionary and current segment */ 1060 | size_t copySize = (size_t)(lowPrefix-match); 1061 | memcpy(op, dictEnd - copySize, copySize); 1062 | op += copySize; 1063 | copySize = length - copySize; 1064 | if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ 1065 | { 1066 | BYTE* const endOfMatch = op + copySize; 1067 | const BYTE* copyFrom = lowPrefix; 1068 | while (op < endOfMatch) *op++ = *copyFrom++; 1069 | } 1070 | else 1071 | { 1072 | memcpy(op, lowPrefix, copySize); 1073 | op += copySize; 1074 | } 1075 | } 1076 | continue; 1077 | } 1078 | 1079 | /* copy repeated sequence */ 1080 | cpy = op + length; 1081 | if (unlikely((op-match)<8)) 1082 | { 1083 | const size_t dec64 = dec64table[op-match]; 1084 | op[0] = match[0]; 1085 | op[1] = match[1]; 1086 | op[2] = match[2]; 1087 | op[3] = match[3]; 1088 | match += dec32table[op-match]; 1089 | LZ4_copy4(op+4, match); 1090 | op += 8; match -= dec64; 1091 | } else { LZ4_copy8(op, match); op+=8; match+=8; } 1092 | 1093 | if (unlikely(cpy>oend-12)) 1094 | { 1095 | if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ 1096 | if (op < oend-8) 1097 | { 1098 | LZ4_wildCopy(op, match, oend-8); 1099 | match += (oend-8) - op; 1100 | op = oend-8; 1101 | } 1102 | while (opprefixSize = (size_t) dictSize; 1175 | lz4sd->prefixEnd = (BYTE*) dictionary + dictSize; 1176 | lz4sd->externalDict = NULL; 1177 | lz4sd->extDictSize = 0; 1178 | return 1; 1179 | } 1180 | 1181 | /* 1182 | *_continue() : 1183 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 1184 | Previously decoded blocks must still be available at the memory position where they were decoded. 1185 | If it's not possible, save the relevant part of decoded data into a safe buffer, 1186 | and indicate where it stands using LZ4_setStreamDecode() 1187 | */ 1188 | int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) 1189 | { 1190 | LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; 1191 | int result; 1192 | 1193 | if (lz4sd->prefixEnd == (BYTE*)dest) 1194 | { 1195 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1196 | endOnInputSize, full, 0, 1197 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1198 | if (result <= 0) return result; 1199 | lz4sd->prefixSize += result; 1200 | lz4sd->prefixEnd += result; 1201 | } 1202 | else 1203 | { 1204 | lz4sd->extDictSize = lz4sd->prefixSize; 1205 | lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; 1206 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1207 | endOnInputSize, full, 0, 1208 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1209 | if (result <= 0) return result; 1210 | lz4sd->prefixSize = result; 1211 | lz4sd->prefixEnd = (BYTE*)dest + result; 1212 | } 1213 | 1214 | return result; 1215 | } 1216 | 1217 | int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) 1218 | { 1219 | LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; 1220 | int result; 1221 | 1222 | if (lz4sd->prefixEnd == (BYTE*)dest) 1223 | { 1224 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1225 | endOnOutputSize, full, 0, 1226 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1227 | if (result <= 0) return result; 1228 | lz4sd->prefixSize += originalSize; 1229 | lz4sd->prefixEnd += originalSize; 1230 | } 1231 | else 1232 | { 1233 | lz4sd->extDictSize = lz4sd->prefixSize; 1234 | lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; 1235 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1236 | endOnOutputSize, full, 0, 1237 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1238 | if (result <= 0) return result; 1239 | lz4sd->prefixSize = originalSize; 1240 | lz4sd->prefixEnd = (BYTE*)dest + originalSize; 1241 | } 1242 | 1243 | return result; 1244 | } 1245 | 1246 | 1247 | /* 1248 | Advanced decoding functions : 1249 | *_usingDict() : 1250 | These decoding functions work the same as "_continue" ones, 1251 | the dictionary must be explicitly provided within parameters 1252 | */ 1253 | 1254 | FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) 1255 | { 1256 | if (dictSize==0) 1257 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); 1258 | if (dictStart+dictSize == dest) 1259 | { 1260 | if (dictSize >= (int)(64 KB - 1)) 1261 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); 1262 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); 1263 | } 1264 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); 1265 | } 1266 | 1267 | int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1268 | { 1269 | return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); 1270 | } 1271 | 1272 | int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) 1273 | { 1274 | return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); 1275 | } 1276 | 1277 | /* debug function */ 1278 | int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1279 | { 1280 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (BYTE*)dictStart, dictSize); 1281 | } 1282 | 1283 | 1284 | /*************************************************** 1285 | Obsolete Functions 1286 | ***************************************************/ 1287 | /* 1288 | These function names are deprecated and should no longer be used. 1289 | They are only provided here for compatibility with older user programs. 1290 | - LZ4_uncompress is totally equivalent to LZ4_decompress_fast 1291 | - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe 1292 | */ 1293 | int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } 1294 | int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } 1295 | 1296 | 1297 | /* Obsolete Streaming functions */ 1298 | 1299 | int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } 1300 | 1301 | static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base) 1302 | { 1303 | MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); 1304 | lz4ds->bufferStart = base; 1305 | } 1306 | 1307 | int LZ4_resetStreamState(void* state, const char* inputBuffer) 1308 | { 1309 | if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ 1310 | LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer); 1311 | return 0; 1312 | } 1313 | 1314 | void* LZ4_create (const char* inputBuffer) 1315 | { 1316 | void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); 1317 | LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer); 1318 | return lz4ds; 1319 | } 1320 | 1321 | char* LZ4_slideInputBuffer (void* LZ4_Data) 1322 | { 1323 | LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; 1324 | int dictSize = LZ4_saveDict((LZ4_stream_t*)ctx, (char*)ctx->bufferStart, 64 KB); 1325 | return (char*)(ctx->bufferStart + dictSize); 1326 | } 1327 | 1328 | /* Obsolete compresson functions using User-allocated state */ 1329 | 1330 | int LZ4_sizeofState() { return LZ4_STREAMSIZE; } 1331 | 1332 | int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize) 1333 | { 1334 | if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ 1335 | MEM_INIT(state, 0, LZ4_STREAMSIZE); 1336 | 1337 | if (inputSize < LZ4_64Klimit) 1338 | return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue); 1339 | else 1340 | return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); 1341 | } 1342 | 1343 | int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize) 1344 | { 1345 | if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */ 1346 | MEM_INIT(state, 0, LZ4_STREAMSIZE); 1347 | 1348 | if (inputSize < LZ4_64Klimit) 1349 | return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue); 1350 | else 1351 | return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue); 1352 | } 1353 | 1354 | /* Obsolete streaming decompression functions */ 1355 | 1356 | int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) 1357 | { 1358 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1359 | } 1360 | 1361 | int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) 1362 | { 1363 | return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1364 | } 1365 | 1366 | #endif /* LZ4_COMMONDEFS_ONLY */ 1367 | 1368 | --------------------------------------------------------------------------------