├── .golangci.yml ├── .travis.yml ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── leveldb ├── batch.go ├── batch_test.go ├── bench_test.go ├── cache │ ├── bench_test.go │ ├── cache.go │ ├── cache_test.go │ └── lru.go ├── comparer.go ├── comparer │ ├── bytes_comparer.go │ └── comparer.go ├── corrupt_test.go ├── db.go ├── db_compaction.go ├── db_iter.go ├── db_snapshot.go ├── db_state.go ├── db_test.go ├── db_transaction.go ├── db_util.go ├── db_write.go ├── doc.go ├── errors.go ├── errors │ └── errors.go ├── external_test.go ├── filter.go ├── filter │ ├── bloom.go │ ├── bloom_test.go │ └── filter.go ├── iterator │ ├── array_iter.go │ ├── array_iter_test.go │ ├── indexed_iter.go │ ├── indexed_iter_test.go │ ├── iter.go │ ├── iter_suite_test.go │ ├── merged_iter.go │ └── merged_iter_test.go ├── journal │ ├── journal.go │ └── journal_test.go ├── key.go ├── key_test.go ├── leveldb_suite_test.go ├── memdb │ ├── bench_test.go │ ├── memdb.go │ ├── memdb_suite_test.go │ └── memdb_test.go ├── opt │ ├── options.go │ ├── options_darwin.go │ └── options_default.go ├── options.go ├── session.go ├── session_compaction.go ├── session_record.go ├── session_record_test.go ├── session_util.go ├── storage.go ├── storage │ ├── file_storage.go │ ├── file_storage_nacl.go │ ├── file_storage_plan9.go │ ├── file_storage_solaris.go │ ├── file_storage_test.go │ ├── file_storage_unix.go │ ├── file_storage_windows.go │ ├── mem_storage.go │ ├── mem_storage_test.go │ └── storage.go ├── table.go ├── table │ ├── block_test.go │ ├── reader.go │ ├── table.go │ ├── table_suite_test.go │ ├── table_test.go │ └── writer.go ├── table_test.go ├── testutil │ ├── db.go │ ├── ginkgo.go │ ├── iter.go │ ├── kv.go │ ├── kvtest.go │ ├── storage.go │ └── util.go ├── testutil_test.go ├── util.go ├── util │ ├── buffer.go │ ├── buffer_pool.go │ ├── buffer_test.go │ ├── crc32.go │ ├── hash.go │ ├── hash_test.go │ ├── range.go │ └── util.go ├── version.go └── version_test.go └── manualtest ├── dbstress ├── key.go └── main.go └── filelock └── main.go /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | gocritic: 3 | disabled-checks: 4 | - ifElseChain 5 | - elseif 6 | 7 | linters: 8 | enable: 9 | - gofmt 10 | - gocritic 11 | - unconvert 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | before_install: 4 | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.2 5 | 6 | go: 7 | - 1.14.x 8 | - 1.18.x 9 | - tip 10 | 11 | script: 12 | - go vet ./... 13 | - golangci-lint run 14 | - go test -short -timeout 1h ./... 15 | - go test -timeout 30m -race -run "TestDB_(Concurrent|GoleveldbIssue74)" ./leveldb 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Suryandaru Triandana 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is an implementation of the [LevelDB key/value database](https://github.com/google/leveldb) in the [Go programming language](https://go.dev). 2 | 3 | [![Build Status](https://app.travis-ci.com/syndtr/goleveldb.svg?branch=master)](https://app.travis-ci.com/syndtr/goleveldb) 4 | 5 | Installation 6 | ----------- 7 | 8 | go get github.com/syndtr/goleveldb/leveldb 9 | 10 | Requirements 11 | ----------- 12 | 13 | * Need at least `go1.14` or newer. 14 | 15 | Usage 16 | ----------- 17 | 18 | Create or open a database: 19 | ```go 20 | // The returned DB instance is safe for concurrent use. Which mean that all 21 | // DB's methods may be called concurrently from multiple goroutine. 22 | db, err := leveldb.OpenFile("path/to/db", nil) 23 | ... 24 | defer db.Close() 25 | ... 26 | ``` 27 | Read or modify the database content: 28 | ```go 29 | // Remember that the contents of the returned slice should not be modified. 30 | data, err := db.Get([]byte("key"), nil) 31 | ... 32 | err = db.Put([]byte("key"), []byte("value"), nil) 33 | ... 34 | err = db.Delete([]byte("key"), nil) 35 | ... 36 | ``` 37 | 38 | Iterate over database content: 39 | ```go 40 | iter := db.NewIterator(nil, nil) 41 | for iter.Next() { 42 | // Remember that the contents of the returned slice should not be modified, and 43 | // only valid until the next call to Next. 44 | key := iter.Key() 45 | value := iter.Value() 46 | ... 47 | } 48 | iter.Release() 49 | err = iter.Error() 50 | ... 51 | ``` 52 | Seek-then-Iterate: 53 | ```go 54 | iter := db.NewIterator(nil, nil) 55 | for ok := iter.Seek(key); ok; ok = iter.Next() { 56 | // Use key/value. 57 | ... 58 | } 59 | iter.Release() 60 | err = iter.Error() 61 | ... 62 | ``` 63 | Iterate over subset of database content: 64 | ```go 65 | iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) 66 | for iter.Next() { 67 | // Use key/value. 68 | ... 69 | } 70 | iter.Release() 71 | err = iter.Error() 72 | ... 73 | ``` 74 | Iterate over subset of database content with a particular prefix: 75 | ```go 76 | iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) 77 | for iter.Next() { 78 | // Use key/value. 79 | ... 80 | } 81 | iter.Release() 82 | err = iter.Error() 83 | ... 84 | ``` 85 | Batch writes: 86 | ```go 87 | batch := new(leveldb.Batch) 88 | batch.Put([]byte("foo"), []byte("value")) 89 | batch.Put([]byte("bar"), []byte("another value")) 90 | batch.Delete([]byte("baz")) 91 | err = db.Write(batch, nil) 92 | ... 93 | ``` 94 | Use bloom filter: 95 | ```go 96 | o := &opt.Options{ 97 | Filter: filter.NewBloomFilter(10), 98 | } 99 | db, err := leveldb.OpenFile("path/to/db", o) 100 | ... 101 | defer db.Close() 102 | ... 103 | ``` 104 | Documentation 105 | ----------- 106 | 107 | You can read package documentation [here](https://pkg.go.dev/github.com/syndtr/goleveldb). 108 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/syndtr/goleveldb 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.5.4 // indirect 7 | github.com/golang/snappy v0.0.4 8 | github.com/onsi/ginkgo v1.16.5 9 | github.com/onsi/gomega v1.19.0 10 | github.com/stretchr/testify v1.7.2 11 | golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect 12 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /leveldb/batch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "math/rand" 13 | "testing" 14 | "testing/quick" 15 | 16 | "github.com/syndtr/goleveldb/leveldb/testutil" 17 | ) 18 | 19 | func TestBatchHeader(t *testing.T) { 20 | f := func(seq uint64, length uint32) bool { 21 | encoded := encodeBatchHeader(nil, seq, int(length)) 22 | decSeq, decLength, err := decodeBatchHeader(encoded) 23 | return err == nil && decSeq == seq && decLength == int(length) 24 | } 25 | config := &quick.Config{ 26 | Rand: testutil.NewRand(), 27 | } 28 | if err := quick.Check(f, config); err != nil { 29 | t.Error(err) 30 | } 31 | } 32 | 33 | type batchKV struct { 34 | kt keyType 35 | k, v []byte 36 | } 37 | 38 | func TestBatch(t *testing.T) { 39 | var ( 40 | kvs []batchKV 41 | internalLen int 42 | ) 43 | batch := new(Batch) 44 | rbatch := new(Batch) 45 | abatch := new(Batch) 46 | testBatch := func(i int, kt keyType, k, v []byte) error { 47 | kv := kvs[i] 48 | if kv.kt != kt { 49 | return fmt.Errorf("invalid key type, index=%d: %d vs %d", i, kv.kt, kt) 50 | } 51 | if !bytes.Equal(kv.k, k) { 52 | return fmt.Errorf("invalid key, index=%d", i) 53 | } 54 | if !bytes.Equal(kv.v, v) { 55 | return fmt.Errorf("invalid value, index=%d", i) 56 | } 57 | return nil 58 | } 59 | f := func(ktr uint8, k, v []byte) bool { 60 | kt := keyType(ktr % 2) 61 | if kt == keyTypeVal { 62 | batch.Put(k, v) 63 | rbatch.Put(k, v) 64 | kvs = append(kvs, batchKV{kt: kt, k: k, v: v}) 65 | internalLen += len(k) + len(v) + 8 66 | } else { 67 | batch.Delete(k) 68 | rbatch.Delete(k) 69 | kvs = append(kvs, batchKV{kt: kt, k: k}) 70 | internalLen += len(k) + 8 71 | } 72 | if batch.Len() != len(kvs) { 73 | t.Logf("batch.Len: %d vs %d", len(kvs), batch.Len()) 74 | return false 75 | } 76 | if batch.internalLen != internalLen { 77 | t.Logf("abatch.internalLen: %d vs %d", internalLen, batch.internalLen) 78 | return false 79 | } 80 | if len(kvs)%1000 == 0 { 81 | if err := batch.replayInternal(testBatch); err != nil { 82 | t.Logf("batch.replayInternal: %v", err) 83 | return false 84 | } 85 | 86 | abatch.append(rbatch) 87 | rbatch.Reset() 88 | if abatch.Len() != len(kvs) { 89 | t.Logf("abatch.Len: %d vs %d", len(kvs), abatch.Len()) 90 | return false 91 | } 92 | if abatch.internalLen != internalLen { 93 | t.Logf("abatch.internalLen: %d vs %d", internalLen, abatch.internalLen) 94 | return false 95 | } 96 | if err := abatch.replayInternal(testBatch); err != nil { 97 | t.Logf("abatch.replayInternal: %v", err) 98 | return false 99 | } 100 | 101 | nbatch := new(Batch) 102 | if err := nbatch.Load(batch.Dump()); err != nil { 103 | t.Logf("nbatch.Load: %v", err) 104 | return false 105 | } 106 | if nbatch.Len() != len(kvs) { 107 | t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) 108 | return false 109 | } 110 | if nbatch.internalLen != internalLen { 111 | t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) 112 | return false 113 | } 114 | if err := nbatch.replayInternal(testBatch); err != nil { 115 | t.Logf("nbatch.replayInternal: %v", err) 116 | return false 117 | } 118 | } 119 | if len(kvs)%10000 == 0 { 120 | nbatch := new(Batch) 121 | if err := batch.Replay(nbatch); err != nil { 122 | t.Logf("batch.Replay: %v", err) 123 | return false 124 | } 125 | if nbatch.Len() != len(kvs) { 126 | t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) 127 | return false 128 | } 129 | if nbatch.internalLen != internalLen { 130 | t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) 131 | return false 132 | } 133 | if err := nbatch.replayInternal(testBatch); err != nil { 134 | t.Logf("nbatch.replayInternal: %v", err) 135 | return false 136 | } 137 | } 138 | return true 139 | } 140 | config := &quick.Config{ 141 | MaxCount: 40000, 142 | Rand: testutil.NewRand(), 143 | } 144 | if err := quick.Check(f, config); err != nil { 145 | t.Error(err) 146 | } 147 | t.Logf("length=%d internalLen=%d", len(kvs), internalLen) 148 | } 149 | 150 | func BenchmarkDefaultBatchWrite(b *testing.B) { 151 | benchmarkBatchWrite(b, nil) 152 | } 153 | 154 | func BenchmarkFastAllocationBatchWrite(b *testing.B) { 155 | benchmarkBatchWrite(b, &BatchConfig{ 156 | GrowLimit: 10 * batchGrowLimit, 157 | }) 158 | } 159 | 160 | func benchmarkBatchWrite(b *testing.B, config *BatchConfig) { 161 | var ( 162 | keys [][]byte 163 | vals [][]byte 164 | r = rand.New(rand.NewSource(1337)) 165 | ) 166 | for i := 0; i < 50000; i++ { 167 | keys = append(keys, randomString(r, 32)) 168 | vals = append(vals, randomString(r, 100)) 169 | } 170 | b.ResetTimer() 171 | for round := 0; round < b.N; round++ { 172 | batch := MakeBatchWithConfig(config) 173 | for i := 0; i < len(keys); i++ { 174 | batch.Put(keys[i], vals[i]) 175 | } 176 | } 177 | b.ReportAllocs() 178 | } 179 | -------------------------------------------------------------------------------- /leveldb/cache/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package cache 8 | 9 | import ( 10 | "sync/atomic" 11 | "testing" 12 | ) 13 | 14 | func BenchmarkCache_InsertRemove(b *testing.B) { 15 | b.StopTimer() 16 | c := NewCache(nil) 17 | 18 | b.StartTimer() 19 | for i := 0; i < b.N; i++ { 20 | c.Get(0, uint64(i), func() (int, Value) { 21 | return 1, uint64(i) 22 | }).Release() 23 | } 24 | b.ReportMetric(float64(c.Nodes()), "nodes") 25 | b.Logf("STATS: %#v", c.GetStats()) 26 | } 27 | 28 | func BenchmarkCache_Insert(b *testing.B) { 29 | b.StopTimer() 30 | c := NewCache(nil) 31 | 32 | b.StartTimer() 33 | for i := 0; i < b.N; i++ { 34 | c.Get(0, uint64(i), func() (int, Value) { 35 | return 1, uint64(i) 36 | }) 37 | } 38 | b.ReportMetric(float64(c.Nodes()), "nodes") 39 | b.Logf("STATS: %#v", c.GetStats()) 40 | } 41 | 42 | func BenchmarkCache_Lookup(b *testing.B) { 43 | b.StopTimer() 44 | c := NewCache(nil) 45 | for i := 0; i < b.N; i++ { 46 | c.Get(0, uint64(i), func() (int, Value) { 47 | return 1, uint64(i) 48 | }) 49 | } 50 | 51 | b.StartTimer() 52 | for i := 0; i < b.N; i++ { 53 | c.Get(0, uint64(i), nil).Release() 54 | } 55 | b.ReportMetric(float64(c.Nodes()), "nodes") 56 | b.Logf("STATS: %#v", c.GetStats()) 57 | } 58 | 59 | func BenchmarkCache_AppendRemove(b *testing.B) { 60 | b.StopTimer() 61 | c := NewCache(nil) 62 | for i := 0; i < b.N; i++ { 63 | c.Get(0, uint64(i), func() (int, Value) { 64 | return 1, uint64(i) 65 | }) 66 | } 67 | 68 | b.StartTimer() 69 | for i := 0; i < b.N; i++ { 70 | c.Get(1, uint64(i), func() (int, Value) { 71 | return 1, uint64(i) 72 | }).Release() 73 | } 74 | b.ReportMetric(float64(c.Nodes()), "nodes") 75 | b.Logf("STATS: %#v", c.GetStats()) 76 | } 77 | 78 | func BenchmarkCache_Append(b *testing.B) { 79 | b.StopTimer() 80 | c := NewCache(nil) 81 | for i := 0; i < b.N; i++ { 82 | c.Get(0, uint64(i), func() (int, Value) { 83 | return 1, uint64(i) 84 | }) 85 | } 86 | 87 | b.StartTimer() 88 | for i := 0; i < b.N; i++ { 89 | c.Get(1, uint64(i), func() (int, Value) { 90 | return 1, uint64(i) 91 | }) 92 | } 93 | b.ReportMetric(float64(c.Nodes()), "nodes") 94 | b.Logf("STATS: %#v", c.GetStats()) 95 | } 96 | 97 | func BenchmarkCache_Delete(b *testing.B) { 98 | b.StopTimer() 99 | c := NewCache(nil) 100 | handles := make([]*Handle, b.N) 101 | for i := 0; i < b.N; i++ { 102 | handles[i] = c.Get(0, uint64(i), func() (int, Value) { 103 | return 1, uint64(i) 104 | }) 105 | } 106 | 107 | b.StartTimer() 108 | for i := 0; i < b.N; i++ { 109 | handles[i].Release() 110 | } 111 | b.ReportMetric(float64(c.Nodes()), "nodes") 112 | b.Logf("STATS: %#v", c.GetStats()) 113 | } 114 | 115 | func BenchmarkCacheParallel_Insert(b *testing.B) { 116 | b.StopTimer() 117 | c := NewCache(nil) 118 | 119 | var ns uint64 120 | b.StartTimer() 121 | b.RunParallel(func(pb *testing.PB) { 122 | ns := atomic.AddUint64(&ns, 1) 123 | i := uint64(0) 124 | for pb.Next() { 125 | c.Get(ns, i, func() (int, Value) { 126 | return 1, i 127 | }) 128 | i++ 129 | } 130 | }) 131 | b.ReportMetric(float64(c.Nodes()), "nodes") 132 | b.Logf("STATS: %#v", c.GetStats()) 133 | } 134 | 135 | func BenchmarkCacheParallel_Lookup(b *testing.B) { 136 | b.StopTimer() 137 | c := NewCache(nil) 138 | for i := 0; i < b.N; i++ { 139 | c.Get(0, uint64(i), func() (int, Value) { 140 | return 1, uint64(i) 141 | }) 142 | } 143 | 144 | var counter uint64 145 | b.StartTimer() 146 | b.RunParallel(func(pb *testing.PB) { 147 | for pb.Next() { 148 | i := atomic.AddUint64(&counter, 1) - 1 149 | c.Get(0, i, nil).Release() 150 | } 151 | }) 152 | b.ReportMetric(float64(c.Nodes()), "nodes") 153 | b.Logf("STATS: %#v", c.GetStats()) 154 | } 155 | 156 | func BenchmarkCacheParallel_Append(b *testing.B) { 157 | b.StopTimer() 158 | c := NewCache(nil) 159 | for i := 0; i < b.N; i++ { 160 | c.Get(0, uint64(i), func() (int, Value) { 161 | return 1, uint64(i) 162 | }) 163 | } 164 | 165 | var ns uint64 166 | b.StartTimer() 167 | b.RunParallel(func(pb *testing.PB) { 168 | ns := atomic.AddUint64(&ns, 1) 169 | i := uint64(0) 170 | for pb.Next() { 171 | c.Get(ns, i, func() (int, Value) { 172 | return 1, i 173 | }) 174 | i++ 175 | } 176 | }) 177 | b.ReportMetric(float64(c.Nodes()), "nodes") 178 | b.Logf("STATS: %#v", c.GetStats()) 179 | } 180 | 181 | func BenchmarkCacheParallel_Delete(b *testing.B) { 182 | b.StopTimer() 183 | c := NewCache(nil) 184 | handles := make([]*Handle, b.N) 185 | for i := 0; i < b.N; i++ { 186 | handles[i] = c.Get(0, uint64(i), func() (int, Value) { 187 | return 1, uint64(i) 188 | }) 189 | } 190 | 191 | var counter int64 192 | b.StartTimer() 193 | b.RunParallel(func(pb *testing.PB) { 194 | for pb.Next() { 195 | i := atomic.AddInt64(&counter, 1) - 1 196 | handles[i].Release() 197 | } 198 | }) 199 | b.ReportMetric(float64(c.Nodes()), "nodes") 200 | b.Logf("STATS: %#v", c.GetStats()) 201 | } 202 | -------------------------------------------------------------------------------- /leveldb/cache/lru.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package cache 8 | 9 | import ( 10 | "sync" 11 | "unsafe" 12 | ) 13 | 14 | type lruNode struct { 15 | n *Node 16 | h *Handle 17 | ban bool 18 | 19 | next, prev *lruNode 20 | } 21 | 22 | func (n *lruNode) insert(at *lruNode) { 23 | x := at.next 24 | at.next = n 25 | n.prev = at 26 | n.next = x 27 | x.prev = n 28 | } 29 | 30 | func (n *lruNode) remove() { 31 | if n.prev != nil { 32 | n.prev.next = n.next 33 | n.next.prev = n.prev 34 | n.prev = nil 35 | n.next = nil 36 | } else { 37 | panic("BUG: removing removed node") 38 | } 39 | } 40 | 41 | type lru struct { 42 | mu sync.Mutex 43 | capacity int 44 | used int 45 | recent lruNode 46 | } 47 | 48 | func (r *lru) reset() { 49 | r.recent.next = &r.recent 50 | r.recent.prev = &r.recent 51 | r.used = 0 52 | } 53 | 54 | func (r *lru) Capacity() int { 55 | r.mu.Lock() 56 | defer r.mu.Unlock() 57 | return r.capacity 58 | } 59 | 60 | func (r *lru) SetCapacity(capacity int) { 61 | var evicted []*lruNode 62 | 63 | r.mu.Lock() 64 | r.capacity = capacity 65 | for r.used > r.capacity { 66 | rn := r.recent.prev 67 | if rn == nil { 68 | panic("BUG: invalid LRU used or capacity counter") 69 | } 70 | rn.remove() 71 | rn.n.CacheData = nil 72 | r.used -= rn.n.Size() 73 | evicted = append(evicted, rn) 74 | } 75 | r.mu.Unlock() 76 | 77 | for _, rn := range evicted { 78 | rn.h.Release() 79 | } 80 | } 81 | 82 | func (r *lru) Promote(n *Node) { 83 | var evicted []*lruNode 84 | 85 | r.mu.Lock() 86 | if n.CacheData == nil { 87 | if n.Size() <= r.capacity { 88 | rn := &lruNode{n: n, h: n.GetHandle()} 89 | rn.insert(&r.recent) 90 | n.CacheData = unsafe.Pointer(rn) 91 | r.used += n.Size() 92 | 93 | for r.used > r.capacity { 94 | rn := r.recent.prev 95 | if rn == nil { 96 | panic("BUG: invalid LRU used or capacity counter") 97 | } 98 | rn.remove() 99 | rn.n.CacheData = nil 100 | r.used -= rn.n.Size() 101 | evicted = append(evicted, rn) 102 | } 103 | } 104 | } else { 105 | rn := (*lruNode)(n.CacheData) 106 | if !rn.ban { 107 | rn.remove() 108 | rn.insert(&r.recent) 109 | } 110 | } 111 | r.mu.Unlock() 112 | 113 | for _, rn := range evicted { 114 | rn.h.Release() 115 | } 116 | } 117 | 118 | func (r *lru) Ban(n *Node) { 119 | r.mu.Lock() 120 | if n.CacheData == nil { 121 | n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) 122 | } else { 123 | rn := (*lruNode)(n.CacheData) 124 | if !rn.ban { 125 | rn.remove() 126 | rn.ban = true 127 | r.used -= rn.n.Size() 128 | r.mu.Unlock() 129 | 130 | rn.h.Release() 131 | rn.h = nil 132 | return 133 | } 134 | } 135 | r.mu.Unlock() 136 | } 137 | 138 | func (r *lru) Evict(n *Node) { 139 | r.mu.Lock() 140 | rn := (*lruNode)(n.CacheData) 141 | if rn == nil || rn.ban { 142 | r.mu.Unlock() 143 | return 144 | } 145 | rn.remove() 146 | r.used -= n.Size() 147 | n.CacheData = nil 148 | r.mu.Unlock() 149 | 150 | rn.h.Release() 151 | } 152 | 153 | // NewLRU create a new LRU-cache. 154 | func NewLRU(capacity int) Cacher { 155 | r := &lru{capacity: capacity} 156 | r.reset() 157 | return r 158 | } 159 | -------------------------------------------------------------------------------- /leveldb/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/comparer" 11 | ) 12 | 13 | type iComparer struct { 14 | ucmp comparer.Comparer 15 | } 16 | 17 | func (icmp *iComparer) uName() string { 18 | return icmp.ucmp.Name() 19 | } 20 | 21 | func (icmp *iComparer) uCompare(a, b []byte) int { 22 | return icmp.ucmp.Compare(a, b) 23 | } 24 | 25 | func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte { 26 | return icmp.ucmp.Separator(dst, a, b) 27 | } 28 | 29 | func (icmp *iComparer) uSuccessor(dst, b []byte) []byte { 30 | return icmp.ucmp.Successor(dst, b) 31 | } 32 | 33 | func (icmp *iComparer) Name() string { 34 | return icmp.uName() 35 | } 36 | 37 | func (icmp *iComparer) Compare(a, b []byte) int { 38 | x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) 39 | if x == 0 { 40 | if m, n := internalKey(a).num(), internalKey(b).num(); m > n { 41 | return -1 42 | } else if m < n { 43 | return 1 44 | } 45 | } 46 | return x 47 | } 48 | 49 | func (icmp *iComparer) Separator(dst, a, b []byte) []byte { 50 | ua, ub := internalKey(a).ukey(), internalKey(b).ukey() 51 | dst = icmp.uSeparator(dst, ua, ub) 52 | if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 { 53 | // Append earliest possible number. 54 | return append(dst, keyMaxNumBytes...) 55 | } 56 | return nil 57 | } 58 | 59 | func (icmp *iComparer) Successor(dst, b []byte) []byte { 60 | ub := internalKey(b).ukey() 61 | dst = icmp.uSuccessor(dst, ub) 62 | if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 { 63 | // Append earliest possible number. 64 | return append(dst, keyMaxNumBytes...) 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /leveldb/comparer/bytes_comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package comparer 8 | 9 | import "bytes" 10 | 11 | type bytesComparer struct{} 12 | 13 | func (bytesComparer) Compare(a, b []byte) int { 14 | return bytes.Compare(a, b) 15 | } 16 | 17 | func (bytesComparer) Name() string { 18 | return "leveldb.BytewiseComparator" 19 | } 20 | 21 | func (bytesComparer) Separator(dst, a, b []byte) []byte { 22 | i, n := 0, len(a) 23 | if n > len(b) { 24 | n = len(b) 25 | } 26 | for ; i < n && a[i] == b[i]; i++ { 27 | } 28 | if i >= n { 29 | // Do not shorten if one string is a prefix of the other 30 | } else if c := a[i]; c < 0xff && c+1 < b[i] { 31 | dst = append(dst, a[:i+1]...) 32 | dst[len(dst)-1]++ 33 | return dst 34 | } 35 | return nil 36 | } 37 | 38 | func (bytesComparer) Successor(dst, b []byte) []byte { 39 | for i, c := range b { 40 | if c != 0xff { 41 | dst = append(dst, b[:i+1]...) 42 | dst[len(dst)-1]++ 43 | return dst 44 | } 45 | } 46 | return nil 47 | } 48 | 49 | // DefaultComparer are default implementation of the Comparer interface. 50 | // It uses the natural ordering, consistent with bytes.Compare. 51 | var DefaultComparer = bytesComparer{} 52 | -------------------------------------------------------------------------------- /leveldb/comparer/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package comparer provides interface and implementation for ordering 8 | // sets of data. 9 | package comparer 10 | 11 | // BasicComparer is the interface that wraps the basic Compare method. 12 | type BasicComparer interface { 13 | // Compare returns -1, 0, or +1 depending on whether a is 'less than', 14 | // 'equal to' or 'greater than' b. The two arguments can only be 'equal' 15 | // if their contents are exactly equal. Furthermore, the empty slice 16 | // must be 'less than' any non-empty slice. 17 | Compare(a, b []byte) int 18 | } 19 | 20 | // Comparer defines a total ordering over the space of []byte keys: a 'less 21 | // than' relationship. 22 | type Comparer interface { 23 | BasicComparer 24 | 25 | // Name returns name of the comparer. 26 | // 27 | // The Level-DB on-disk format stores the comparer name, and opening a 28 | // database with a different comparer from the one it was created with 29 | // will result in an error. 30 | // 31 | // An implementation to a new name whenever the comparer implementation 32 | // changes in a way that will cause the relative ordering of any two keys 33 | // to change. 34 | // 35 | // Names starting with "leveldb." are reserved and should not be used 36 | // by any users of this package. 37 | Name() string 38 | 39 | // Bellow are advanced functions used to reduce the space requirements 40 | // for internal data structures such as index blocks. 41 | 42 | // Separator appends a sequence of bytes x to dst such that a <= x && x < b, 43 | // where 'less than' is consistent with Compare. An implementation should 44 | // return nil if x equal to a. 45 | // 46 | // Either contents of a or b should not by any means modified. Doing so 47 | // may cause corruption on the internal state. 48 | Separator(dst, a, b []byte) []byte 49 | 50 | // Successor appends a sequence of bytes x to dst such that x >= b, where 51 | // 'less than' is consistent with Compare. An implementation should return 52 | // nil if x equal to b. 53 | // 54 | // Contents of b should not by any means modified. Doing so may cause 55 | // corruption on the internal state. 56 | Successor(dst, b []byte) []byte 57 | } 58 | -------------------------------------------------------------------------------- /leveldb/db_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "math/rand" 11 | "runtime" 12 | "sync" 13 | "sync/atomic" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/iterator" 16 | "github.com/syndtr/goleveldb/leveldb/opt" 17 | "github.com/syndtr/goleveldb/leveldb/util" 18 | ) 19 | 20 | type memdbReleaser struct { 21 | once sync.Once 22 | m *memDB 23 | } 24 | 25 | func (mr *memdbReleaser) Release() { 26 | mr.once.Do(func() { 27 | mr.m.decref() 28 | }) 29 | } 30 | 31 | func (db *DB) newRawIterator(auxm *memDB, auxt tFiles, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 32 | strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader) 33 | em, fm := db.getMems() 34 | v := db.s.version() 35 | 36 | tableIts := v.getIterators(slice, ro) 37 | n := len(tableIts) + len(auxt) + 3 38 | its := make([]iterator.Iterator, 0, n) 39 | 40 | if auxm != nil { 41 | ami := auxm.NewIterator(slice) 42 | ami.SetReleaser(&memdbReleaser{m: auxm}) 43 | its = append(its, ami) 44 | } 45 | for _, t := range auxt { 46 | its = append(its, v.s.tops.newIterator(t, slice, ro)) 47 | } 48 | 49 | emi := em.NewIterator(slice) 50 | emi.SetReleaser(&memdbReleaser{m: em}) 51 | its = append(its, emi) 52 | if fm != nil { 53 | fmi := fm.NewIterator(slice) 54 | fmi.SetReleaser(&memdbReleaser{m: fm}) 55 | its = append(its, fmi) 56 | } 57 | its = append(its, tableIts...) 58 | mi := iterator.NewMergedIterator(its, db.s.icmp, strict) 59 | mi.SetReleaser(&versionReleaser{v: v}) 60 | return mi 61 | } 62 | 63 | func (db *DB) newIterator(auxm *memDB, auxt tFiles, seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter { 64 | var islice *util.Range 65 | if slice != nil { 66 | islice = &util.Range{} 67 | if slice.Start != nil { 68 | islice.Start = makeInternalKey(nil, slice.Start, keyMaxSeq, keyTypeSeek) 69 | } 70 | if slice.Limit != nil { 71 | islice.Limit = makeInternalKey(nil, slice.Limit, keyMaxSeq, keyTypeSeek) 72 | } 73 | } 74 | rawIter := db.newRawIterator(auxm, auxt, islice, ro) 75 | iter := &dbIter{ 76 | db: db, 77 | icmp: db.s.icmp, 78 | iter: rawIter, 79 | seq: seq, 80 | strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader), 81 | disableSampling: db.s.o.GetDisableSeeksCompaction() || db.s.o.GetIteratorSamplingRate() <= 0, 82 | key: make([]byte, 0), 83 | value: make([]byte, 0), 84 | } 85 | if !iter.disableSampling { 86 | iter.samplingGap = db.iterSamplingRate() 87 | } 88 | atomic.AddInt32(&db.aliveIters, 1) 89 | runtime.SetFinalizer(iter, (*dbIter).Release) 90 | return iter 91 | } 92 | 93 | func (db *DB) iterSamplingRate() int { 94 | return rand.Intn(2 * db.s.o.GetIteratorSamplingRate()) 95 | } 96 | 97 | type dir int 98 | 99 | const ( 100 | dirReleased dir = iota - 1 101 | dirSOI 102 | dirEOI 103 | dirBackward 104 | dirForward 105 | ) 106 | 107 | // dbIter represent an interator states over a database session. 108 | type dbIter struct { 109 | db *DB 110 | icmp *iComparer 111 | iter iterator.Iterator 112 | seq uint64 113 | strict bool 114 | disableSampling bool 115 | 116 | samplingGap int 117 | dir dir 118 | key []byte 119 | value []byte 120 | err error 121 | releaser util.Releaser 122 | } 123 | 124 | func (i *dbIter) sampleSeek() { 125 | if i.disableSampling { 126 | return 127 | } 128 | 129 | ikey := i.iter.Key() 130 | i.samplingGap -= len(ikey) + len(i.iter.Value()) 131 | for i.samplingGap < 0 { 132 | i.samplingGap += i.db.iterSamplingRate() 133 | i.db.sampleSeek(ikey) 134 | } 135 | } 136 | 137 | func (i *dbIter) setErr(err error) { 138 | i.err = err 139 | i.key = nil 140 | i.value = nil 141 | } 142 | 143 | func (i *dbIter) iterErr() { 144 | if err := i.iter.Error(); err != nil { 145 | i.setErr(err) 146 | } 147 | } 148 | 149 | func (i *dbIter) Valid() bool { 150 | return i.err == nil && i.dir > dirEOI 151 | } 152 | 153 | func (i *dbIter) First() bool { 154 | if i.err != nil { 155 | return false 156 | } else if i.dir == dirReleased { 157 | i.err = ErrIterReleased 158 | return false 159 | } 160 | 161 | if i.iter.First() { 162 | i.dir = dirSOI 163 | return i.next() 164 | } 165 | i.dir = dirEOI 166 | i.iterErr() 167 | return false 168 | } 169 | 170 | func (i *dbIter) Last() bool { 171 | if i.err != nil { 172 | return false 173 | } else if i.dir == dirReleased { 174 | i.err = ErrIterReleased 175 | return false 176 | } 177 | 178 | if i.iter.Last() { 179 | return i.prev() 180 | } 181 | i.dir = dirSOI 182 | i.iterErr() 183 | return false 184 | } 185 | 186 | func (i *dbIter) Seek(key []byte) bool { 187 | if i.err != nil { 188 | return false 189 | } else if i.dir == dirReleased { 190 | i.err = ErrIterReleased 191 | return false 192 | } 193 | 194 | ikey := makeInternalKey(nil, key, i.seq, keyTypeSeek) 195 | if i.iter.Seek(ikey) { 196 | i.dir = dirSOI 197 | return i.next() 198 | } 199 | i.dir = dirEOI 200 | i.iterErr() 201 | return false 202 | } 203 | 204 | func (i *dbIter) next() bool { 205 | for { 206 | if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 207 | i.sampleSeek() 208 | if seq <= i.seq { 209 | switch kt { 210 | case keyTypeDel: 211 | // Skip deleted key. 212 | i.key = append(i.key[:0], ukey...) 213 | i.dir = dirForward 214 | case keyTypeVal: 215 | if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 { 216 | i.key = append(i.key[:0], ukey...) 217 | i.value = append(i.value[:0], i.iter.Value()...) 218 | i.dir = dirForward 219 | return true 220 | } 221 | } 222 | } 223 | } else if i.strict { 224 | i.setErr(kerr) 225 | break 226 | } 227 | if !i.iter.Next() { 228 | i.dir = dirEOI 229 | i.iterErr() 230 | break 231 | } 232 | } 233 | return false 234 | } 235 | 236 | func (i *dbIter) Next() bool { 237 | if i.dir == dirEOI || i.err != nil { 238 | return false 239 | } else if i.dir == dirReleased { 240 | i.err = ErrIterReleased 241 | return false 242 | } 243 | 244 | if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) { 245 | i.dir = dirEOI 246 | i.iterErr() 247 | return false 248 | } 249 | return i.next() 250 | } 251 | 252 | func (i *dbIter) prev() bool { 253 | i.dir = dirBackward 254 | del := true 255 | if i.iter.Valid() { 256 | for { 257 | if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 258 | i.sampleSeek() 259 | if seq <= i.seq { 260 | if !del && i.icmp.uCompare(ukey, i.key) < 0 { 261 | return true 262 | } 263 | del = (kt == keyTypeDel) 264 | if !del { 265 | i.key = append(i.key[:0], ukey...) 266 | i.value = append(i.value[:0], i.iter.Value()...) 267 | } 268 | } 269 | } else if i.strict { 270 | i.setErr(kerr) 271 | return false 272 | } 273 | if !i.iter.Prev() { 274 | break 275 | } 276 | } 277 | } 278 | if del { 279 | i.dir = dirSOI 280 | i.iterErr() 281 | return false 282 | } 283 | return true 284 | } 285 | 286 | func (i *dbIter) Prev() bool { 287 | if i.dir == dirSOI || i.err != nil { 288 | return false 289 | } else if i.dir == dirReleased { 290 | i.err = ErrIterReleased 291 | return false 292 | } 293 | 294 | switch i.dir { 295 | case dirEOI: 296 | return i.Last() 297 | case dirForward: 298 | for i.iter.Prev() { 299 | if ukey, _, _, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 300 | i.sampleSeek() 301 | if i.icmp.uCompare(ukey, i.key) < 0 { 302 | goto cont 303 | } 304 | } else if i.strict { 305 | i.setErr(kerr) 306 | return false 307 | } 308 | } 309 | i.dir = dirSOI 310 | i.iterErr() 311 | return false 312 | } 313 | 314 | cont: 315 | return i.prev() 316 | } 317 | 318 | func (i *dbIter) Key() []byte { 319 | if i.err != nil || i.dir <= dirEOI { 320 | return nil 321 | } 322 | return i.key 323 | } 324 | 325 | func (i *dbIter) Value() []byte { 326 | if i.err != nil || i.dir <= dirEOI { 327 | return nil 328 | } 329 | return i.value 330 | } 331 | 332 | func (i *dbIter) Release() { 333 | if i.dir != dirReleased { 334 | // Clear the finalizer. 335 | runtime.SetFinalizer(i, nil) 336 | 337 | if i.releaser != nil { 338 | i.releaser.Release() 339 | i.releaser = nil 340 | } 341 | 342 | i.dir = dirReleased 343 | i.key = nil 344 | i.value = nil 345 | i.iter.Release() 346 | i.iter = nil 347 | atomic.AddInt32(&i.db.aliveIters, -1) 348 | i.db = nil 349 | } 350 | } 351 | 352 | func (i *dbIter) SetReleaser(releaser util.Releaser) { 353 | if i.dir == dirReleased { 354 | panic(util.ErrReleased) 355 | } 356 | if i.releaser != nil && releaser != nil { 357 | panic(util.ErrHasReleaser) 358 | } 359 | i.releaser = releaser 360 | } 361 | 362 | func (i *dbIter) Error() error { 363 | return i.err 364 | } 365 | -------------------------------------------------------------------------------- /leveldb/db_snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "container/list" 11 | "fmt" 12 | "runtime" 13 | "sync" 14 | "sync/atomic" 15 | 16 | "github.com/syndtr/goleveldb/leveldb/iterator" 17 | "github.com/syndtr/goleveldb/leveldb/opt" 18 | "github.com/syndtr/goleveldb/leveldb/util" 19 | ) 20 | 21 | type snapshotElement struct { 22 | seq uint64 23 | ref int 24 | e *list.Element 25 | } 26 | 27 | // Acquires a snapshot, based on latest sequence. 28 | func (db *DB) acquireSnapshot() *snapshotElement { 29 | db.snapsMu.Lock() 30 | defer db.snapsMu.Unlock() 31 | 32 | seq := db.getSeq() 33 | 34 | if e := db.snapsList.Back(); e != nil { 35 | se := e.Value.(*snapshotElement) 36 | if se.seq == seq { 37 | se.ref++ 38 | return se 39 | } else if seq < se.seq { 40 | panic("leveldb: sequence number is not increasing") 41 | } 42 | } 43 | se := &snapshotElement{seq: seq, ref: 1} 44 | se.e = db.snapsList.PushBack(se) 45 | return se 46 | } 47 | 48 | // Releases given snapshot element. 49 | func (db *DB) releaseSnapshot(se *snapshotElement) { 50 | db.snapsMu.Lock() 51 | defer db.snapsMu.Unlock() 52 | 53 | se.ref-- 54 | if se.ref == 0 { 55 | db.snapsList.Remove(se.e) 56 | se.e = nil 57 | } else if se.ref < 0 { 58 | panic("leveldb: Snapshot: negative element reference") 59 | } 60 | } 61 | 62 | // Gets minimum sequence that not being snapshotted. 63 | func (db *DB) minSeq() uint64 { 64 | db.snapsMu.Lock() 65 | defer db.snapsMu.Unlock() 66 | 67 | if e := db.snapsList.Front(); e != nil { 68 | return e.Value.(*snapshotElement).seq 69 | } 70 | 71 | return db.getSeq() 72 | } 73 | 74 | // Snapshot is a DB snapshot. 75 | type Snapshot struct { 76 | db *DB 77 | elem *snapshotElement 78 | mu sync.RWMutex 79 | released bool 80 | } 81 | 82 | // Creates new snapshot object. 83 | func (db *DB) newSnapshot() *Snapshot { 84 | snap := &Snapshot{ 85 | db: db, 86 | elem: db.acquireSnapshot(), 87 | } 88 | atomic.AddInt32(&db.aliveSnaps, 1) 89 | runtime.SetFinalizer(snap, (*Snapshot).Release) 90 | return snap 91 | } 92 | 93 | func (snap *Snapshot) String() string { 94 | return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq) 95 | } 96 | 97 | // Get gets the value for the given key. It returns ErrNotFound if 98 | // the DB does not contains the key. 99 | // 100 | // The caller should not modify the contents of the returned slice, but 101 | // it is safe to modify the contents of the argument after Get returns. 102 | func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { 103 | snap.mu.RLock() 104 | defer snap.mu.RUnlock() 105 | if snap.released { 106 | err = ErrSnapshotReleased 107 | return 108 | } 109 | err = snap.db.ok() 110 | if err != nil { 111 | return 112 | } 113 | return snap.db.get(nil, nil, key, snap.elem.seq, ro) 114 | } 115 | 116 | // Has returns true if the DB does contains the given key. 117 | // 118 | // It is safe to modify the contents of the argument after Get returns. 119 | func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { 120 | snap.mu.RLock() 121 | defer snap.mu.RUnlock() 122 | if snap.released { 123 | err = ErrSnapshotReleased 124 | return 125 | } 126 | err = snap.db.ok() 127 | if err != nil { 128 | return 129 | } 130 | return snap.db.has(nil, nil, key, snap.elem.seq, ro) 131 | } 132 | 133 | // NewIterator returns an iterator for the snapshot of the underlying DB. 134 | // The returned iterator is not safe for concurrent use, but it is safe to use 135 | // multiple iterators concurrently, with each in a dedicated goroutine. 136 | // It is also safe to use an iterator concurrently with modifying its 137 | // underlying DB. The resultant key/value pairs are guaranteed to be 138 | // consistent. 139 | // 140 | // Slice allows slicing the iterator to only contains keys in the given 141 | // range. A nil Range.Start is treated as a key before all keys in the 142 | // DB. And a nil Range.Limit is treated as a key after all keys in 143 | // the DB. 144 | // 145 | // WARNING: Any slice returned by interator (e.g. slice returned by calling 146 | // Iterator.Key() or Iterator.Value() methods), its content should not be 147 | // modified unless noted otherwise. 148 | // 149 | // The iterator must be released after use, by calling Release method. 150 | // Releasing the snapshot doesn't mean releasing the iterator too, the 151 | // iterator would be still valid until released. 152 | // 153 | // Also read Iterator documentation of the leveldb/iterator package. 154 | func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 155 | snap.mu.Lock() 156 | defer snap.mu.Unlock() 157 | if snap.released { 158 | return iterator.NewEmptyIterator(ErrSnapshotReleased) 159 | } 160 | if err := snap.db.ok(); err != nil { 161 | return iterator.NewEmptyIterator(err) 162 | } 163 | // Since iterator already hold version ref, it doesn't need to 164 | // hold snapshot ref. 165 | return snap.db.newIterator(nil, nil, snap.elem.seq, slice, ro) 166 | } 167 | 168 | // Release releases the snapshot. This will not release any returned 169 | // iterators, the iterators would still be valid until released or the 170 | // underlying DB is closed. 171 | // 172 | // Other methods should not be called after the snapshot has been released. 173 | func (snap *Snapshot) Release() { 174 | snap.mu.Lock() 175 | defer snap.mu.Unlock() 176 | 177 | if !snap.released { 178 | // Clear the finalizer. 179 | runtime.SetFinalizer(snap, nil) 180 | 181 | snap.released = true 182 | snap.db.releaseSnapshot(snap.elem) 183 | atomic.AddInt32(&snap.db.aliveSnaps, -1) 184 | snap.db = nil 185 | snap.elem = nil 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /leveldb/db_state.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "errors" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/journal" 15 | "github.com/syndtr/goleveldb/leveldb/memdb" 16 | "github.com/syndtr/goleveldb/leveldb/storage" 17 | ) 18 | 19 | var ( 20 | errHasFrozenMem = errors.New("has frozen mem") 21 | ) 22 | 23 | type memDB struct { 24 | db *DB 25 | *memdb.DB 26 | ref int32 27 | } 28 | 29 | func (m *memDB) getref() int32 { 30 | return atomic.LoadInt32(&m.ref) 31 | } 32 | 33 | func (m *memDB) incref() { 34 | atomic.AddInt32(&m.ref, 1) 35 | } 36 | 37 | func (m *memDB) decref() { 38 | if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { 39 | // Only put back memdb with std capacity. 40 | if m.Capacity() == m.db.s.o.GetWriteBuffer() { 41 | m.Reset() 42 | m.db.mpoolPut(m.DB) 43 | } 44 | m.db = nil 45 | m.DB = nil 46 | } else if ref < 0 { 47 | panic("negative memdb ref") 48 | } 49 | } 50 | 51 | // Get latest sequence number. 52 | func (db *DB) getSeq() uint64 { 53 | return atomic.LoadUint64(&db.seq) 54 | } 55 | 56 | // Atomically adds delta to seq. 57 | func (db *DB) addSeq(delta uint64) { 58 | atomic.AddUint64(&db.seq, delta) 59 | } 60 | 61 | func (db *DB) setSeq(seq uint64) { 62 | atomic.StoreUint64(&db.seq, seq) 63 | } 64 | 65 | func (db *DB) sampleSeek(ikey internalKey) { 66 | v := db.s.version() 67 | if v.sampleSeek(ikey) { 68 | // Trigger table compaction. 69 | db.compTrigger(db.tcompCmdC) 70 | } 71 | v.release() 72 | } 73 | 74 | func (db *DB) mpoolPut(mem *memdb.DB) { 75 | if !db.isClosed() { 76 | select { 77 | case db.memPool <- mem: 78 | default: 79 | } 80 | } 81 | } 82 | 83 | func (db *DB) mpoolGet(n int) *memDB { 84 | var mdb *memdb.DB 85 | select { 86 | case mdb = <-db.memPool: 87 | default: 88 | } 89 | if mdb == nil || mdb.Capacity() < n { 90 | mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)) 91 | } 92 | return &memDB{ 93 | db: db, 94 | DB: mdb, 95 | } 96 | } 97 | 98 | func (db *DB) mpoolDrain() { 99 | ticker := time.NewTicker(30 * time.Second) 100 | for { 101 | select { 102 | case <-ticker.C: 103 | select { 104 | case <-db.memPool: 105 | default: 106 | } 107 | case <-db.closeC: 108 | ticker.Stop() 109 | // Make sure the pool is drained. 110 | select { 111 | case <-db.memPool: 112 | case <-time.After(time.Second): 113 | } 114 | close(db.memPool) 115 | return 116 | } 117 | } 118 | } 119 | 120 | // Create new memdb and froze the old one; need external synchronization. 121 | // newMem only called synchronously by the writer. 122 | func (db *DB) newMem(n int) (mem *memDB, err error) { 123 | fd := storage.FileDesc{Type: storage.TypeJournal, Num: db.s.allocFileNum()} 124 | w, err := db.s.stor.Create(fd) 125 | if err != nil { 126 | db.s.reuseFileNum(fd.Num) 127 | return 128 | } 129 | 130 | db.memMu.Lock() 131 | defer db.memMu.Unlock() 132 | 133 | if db.frozenMem != nil { 134 | return nil, errHasFrozenMem 135 | } 136 | 137 | if db.journal == nil { 138 | db.journal = journal.NewWriter(w) 139 | } else { 140 | if err := db.journal.Reset(w); err != nil { 141 | return nil, err 142 | } 143 | if err := db.journalWriter.Close(); err != nil { 144 | return nil, err 145 | } 146 | db.frozenJournalFd = db.journalFd 147 | } 148 | db.journalWriter = w 149 | db.journalFd = fd 150 | db.frozenMem = db.mem 151 | mem = db.mpoolGet(n) 152 | mem.incref() // for self 153 | mem.incref() // for caller 154 | db.mem = mem 155 | // The seq only incremented by the writer. And whoever called newMem 156 | // should hold write lock, so no need additional synchronization here. 157 | db.frozenSeq = db.seq 158 | return 159 | } 160 | 161 | // Get all memdbs. 162 | func (db *DB) getMems() (e, f *memDB) { 163 | db.memMu.RLock() 164 | defer db.memMu.RUnlock() 165 | if db.mem != nil { 166 | db.mem.incref() 167 | } else if !db.isClosed() { 168 | panic("nil effective mem") 169 | } 170 | if db.frozenMem != nil { 171 | db.frozenMem.incref() 172 | } 173 | return db.mem, db.frozenMem 174 | } 175 | 176 | // Get effective memdb. 177 | func (db *DB) getEffectiveMem() *memDB { 178 | db.memMu.RLock() 179 | defer db.memMu.RUnlock() 180 | if db.mem != nil { 181 | db.mem.incref() 182 | } else if !db.isClosed() { 183 | panic("nil effective mem") 184 | } 185 | return db.mem 186 | } 187 | 188 | // Get frozen memdb. 189 | func (db *DB) getFrozenMem() *memDB { 190 | db.memMu.RLock() 191 | defer db.memMu.RUnlock() 192 | if db.frozenMem != nil { 193 | db.frozenMem.incref() 194 | } 195 | return db.frozenMem 196 | } 197 | 198 | // Drop frozen memdb; assume that frozen memdb isn't nil. 199 | func (db *DB) dropFrozenMem() { 200 | db.memMu.Lock() 201 | if err := db.s.stor.Remove(db.frozenJournalFd); err != nil { 202 | db.logf("journal@remove removing @%d %q", db.frozenJournalFd.Num, err) 203 | } else { 204 | db.logf("journal@remove removed @%d", db.frozenJournalFd.Num) 205 | } 206 | db.frozenJournalFd = storage.FileDesc{} 207 | db.frozenMem.decref() 208 | db.frozenMem = nil 209 | db.memMu.Unlock() 210 | } 211 | 212 | // Clear mems ptr; used by DB.Close(). 213 | func (db *DB) clearMems() { 214 | db.memMu.Lock() 215 | db.mem = nil 216 | db.frozenMem = nil 217 | db.memMu.Unlock() 218 | } 219 | 220 | // Set closed flag; return true if not already closed. 221 | func (db *DB) setClosed() bool { 222 | return atomic.CompareAndSwapUint32(&db.closed, 0, 1) 223 | } 224 | 225 | // Check whether DB was closed. 226 | func (db *DB) isClosed() bool { 227 | return atomic.LoadUint32(&db.closed) != 0 228 | } 229 | 230 | // Check read ok status. 231 | func (db *DB) ok() error { 232 | if db.isClosed() { 233 | return ErrClosed 234 | } 235 | return nil 236 | } 237 | -------------------------------------------------------------------------------- /leveldb/db_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | "github.com/syndtr/goleveldb/leveldb/iterator" 12 | "github.com/syndtr/goleveldb/leveldb/opt" 13 | "github.com/syndtr/goleveldb/leveldb/storage" 14 | "github.com/syndtr/goleveldb/leveldb/util" 15 | ) 16 | 17 | // Reader is the interface that wraps basic Get and NewIterator methods. 18 | // This interface implemented by both DB and Snapshot. 19 | type Reader interface { 20 | Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) 21 | NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator 22 | } 23 | 24 | // Sizes is list of size. 25 | type Sizes []int64 26 | 27 | // Sum returns sum of the sizes. 28 | func (sizes Sizes) Sum() int64 { 29 | var sum int64 30 | for _, size := range sizes { 31 | sum += size 32 | } 33 | return sum 34 | } 35 | 36 | // Logging. 37 | func (db *DB) log(v ...interface{}) { db.s.log(v...) } 38 | func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) } 39 | 40 | // Check and clean files. 41 | func (db *DB) checkAndCleanFiles() error { 42 | v := db.s.version() 43 | defer v.release() 44 | 45 | tmap := make(map[int64]bool) 46 | for _, tables := range v.levels { 47 | for _, t := range tables { 48 | tmap[t.fd.Num] = false 49 | } 50 | } 51 | 52 | fds, err := db.s.stor.List(storage.TypeAll) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | var nt int 58 | var rem []storage.FileDesc 59 | for _, fd := range fds { 60 | keep := true 61 | switch fd.Type { 62 | case storage.TypeManifest: 63 | keep = fd.Num >= db.s.manifestFd.Num 64 | case storage.TypeJournal: 65 | if !db.frozenJournalFd.Zero() { 66 | keep = fd.Num >= db.frozenJournalFd.Num 67 | } else { 68 | keep = fd.Num >= db.journalFd.Num 69 | } 70 | case storage.TypeTable: 71 | _, keep = tmap[fd.Num] 72 | if keep { 73 | tmap[fd.Num] = true 74 | nt++ 75 | } 76 | } 77 | 78 | if !keep { 79 | rem = append(rem, fd) 80 | } 81 | } 82 | 83 | if nt != len(tmap) { 84 | var mfds []storage.FileDesc 85 | for num, present := range tmap { 86 | if !present { 87 | mfds = append(mfds, storage.FileDesc{Type: storage.TypeTable, Num: num}) 88 | db.logf("db@janitor table missing @%d", num) 89 | } 90 | } 91 | return errors.NewErrCorrupted(storage.FileDesc{}, &errors.ErrMissingFiles{Fds: mfds}) 92 | } 93 | 94 | db.logf("db@janitor F·%d G·%d", len(fds), len(rem)) 95 | for _, fd := range rem { 96 | db.logf("db@janitor removing %s-%d", fd.Type, fd.Num) 97 | if err := db.s.stor.Remove(fd); err != nil { 98 | return err 99 | } 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /leveldb/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package leveldb provides implementation of LevelDB key/value database. 8 | // 9 | // Create or open a database: 10 | // 11 | // // The returned DB instance is safe for concurrent use. Which mean that all 12 | // // DB's methods may be called concurrently from multiple goroutine. 13 | // db, err := leveldb.OpenFile("path/to/db", nil) 14 | // ... 15 | // defer db.Close() 16 | // ... 17 | // 18 | // Read or modify the database content: 19 | // 20 | // // Remember that the contents of the returned slice should not be modified. 21 | // data, err := db.Get([]byte("key"), nil) 22 | // ... 23 | // err = db.Put([]byte("key"), []byte("value"), nil) 24 | // ... 25 | // err = db.Delete([]byte("key"), nil) 26 | // ... 27 | // 28 | // Iterate over database content: 29 | // 30 | // iter := db.NewIterator(nil, nil) 31 | // for iter.Next() { 32 | // // Remember that the contents of the returned slice should not be modified, and 33 | // // only valid until the next call to Next. 34 | // key := iter.Key() 35 | // value := iter.Value() 36 | // ... 37 | // } 38 | // iter.Release() 39 | // err = iter.Error() 40 | // ... 41 | // 42 | // Iterate over subset of database content with a particular prefix: 43 | // iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) 44 | // for iter.Next() { 45 | // // Use key/value. 46 | // ... 47 | // } 48 | // iter.Release() 49 | // err = iter.Error() 50 | // ... 51 | // 52 | // Seek-then-Iterate: 53 | // 54 | // iter := db.NewIterator(nil, nil) 55 | // for ok := iter.Seek(key); ok; ok = iter.Next() { 56 | // // Use key/value. 57 | // ... 58 | // } 59 | // iter.Release() 60 | // err = iter.Error() 61 | // ... 62 | // 63 | // Iterate over subset of database content: 64 | // 65 | // iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) 66 | // for iter.Next() { 67 | // // Use key/value. 68 | // ... 69 | // } 70 | // iter.Release() 71 | // err = iter.Error() 72 | // ... 73 | // 74 | // Batch writes: 75 | // 76 | // batch := new(leveldb.Batch) 77 | // batch.Put([]byte("foo"), []byte("value")) 78 | // batch.Put([]byte("bar"), []byte("another value")) 79 | // batch.Delete([]byte("baz")) 80 | // err = db.Write(batch, nil) 81 | // ... 82 | // 83 | // Use bloom filter: 84 | // 85 | // o := &opt.Options{ 86 | // Filter: filter.NewBloomFilter(10), 87 | // } 88 | // db, err := leveldb.OpenFile("path/to/db", o) 89 | // ... 90 | // defer db.Close() 91 | // ... 92 | package leveldb 93 | -------------------------------------------------------------------------------- /leveldb/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | ) 12 | 13 | // Common errors. 14 | var ( 15 | ErrNotFound = errors.ErrNotFound 16 | ErrReadOnly = errors.New("leveldb: read-only mode") 17 | ErrSnapshotReleased = errors.New("leveldb: snapshot released") 18 | ErrIterReleased = errors.New("leveldb: iterator released") 19 | ErrClosed = errors.New("leveldb: closed") 20 | ) 21 | -------------------------------------------------------------------------------- /leveldb/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package errors provides common error types used throughout leveldb. 8 | package errors 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/storage" 15 | "github.com/syndtr/goleveldb/leveldb/util" 16 | ) 17 | 18 | // Common errors. 19 | var ( 20 | ErrNotFound = New("leveldb: not found") 21 | ErrReleased = util.ErrReleased 22 | ErrHasReleaser = util.ErrHasReleaser 23 | ) 24 | 25 | // New returns an error that formats as the given text. 26 | func New(text string) error { 27 | return errors.New(text) 28 | } 29 | 30 | // ErrCorrupted is the type that wraps errors that indicate corruption in 31 | // the database. 32 | type ErrCorrupted struct { 33 | Fd storage.FileDesc 34 | Err error 35 | } 36 | 37 | func (e *ErrCorrupted) Error() string { 38 | if !e.Fd.Zero() { 39 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 40 | } 41 | return e.Err.Error() 42 | } 43 | 44 | // NewErrCorrupted creates new ErrCorrupted error. 45 | func NewErrCorrupted(fd storage.FileDesc, err error) error { 46 | return &ErrCorrupted{fd, err} 47 | } 48 | 49 | // IsCorrupted returns a boolean indicating whether the error is indicating 50 | // a corruption. 51 | func IsCorrupted(err error) bool { 52 | switch err.(type) { 53 | case *ErrCorrupted: 54 | return true 55 | case *storage.ErrCorrupted: 56 | return true 57 | } 58 | return false 59 | } 60 | 61 | // ErrMissingFiles is the type that indicating a corruption due to missing 62 | // files. ErrMissingFiles always wrapped with ErrCorrupted. 63 | type ErrMissingFiles struct { 64 | Fds []storage.FileDesc 65 | } 66 | 67 | func (e *ErrMissingFiles) Error() string { return "file missing" } 68 | 69 | // SetFd sets 'file info' of the given error with the given file. 70 | // Currently only ErrCorrupted is supported, otherwise will do nothing. 71 | func SetFd(err error, fd storage.FileDesc) error { 72 | switch x := err.(type) { 73 | case *ErrCorrupted: 74 | x.Fd = fd 75 | return x 76 | default: 77 | return err 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /leveldb/external_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/opt" 14 | "github.com/syndtr/goleveldb/leveldb/testutil" 15 | ) 16 | 17 | var _ = testutil.Defer(func() { 18 | Describe("Leveldb external", func() { 19 | o := &opt.Options{ 20 | DisableBlockCache: true, 21 | BlockRestartInterval: 5, 22 | BlockSize: 80, 23 | Compression: opt.NoCompression, 24 | OpenFilesCacheCapacity: -1, 25 | Strict: opt.StrictAll, 26 | WriteBuffer: 1000, 27 | CompactionTableSize: 2000, 28 | } 29 | 30 | Describe("write test", func() { 31 | It("should do write correctly", func(done Done) { 32 | db := newTestingDB(o, nil, nil) 33 | t := testutil.DBTesting{ 34 | DB: db, 35 | Deleted: testutil.KeyValue_Generate(nil, 500, 1, 1, 50, 5, 5).Clone(), 36 | } 37 | testutil.DoDBTesting(&t) 38 | db.TestClose() 39 | done <- true 40 | }, 80.0) 41 | }) 42 | 43 | Describe("read test", func() { 44 | testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB { 45 | // Building the DB. 46 | db := newTestingDB(o, nil, nil) 47 | kv.IterateShuffled(nil, func(i int, key, value []byte) { 48 | err := db.TestPut(key, value) 49 | Expect(err).NotTo(HaveOccurred()) 50 | }) 51 | 52 | return db 53 | }, func(db testutil.DB) { 54 | db.(*testingDB).TestClose() 55 | }) 56 | }) 57 | 58 | Describe("transaction test", func() { 59 | It("should do transaction correctly", func(done Done) { 60 | db := newTestingDB(o, nil, nil) 61 | 62 | By("creating first transaction") 63 | var err error 64 | tr := &testingTransaction{} 65 | tr.Transaction, err = db.OpenTransaction() 66 | Expect(err).NotTo(HaveOccurred()) 67 | t0 := &testutil.DBTesting{ 68 | DB: tr, 69 | Deleted: testutil.KeyValue_Generate(nil, 200, 1, 1, 50, 5, 5).Clone(), 70 | } 71 | testutil.DoDBTesting(t0) 72 | testutil.TestGet(tr, t0.Present) 73 | testutil.TestHas(tr, t0.Present) 74 | 75 | By("committing first transaction") 76 | err = tr.Commit() 77 | Expect(err).NotTo(HaveOccurred()) 78 | testutil.TestIter(db, nil, t0.Present) 79 | testutil.TestGet(db, t0.Present) 80 | testutil.TestHas(db, t0.Present) 81 | 82 | By("manipulating DB without transaction") 83 | t0.DB = db 84 | testutil.DoDBTesting(t0) 85 | 86 | By("creating second transaction") 87 | tr.Transaction, err = db.OpenTransaction() 88 | Expect(err).NotTo(HaveOccurred()) 89 | t1 := &testutil.DBTesting{ 90 | DB: tr, 91 | Deleted: t0.Deleted.Clone(), 92 | Present: t0.Present.Clone(), 93 | } 94 | testutil.DoDBTesting(t1) 95 | testutil.TestIter(db, nil, t0.Present) 96 | 97 | By("discarding second transaction") 98 | tr.Discard() 99 | testutil.TestIter(db, nil, t0.Present) 100 | 101 | By("creating third transaction") 102 | tr.Transaction, err = db.OpenTransaction() 103 | Expect(err).NotTo(HaveOccurred()) 104 | t0.DB = tr 105 | testutil.DoDBTesting(t0) 106 | 107 | By("committing third transaction") 108 | err = tr.Commit() 109 | Expect(err).NotTo(HaveOccurred()) 110 | testutil.TestIter(db, nil, t0.Present) 111 | 112 | db.TestClose() 113 | done <- true 114 | }, 240.0) 115 | }) 116 | }) 117 | }) 118 | -------------------------------------------------------------------------------- /leveldb/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/filter" 11 | ) 12 | 13 | type iFilter struct { 14 | filter.Filter 15 | } 16 | 17 | func (f iFilter) Contains(filter, key []byte) bool { 18 | return f.Filter.Contains(filter, internalKey(key).ukey()) 19 | } 20 | 21 | func (f iFilter) NewGenerator() filter.FilterGenerator { 22 | return iFilterGenerator{f.Filter.NewGenerator()} 23 | } 24 | 25 | type iFilterGenerator struct { 26 | filter.FilterGenerator 27 | } 28 | 29 | func (g iFilterGenerator) Add(key []byte) { 30 | g.FilterGenerator.Add(internalKey(key).ukey()) 31 | } 32 | -------------------------------------------------------------------------------- /leveldb/filter/bloom.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package filter 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/util" 11 | ) 12 | 13 | func bloomHash(key []byte) uint32 { 14 | return util.Hash(key, 0xbc9f1d34) 15 | } 16 | 17 | type bloomFilter int 18 | 19 | // Name: The bloom filter serializes its parameters and is backward compatible 20 | // with respect to them. Therefor, its parameters are not added to its 21 | // name. 22 | func (bloomFilter) Name() string { 23 | return "leveldb.BuiltinBloomFilter" 24 | } 25 | 26 | func (f bloomFilter) Contains(filter, key []byte) bool { 27 | nBytes := len(filter) - 1 28 | if nBytes < 1 { 29 | return false 30 | } 31 | nBits := uint32(nBytes * 8) 32 | 33 | // Use the encoded k so that we can read filters generated by 34 | // bloom filters created using different parameters. 35 | k := filter[nBytes] 36 | if k > 30 { 37 | // Reserved for potentially new encodings for short bloom filters. 38 | // Consider it a match. 39 | return true 40 | } 41 | 42 | kh := bloomHash(key) 43 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 44 | for j := uint8(0); j < k; j++ { 45 | bitpos := kh % nBits 46 | if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 { 47 | return false 48 | } 49 | kh += delta 50 | } 51 | return true 52 | } 53 | 54 | func (f bloomFilter) NewGenerator() FilterGenerator { 55 | // Round down to reduce probing cost a little bit. 56 | k := uint8(f * 69 / 100) // 0.69 =~ ln(2) 57 | if k < 1 { 58 | k = 1 59 | } else if k > 30 { 60 | k = 30 61 | } 62 | return &bloomFilterGenerator{ 63 | n: int(f), 64 | k: k, 65 | } 66 | } 67 | 68 | type bloomFilterGenerator struct { 69 | n int 70 | k uint8 71 | 72 | keyHashes []uint32 73 | } 74 | 75 | func (g *bloomFilterGenerator) Add(key []byte) { 76 | // Use double-hashing to generate a sequence of hash values. 77 | // See analysis in [Kirsch,Mitzenmacher 2006]. 78 | g.keyHashes = append(g.keyHashes, bloomHash(key)) 79 | } 80 | 81 | func (g *bloomFilterGenerator) Generate(b Buffer) { 82 | // Compute bloom filter size (in both bits and bytes) 83 | nBits := uint32(len(g.keyHashes) * g.n) 84 | // For small n, we can see a very high false positive rate. Fix it 85 | // by enforcing a minimum bloom filter length. 86 | if nBits < 64 { 87 | nBits = 64 88 | } 89 | nBytes := (nBits + 7) / 8 90 | nBits = nBytes * 8 91 | 92 | dest := b.Alloc(int(nBytes) + 1) 93 | dest[nBytes] = g.k 94 | for _, kh := range g.keyHashes { 95 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 96 | for j := uint8(0); j < g.k; j++ { 97 | bitpos := kh % nBits 98 | dest[bitpos/8] |= (1 << (bitpos % 8)) 99 | kh += delta 100 | } 101 | } 102 | 103 | g.keyHashes = g.keyHashes[:0] 104 | } 105 | 106 | // NewBloomFilter creates a new initialized bloom filter for given 107 | // bitsPerKey. 108 | // 109 | // Since bitsPerKey is persisted individually for each bloom filter 110 | // serialization, bloom filters are backwards compatible with respect to 111 | // changing bitsPerKey. This means that no big performance penalty will 112 | // be experienced when changing the parameter. See documentation for 113 | // opt.Options.Filter for more information. 114 | func NewBloomFilter(bitsPerKey int) Filter { 115 | return bloomFilter(bitsPerKey) 116 | } 117 | -------------------------------------------------------------------------------- /leveldb/filter/bloom_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package filter 8 | 9 | import ( 10 | "encoding/binary" 11 | "github.com/syndtr/goleveldb/leveldb/util" 12 | "testing" 13 | ) 14 | 15 | type harness struct { 16 | t *testing.T 17 | 18 | bloom Filter 19 | generator FilterGenerator 20 | filter []byte 21 | } 22 | 23 | func newHarness(t *testing.T) *harness { 24 | bloom := NewBloomFilter(10) 25 | return &harness{ 26 | t: t, 27 | bloom: bloom, 28 | generator: bloom.NewGenerator(), 29 | } 30 | } 31 | 32 | func (h *harness) add(key []byte) { 33 | h.generator.Add(key) 34 | } 35 | 36 | func (h *harness) addNum(key uint32) { 37 | var b [4]byte 38 | binary.LittleEndian.PutUint32(b[:], key) 39 | h.add(b[:]) 40 | } 41 | 42 | func (h *harness) build() { 43 | b := &util.Buffer{} 44 | h.generator.Generate(b) 45 | h.filter = b.Bytes() 46 | } 47 | 48 | func (h *harness) reset() { 49 | h.filter = nil 50 | } 51 | 52 | func (h *harness) filterLen() int { 53 | return len(h.filter) 54 | } 55 | 56 | func (h *harness) assert(key []byte, want, silent bool) bool { 57 | got := h.bloom.Contains(h.filter, key) 58 | if !silent && got != want { 59 | h.t.Errorf("assert on '%v' failed got '%v', want '%v'", key, got, want) 60 | } 61 | return got 62 | } 63 | 64 | func (h *harness) assertNum(key uint32, want, silent bool) bool { 65 | var b [4]byte 66 | binary.LittleEndian.PutUint32(b[:], key) 67 | return h.assert(b[:], want, silent) 68 | } 69 | 70 | func TestBloomFilter_Empty(t *testing.T) { 71 | h := newHarness(t) 72 | h.build() 73 | h.assert([]byte("hello"), false, false) 74 | h.assert([]byte("world"), false, false) 75 | } 76 | 77 | func TestBloomFilter_Small(t *testing.T) { 78 | h := newHarness(t) 79 | h.add([]byte("hello")) 80 | h.add([]byte("world")) 81 | h.build() 82 | h.assert([]byte("hello"), true, false) 83 | h.assert([]byte("world"), true, false) 84 | h.assert([]byte("x"), false, false) 85 | h.assert([]byte("foo"), false, false) 86 | } 87 | 88 | func nextN(n int) int { 89 | switch { 90 | case n < 10: 91 | n += 1 92 | case n < 100: 93 | n += 10 94 | case n < 1000: 95 | n += 100 96 | default: 97 | n += 1000 98 | } 99 | return n 100 | } 101 | 102 | func TestBloomFilter_VaryingLengths(t *testing.T) { 103 | h := newHarness(t) 104 | var mediocre, good int 105 | for n := 1; n < 10000; n = nextN(n) { 106 | h.reset() 107 | for i := 0; i < n; i++ { 108 | h.addNum(uint32(i)) 109 | } 110 | h.build() 111 | 112 | got := h.filterLen() 113 | want := (n * 10 / 8) + 40 114 | if got > want { 115 | t.Errorf("filter len test failed, '%d' > '%d'", got, want) 116 | } 117 | 118 | for i := 0; i < n; i++ { 119 | h.assertNum(uint32(i), true, false) 120 | } 121 | 122 | var rate float32 123 | for i := 0; i < 10000; i++ { 124 | if h.assertNum(uint32(i+1000000000), true, true) { 125 | rate++ 126 | } 127 | } 128 | rate /= 10000 129 | if rate > 0.02 { 130 | t.Errorf("false positive rate is more than 2%%, got %v, at len %d", rate, n) 131 | } 132 | if rate > 0.0125 { 133 | mediocre++ 134 | } else { 135 | good++ 136 | } 137 | } 138 | t.Logf("false positive rate: %d good, %d mediocre", good, mediocre) 139 | if mediocre > good/5 { 140 | t.Error("mediocre false positive rate is more than expected") 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /leveldb/filter/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package filter provides interface and implementation of probabilistic 8 | // data structure. 9 | // 10 | // The filter is resposible for creating small filter from a set of keys. 11 | // These filter will then used to test whether a key is a member of the set. 12 | // In many cases, a filter can cut down the number of disk seeks from a 13 | // handful to a single disk seek per DB.Get call. 14 | package filter 15 | 16 | // Buffer is the interface that wraps basic Alloc, Write and WriteByte methods. 17 | type Buffer interface { 18 | // Alloc allocs n bytes of slice from the buffer. This also advancing 19 | // write offset. 20 | Alloc(n int) []byte 21 | 22 | // Write appends the contents of p to the buffer. 23 | Write(p []byte) (n int, err error) 24 | 25 | // WriteByte appends the byte c to the buffer. 26 | WriteByte(c byte) error 27 | } 28 | 29 | // Filter is the filter. 30 | type Filter interface { 31 | // Name returns the name of this policy. 32 | // 33 | // Note that if the filter encoding changes in an incompatible way, 34 | // the name returned by this method must be changed. Otherwise, old 35 | // incompatible filters may be passed to methods of this type. 36 | Name() string 37 | 38 | // NewGenerator creates a new filter generator. 39 | NewGenerator() FilterGenerator 40 | 41 | // Contains returns true if the filter contains the given key. 42 | // 43 | // The filter are filters generated by the filter generator. 44 | Contains(filter, key []byte) bool 45 | } 46 | 47 | // FilterGenerator is the filter generator. 48 | type FilterGenerator interface { 49 | // Add adds a key to the filter generator. 50 | // 51 | // The key may become invalid after call to this method end, therefor 52 | // key must be copied if implementation require keeping key for later 53 | // use. The key should not modified directly, doing so may cause 54 | // undefined results. 55 | Add(key []byte) 56 | 57 | // Generate generates filters based on keys passed so far. After call 58 | // to Generate the filter generator maybe resetted, depends on implementation. 59 | Generate(b Buffer) 60 | } 61 | -------------------------------------------------------------------------------- /leveldb/iterator/array_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/util" 11 | ) 12 | 13 | // BasicArray is the interface that wraps basic Len and Search method. 14 | type BasicArray interface { 15 | // Len returns length of the array. 16 | Len() int 17 | 18 | // Search finds smallest index that point to a key that is greater 19 | // than or equal to the given key. 20 | Search(key []byte) int 21 | } 22 | 23 | // Array is the interface that wraps BasicArray and basic Index method. 24 | type Array interface { 25 | BasicArray 26 | 27 | // Index returns key/value pair with index of i. 28 | Index(i int) (key, value []byte) 29 | } 30 | 31 | // Array is the interface that wraps BasicArray and basic Get method. 32 | type ArrayIndexer interface { 33 | BasicArray 34 | 35 | // Get returns a new data iterator with index of i. 36 | Get(i int) Iterator 37 | } 38 | 39 | type basicArrayIterator struct { 40 | util.BasicReleaser 41 | array BasicArray 42 | pos int 43 | err error 44 | } 45 | 46 | func (i *basicArrayIterator) Valid() bool { 47 | return i.pos >= 0 && i.pos < i.array.Len() && !i.Released() 48 | } 49 | 50 | func (i *basicArrayIterator) First() bool { 51 | if i.Released() { 52 | i.err = ErrIterReleased 53 | return false 54 | } 55 | 56 | if i.array.Len() == 0 { 57 | i.pos = -1 58 | return false 59 | } 60 | i.pos = 0 61 | return true 62 | } 63 | 64 | func (i *basicArrayIterator) Last() bool { 65 | if i.Released() { 66 | i.err = ErrIterReleased 67 | return false 68 | } 69 | 70 | n := i.array.Len() 71 | if n == 0 { 72 | i.pos = 0 73 | return false 74 | } 75 | i.pos = n - 1 76 | return true 77 | } 78 | 79 | func (i *basicArrayIterator) Seek(key []byte) bool { 80 | if i.Released() { 81 | i.err = ErrIterReleased 82 | return false 83 | } 84 | 85 | n := i.array.Len() 86 | if n == 0 { 87 | i.pos = 0 88 | return false 89 | } 90 | i.pos = i.array.Search(key) 91 | return i.pos < n 92 | } 93 | 94 | func (i *basicArrayIterator) Next() bool { 95 | if i.Released() { 96 | i.err = ErrIterReleased 97 | return false 98 | } 99 | 100 | i.pos++ 101 | if n := i.array.Len(); i.pos >= n { 102 | i.pos = n 103 | return false 104 | } 105 | return true 106 | } 107 | 108 | func (i *basicArrayIterator) Prev() bool { 109 | if i.Released() { 110 | i.err = ErrIterReleased 111 | return false 112 | } 113 | 114 | i.pos-- 115 | if i.pos < 0 { 116 | i.pos = -1 117 | return false 118 | } 119 | return true 120 | } 121 | 122 | func (i *basicArrayIterator) Error() error { return i.err } 123 | 124 | type arrayIterator struct { 125 | basicArrayIterator 126 | array Array 127 | pos int 128 | key, value []byte 129 | } 130 | 131 | func (i *arrayIterator) updateKV() { 132 | if i.pos == i.basicArrayIterator.pos { 133 | return 134 | } 135 | i.pos = i.basicArrayIterator.pos 136 | if i.Valid() { 137 | i.key, i.value = i.array.Index(i.pos) 138 | } else { 139 | i.key = nil 140 | i.value = nil 141 | } 142 | } 143 | 144 | func (i *arrayIterator) Key() []byte { 145 | i.updateKV() 146 | return i.key 147 | } 148 | 149 | func (i *arrayIterator) Value() []byte { 150 | i.updateKV() 151 | return i.value 152 | } 153 | 154 | type arrayIteratorIndexer struct { 155 | basicArrayIterator 156 | array ArrayIndexer 157 | } 158 | 159 | func (i *arrayIteratorIndexer) Get() Iterator { 160 | if i.Valid() { 161 | return i.array.Get(i.basicArrayIterator.pos) 162 | } 163 | return nil 164 | } 165 | 166 | // NewArrayIterator returns an iterator from the given array. 167 | func NewArrayIterator(array Array) Iterator { 168 | return &arrayIterator{ 169 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 170 | array: array, 171 | pos: -1, 172 | } 173 | } 174 | 175 | // NewArrayIndexer returns an index iterator from the given array. 176 | func NewArrayIndexer(array ArrayIndexer) IteratorIndexer { 177 | return &arrayIteratorIndexer{ 178 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 179 | array: array, 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /leveldb/iterator/array_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | 12 | . "github.com/syndtr/goleveldb/leveldb/iterator" 13 | "github.com/syndtr/goleveldb/leveldb/testutil" 14 | ) 15 | 16 | var _ = testutil.Defer(func() { 17 | Describe("Array iterator", func() { 18 | It("Should iterates and seeks correctly", func() { 19 | // Build key/value. 20 | kv := testutil.KeyValue_Generate(nil, 70, 1, 1, 5, 3, 3) 21 | 22 | // Test the iterator. 23 | t := testutil.IteratorTesting{ 24 | KeyValue: kv.Clone(), 25 | Iter: NewArrayIterator(kv), 26 | } 27 | testutil.DoIteratorTesting(&t) 28 | }) 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /leveldb/iterator/indexed_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | "github.com/syndtr/goleveldb/leveldb/util" 12 | ) 13 | 14 | // IteratorIndexer is the interface that wraps CommonIterator and basic Get 15 | // method. IteratorIndexer provides index for indexed iterator. 16 | type IteratorIndexer interface { 17 | CommonIterator 18 | 19 | // Get returns a new data iterator for the current position, or nil if 20 | // done. 21 | Get() Iterator 22 | } 23 | 24 | type indexedIterator struct { 25 | util.BasicReleaser 26 | index IteratorIndexer 27 | strict bool 28 | 29 | data Iterator 30 | err error 31 | errf func(err error) 32 | } 33 | 34 | func (i *indexedIterator) setData() { 35 | if i.data != nil { 36 | i.data.Release() 37 | } 38 | i.data = i.index.Get() 39 | } 40 | 41 | func (i *indexedIterator) clearData() { 42 | if i.data != nil { 43 | i.data.Release() 44 | } 45 | i.data = nil 46 | } 47 | 48 | func (i *indexedIterator) indexErr() { 49 | if err := i.index.Error(); err != nil { 50 | if i.errf != nil { 51 | i.errf(err) 52 | } 53 | i.err = err 54 | } 55 | } 56 | 57 | func (i *indexedIterator) dataErr() bool { 58 | if err := i.data.Error(); err != nil { 59 | if i.errf != nil { 60 | i.errf(err) 61 | } 62 | if i.strict || !errors.IsCorrupted(err) { 63 | i.err = err 64 | return true 65 | } 66 | } 67 | return false 68 | } 69 | 70 | func (i *indexedIterator) Valid() bool { 71 | return i.data != nil && i.data.Valid() 72 | } 73 | 74 | func (i *indexedIterator) First() bool { 75 | if i.err != nil { 76 | return false 77 | } else if i.Released() { 78 | i.err = ErrIterReleased 79 | return false 80 | } 81 | 82 | if !i.index.First() { 83 | i.indexErr() 84 | i.clearData() 85 | return false 86 | } 87 | i.setData() 88 | return i.Next() 89 | } 90 | 91 | func (i *indexedIterator) Last() bool { 92 | if i.err != nil { 93 | return false 94 | } else if i.Released() { 95 | i.err = ErrIterReleased 96 | return false 97 | } 98 | 99 | if !i.index.Last() { 100 | i.indexErr() 101 | i.clearData() 102 | return false 103 | } 104 | i.setData() 105 | if !i.data.Last() { 106 | if i.dataErr() { 107 | return false 108 | } 109 | i.clearData() 110 | return i.Prev() 111 | } 112 | return true 113 | } 114 | 115 | func (i *indexedIterator) Seek(key []byte) bool { 116 | if i.err != nil { 117 | return false 118 | } else if i.Released() { 119 | i.err = ErrIterReleased 120 | return false 121 | } 122 | 123 | if !i.index.Seek(key) { 124 | i.indexErr() 125 | i.clearData() 126 | return false 127 | } 128 | i.setData() 129 | if !i.data.Seek(key) { 130 | if i.dataErr() { 131 | return false 132 | } 133 | i.clearData() 134 | return i.Next() 135 | } 136 | return true 137 | } 138 | 139 | func (i *indexedIterator) Next() bool { 140 | if i.err != nil { 141 | return false 142 | } else if i.Released() { 143 | i.err = ErrIterReleased 144 | return false 145 | } 146 | 147 | switch { 148 | case i.data != nil && !i.data.Next(): 149 | if i.dataErr() { 150 | return false 151 | } 152 | i.clearData() 153 | fallthrough 154 | case i.data == nil: 155 | if !i.index.Next() { 156 | i.indexErr() 157 | return false 158 | } 159 | i.setData() 160 | return i.Next() 161 | } 162 | return true 163 | } 164 | 165 | func (i *indexedIterator) Prev() bool { 166 | if i.err != nil { 167 | return false 168 | } else if i.Released() { 169 | i.err = ErrIterReleased 170 | return false 171 | } 172 | 173 | switch { 174 | case i.data != nil && !i.data.Prev(): 175 | if i.dataErr() { 176 | return false 177 | } 178 | i.clearData() 179 | fallthrough 180 | case i.data == nil: 181 | if !i.index.Prev() { 182 | i.indexErr() 183 | return false 184 | } 185 | i.setData() 186 | if !i.data.Last() { 187 | if i.dataErr() { 188 | return false 189 | } 190 | i.clearData() 191 | return i.Prev() 192 | } 193 | } 194 | return true 195 | } 196 | 197 | func (i *indexedIterator) Key() []byte { 198 | if i.data == nil { 199 | return nil 200 | } 201 | return i.data.Key() 202 | } 203 | 204 | func (i *indexedIterator) Value() []byte { 205 | if i.data == nil { 206 | return nil 207 | } 208 | return i.data.Value() 209 | } 210 | 211 | func (i *indexedIterator) Release() { 212 | i.clearData() 213 | i.index.Release() 214 | i.BasicReleaser.Release() 215 | } 216 | 217 | func (i *indexedIterator) Error() error { 218 | if i.err != nil { 219 | return i.err 220 | } 221 | if err := i.index.Error(); err != nil { 222 | return err 223 | } 224 | return nil 225 | } 226 | 227 | func (i *indexedIterator) SetErrorCallback(f func(err error)) { 228 | i.errf = f 229 | } 230 | 231 | // NewIndexedIterator returns an 'indexed iterator'. An index is iterator 232 | // that returns another iterator, a 'data iterator'. A 'data iterator' is the 233 | // iterator that contains actual key/value pairs. 234 | // 235 | // If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) 236 | // won't be ignored and will halt 'indexed iterator', otherwise the iterator will 237 | // continue to the next 'data iterator'. Corruption on 'index iterator' will not be 238 | // ignored and will halt the iterator. 239 | func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator { 240 | return &indexedIterator{index: index, strict: strict} 241 | } 242 | -------------------------------------------------------------------------------- /leveldb/iterator/indexed_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | "sort" 11 | 12 | . "github.com/onsi/ginkgo" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/comparer" 15 | . "github.com/syndtr/goleveldb/leveldb/iterator" 16 | "github.com/syndtr/goleveldb/leveldb/testutil" 17 | ) 18 | 19 | type keyValue struct { 20 | key []byte 21 | testutil.KeyValue 22 | } 23 | 24 | type keyValueIndex []keyValue 25 | 26 | func (x keyValueIndex) Search(key []byte) int { 27 | return sort.Search(x.Len(), func(i int) bool { 28 | return comparer.DefaultComparer.Compare(x[i].key, key) >= 0 29 | }) 30 | } 31 | 32 | func (x keyValueIndex) Len() int { return len(x) } 33 | func (x keyValueIndex) Index(i int) (key, value []byte) { return x[i].key, nil } 34 | func (x keyValueIndex) Get(i int) Iterator { return NewArrayIterator(x[i]) } 35 | 36 | var _ = testutil.Defer(func() { 37 | Describe("Indexed iterator", func() { 38 | Test := func(n ...int) func() { 39 | if len(n) == 0 { 40 | rnd := testutil.NewRand() 41 | n = make([]int, rnd.Intn(17)+3) 42 | for i := range n { 43 | n[i] = rnd.Intn(19) + 1 44 | } 45 | } 46 | 47 | return func() { 48 | It("Should iterates and seeks correctly", func(done Done) { 49 | // Build key/value. 50 | index := make(keyValueIndex, len(n)) 51 | sum := 0 52 | for _, x := range n { 53 | sum += x 54 | } 55 | kv := testutil.KeyValue_Generate(nil, sum, 1, 1, 10, 4, 4) 56 | for i, j := 0, 0; i < len(n); i++ { 57 | for x := n[i]; x > 0; x-- { 58 | key, value := kv.Index(j) 59 | index[i].key = key 60 | index[i].Put(key, value) 61 | j++ 62 | } 63 | } 64 | 65 | // Test the iterator. 66 | t := testutil.IteratorTesting{ 67 | KeyValue: kv.Clone(), 68 | Iter: NewIndexedIterator(NewArrayIndexer(index), true), 69 | } 70 | testutil.DoIteratorTesting(&t) 71 | done <- true 72 | }, 15.0) 73 | } 74 | } 75 | 76 | Describe("with 100 keys", Test(100)) 77 | Describe("with 50-50 keys", Test(50, 50)) 78 | Describe("with 50-1 keys", Test(50, 1)) 79 | Describe("with 50-1-50 keys", Test(50, 1, 50)) 80 | Describe("with 1-50 keys", Test(1, 50)) 81 | Describe("with random N-keys", Test()) 82 | }) 83 | }) 84 | -------------------------------------------------------------------------------- /leveldb/iterator/iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package iterator provides interface and implementation to traverse over 8 | // contents of a database. 9 | package iterator 10 | 11 | import ( 12 | "errors" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/util" 15 | ) 16 | 17 | var ( 18 | ErrIterReleased = errors.New("leveldb/iterator: iterator released") 19 | ) 20 | 21 | // IteratorSeeker is the interface that wraps the 'seeks method'. 22 | type IteratorSeeker interface { 23 | // First moves the iterator to the first key/value pair. If the iterator 24 | // only contains one key/value pair then First and Last would moves 25 | // to the same key/value pair. 26 | // It returns whether such pair exist. 27 | First() bool 28 | 29 | // Last moves the iterator to the last key/value pair. If the iterator 30 | // only contains one key/value pair then First and Last would moves 31 | // to the same key/value pair. 32 | // It returns whether such pair exist. 33 | Last() bool 34 | 35 | // Seek moves the iterator to the first key/value pair whose key is greater 36 | // than or equal to the given key. 37 | // It returns whether such pair exist. 38 | // 39 | // It is safe to modify the contents of the argument after Seek returns. 40 | Seek(key []byte) bool 41 | 42 | // Next moves the iterator to the next key/value pair. 43 | // It returns false if the iterator is exhausted. 44 | Next() bool 45 | 46 | // Prev moves the iterator to the previous key/value pair. 47 | // It returns false if the iterator is exhausted. 48 | Prev() bool 49 | } 50 | 51 | // CommonIterator is the interface that wraps common iterator methods. 52 | type CommonIterator interface { 53 | IteratorSeeker 54 | 55 | // util.Releaser is the interface that wraps basic Release method. 56 | // When called Release will releases any resources associated with the 57 | // iterator. 58 | util.Releaser 59 | 60 | // util.ReleaseSetter is the interface that wraps the basic SetReleaser 61 | // method. 62 | util.ReleaseSetter 63 | 64 | // TODO: Remove this when ready. 65 | Valid() bool 66 | 67 | // Error returns any accumulated error. Exhausting all the key/value pairs 68 | // is not considered to be an error. 69 | Error() error 70 | } 71 | 72 | // Iterator iterates over a DB's key/value pairs in key order. 73 | // 74 | // When encounter an error any 'seeks method' will return false and will 75 | // yield no key/value pairs. The error can be queried by calling the Error 76 | // method. Calling Release is still necessary. 77 | // 78 | // An iterator must be released after use, but it is not necessary to read 79 | // an iterator until exhaustion. 80 | // Also, an iterator is not necessarily safe for concurrent use, but it is 81 | // safe to use multiple iterators concurrently, with each in a dedicated 82 | // goroutine. 83 | type Iterator interface { 84 | CommonIterator 85 | 86 | // Key returns the key of the current key/value pair, or nil if done. 87 | // The caller should not modify the contents of the returned slice, and 88 | // its contents may change on the next call to any 'seeks method'. 89 | Key() []byte 90 | 91 | // Value returns the value of the current key/value pair, or nil if done. 92 | // The caller should not modify the contents of the returned slice, and 93 | // its contents may change on the next call to any 'seeks method'. 94 | Value() []byte 95 | } 96 | 97 | // ErrorCallbackSetter is the interface that wraps basic SetErrorCallback 98 | // method. 99 | // 100 | // ErrorCallbackSetter implemented by indexed and merged iterator. 101 | type ErrorCallbackSetter interface { 102 | // SetErrorCallback allows set an error callback of the corresponding 103 | // iterator. Use nil to clear the callback. 104 | SetErrorCallback(f func(err error)) 105 | } 106 | 107 | type emptyIterator struct { 108 | util.BasicReleaser 109 | err error 110 | } 111 | 112 | func (i *emptyIterator) rErr() { 113 | if i.err == nil && i.Released() { 114 | i.err = ErrIterReleased 115 | } 116 | } 117 | 118 | func (*emptyIterator) Valid() bool { return false } 119 | func (i *emptyIterator) First() bool { i.rErr(); return false } 120 | func (i *emptyIterator) Last() bool { i.rErr(); return false } 121 | func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false } 122 | func (i *emptyIterator) Next() bool { i.rErr(); return false } 123 | func (i *emptyIterator) Prev() bool { i.rErr(); return false } 124 | func (*emptyIterator) Key() []byte { return nil } 125 | func (*emptyIterator) Value() []byte { return nil } 126 | func (i *emptyIterator) Error() error { return i.err } 127 | 128 | // NewEmptyIterator creates an empty iterator. The err parameter can be 129 | // nil, but if not nil the given err will be returned by Error method. 130 | func NewEmptyIterator(err error) Iterator { 131 | return &emptyIterator{err: err} 132 | } 133 | -------------------------------------------------------------------------------- /leveldb/iterator/iter_suite_test.go: -------------------------------------------------------------------------------- 1 | package iterator_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/syndtr/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestIterator(t *testing.T) { 10 | testutil.RunSuite(t, "Iterator Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/iterator/merged_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "container/heap" 11 | 12 | "github.com/syndtr/goleveldb/leveldb/comparer" 13 | "github.com/syndtr/goleveldb/leveldb/errors" 14 | "github.com/syndtr/goleveldb/leveldb/util" 15 | ) 16 | 17 | type dir int 18 | 19 | const ( 20 | dirReleased dir = iota - 1 21 | dirSOI 22 | dirEOI 23 | dirBackward 24 | dirForward 25 | ) 26 | 27 | type mergedIterator struct { 28 | cmp comparer.Comparer 29 | iters []Iterator 30 | strict bool 31 | 32 | keys [][]byte 33 | index int 34 | dir dir 35 | err error 36 | errf func(err error) 37 | releaser util.Releaser 38 | 39 | indexes []int // the heap of iterator indexes 40 | reverse bool //nolint: structcheck // if true, indexes is a max-heap 41 | } 42 | 43 | func assertKey(key []byte) []byte { 44 | if key == nil { 45 | panic("leveldb/iterator: nil key") 46 | } 47 | return key 48 | } 49 | 50 | func (i *mergedIterator) iterErr(iter Iterator) bool { 51 | if err := iter.Error(); err != nil { 52 | if i.errf != nil { 53 | i.errf(err) 54 | } 55 | if i.strict || !errors.IsCorrupted(err) { 56 | i.err = err 57 | return true 58 | } 59 | } 60 | return false 61 | } 62 | 63 | func (i *mergedIterator) Valid() bool { 64 | return i.err == nil && i.dir > dirEOI 65 | } 66 | 67 | func (i *mergedIterator) First() bool { 68 | if i.err != nil { 69 | return false 70 | } else if i.dir == dirReleased { 71 | i.err = ErrIterReleased 72 | return false 73 | } 74 | 75 | h := i.indexHeap() 76 | h.Reset(false) 77 | for x, iter := range i.iters { 78 | switch { 79 | case iter.First(): 80 | i.keys[x] = assertKey(iter.Key()) 81 | h.Push(x) 82 | case i.iterErr(iter): 83 | return false 84 | default: 85 | i.keys[x] = nil 86 | } 87 | } 88 | heap.Init(h) 89 | i.dir = dirSOI 90 | return i.next() 91 | } 92 | 93 | func (i *mergedIterator) Last() bool { 94 | if i.err != nil { 95 | return false 96 | } else if i.dir == dirReleased { 97 | i.err = ErrIterReleased 98 | return false 99 | } 100 | 101 | h := i.indexHeap() 102 | h.Reset(true) 103 | for x, iter := range i.iters { 104 | switch { 105 | case iter.Last(): 106 | i.keys[x] = assertKey(iter.Key()) 107 | h.Push(x) 108 | case i.iterErr(iter): 109 | return false 110 | default: 111 | i.keys[x] = nil 112 | } 113 | } 114 | heap.Init(h) 115 | i.dir = dirEOI 116 | return i.prev() 117 | } 118 | 119 | func (i *mergedIterator) Seek(key []byte) bool { 120 | if i.err != nil { 121 | return false 122 | } else if i.dir == dirReleased { 123 | i.err = ErrIterReleased 124 | return false 125 | } 126 | 127 | h := i.indexHeap() 128 | h.Reset(false) 129 | for x, iter := range i.iters { 130 | switch { 131 | case iter.Seek(key): 132 | i.keys[x] = assertKey(iter.Key()) 133 | h.Push(x) 134 | case i.iterErr(iter): 135 | return false 136 | default: 137 | i.keys[x] = nil 138 | } 139 | } 140 | heap.Init(h) 141 | i.dir = dirSOI 142 | return i.next() 143 | } 144 | 145 | func (i *mergedIterator) next() bool { 146 | h := i.indexHeap() 147 | if h.Len() == 0 { 148 | i.dir = dirEOI 149 | return false 150 | } 151 | i.index = heap.Pop(h).(int) 152 | i.dir = dirForward 153 | return true 154 | } 155 | 156 | func (i *mergedIterator) Next() bool { 157 | if i.dir == dirEOI || i.err != nil { 158 | return false 159 | } else if i.dir == dirReleased { 160 | i.err = ErrIterReleased 161 | return false 162 | } 163 | 164 | switch i.dir { 165 | case dirSOI: 166 | return i.First() 167 | case dirBackward: 168 | key := append([]byte(nil), i.keys[i.index]...) 169 | if !i.Seek(key) { 170 | return false 171 | } 172 | return i.Next() 173 | } 174 | 175 | x := i.index 176 | iter := i.iters[x] 177 | switch { 178 | case iter.Next(): 179 | i.keys[x] = assertKey(iter.Key()) 180 | heap.Push(i.indexHeap(), x) 181 | case i.iterErr(iter): 182 | return false 183 | default: 184 | i.keys[x] = nil 185 | } 186 | return i.next() 187 | } 188 | 189 | func (i *mergedIterator) prev() bool { 190 | h := i.indexHeap() 191 | if h.Len() == 0 { 192 | i.dir = dirSOI 193 | return false 194 | } 195 | i.index = heap.Pop(h).(int) 196 | i.dir = dirBackward 197 | return true 198 | } 199 | 200 | func (i *mergedIterator) Prev() bool { 201 | if i.dir == dirSOI || i.err != nil { 202 | return false 203 | } else if i.dir == dirReleased { 204 | i.err = ErrIterReleased 205 | return false 206 | } 207 | 208 | switch i.dir { 209 | case dirEOI: 210 | return i.Last() 211 | case dirForward: 212 | key := append([]byte(nil), i.keys[i.index]...) 213 | h := i.indexHeap() 214 | h.Reset(true) 215 | for x, iter := range i.iters { 216 | if x == i.index { 217 | continue 218 | } 219 | seek := iter.Seek(key) 220 | switch { 221 | case seek && iter.Prev(), !seek && iter.Last(): 222 | i.keys[x] = assertKey(iter.Key()) 223 | h.Push(x) 224 | case i.iterErr(iter): 225 | return false 226 | default: 227 | i.keys[x] = nil 228 | } 229 | } 230 | heap.Init(h) 231 | } 232 | 233 | x := i.index 234 | iter := i.iters[x] 235 | switch { 236 | case iter.Prev(): 237 | i.keys[x] = assertKey(iter.Key()) 238 | heap.Push(i.indexHeap(), x) 239 | case i.iterErr(iter): 240 | return false 241 | default: 242 | i.keys[x] = nil 243 | } 244 | return i.prev() 245 | } 246 | 247 | func (i *mergedIterator) Key() []byte { 248 | if i.err != nil || i.dir <= dirEOI { 249 | return nil 250 | } 251 | return i.keys[i.index] 252 | } 253 | 254 | func (i *mergedIterator) Value() []byte { 255 | if i.err != nil || i.dir <= dirEOI { 256 | return nil 257 | } 258 | return i.iters[i.index].Value() 259 | } 260 | 261 | func (i *mergedIterator) Release() { 262 | if i.dir != dirReleased { 263 | i.dir = dirReleased 264 | for _, iter := range i.iters { 265 | iter.Release() 266 | } 267 | i.iters = nil 268 | i.keys = nil 269 | i.indexes = nil 270 | if i.releaser != nil { 271 | i.releaser.Release() 272 | i.releaser = nil 273 | } 274 | } 275 | } 276 | 277 | func (i *mergedIterator) SetReleaser(releaser util.Releaser) { 278 | if i.dir == dirReleased { 279 | panic(util.ErrReleased) 280 | } 281 | if i.releaser != nil && releaser != nil { 282 | panic(util.ErrHasReleaser) 283 | } 284 | i.releaser = releaser 285 | } 286 | 287 | func (i *mergedIterator) Error() error { 288 | return i.err 289 | } 290 | 291 | func (i *mergedIterator) SetErrorCallback(f func(err error)) { 292 | i.errf = f 293 | } 294 | 295 | func (i *mergedIterator) indexHeap() *indexHeap { 296 | return (*indexHeap)(i) 297 | } 298 | 299 | // NewMergedIterator returns an iterator that merges its input. Walking the 300 | // resultant iterator will return all key/value pairs of all input iterators 301 | // in strictly increasing key order, as defined by cmp. 302 | // The input's key ranges may overlap, but there are assumed to be no duplicate 303 | // keys: if iters[i] contains a key k then iters[j] will not contain that key k. 304 | // None of the iters may be nil. 305 | // 306 | // If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) 307 | // won't be ignored and will halt 'merged iterator', otherwise the iterator will 308 | // continue to the next 'input iterator'. 309 | func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator { 310 | return &mergedIterator{ 311 | iters: iters, 312 | cmp: cmp, 313 | strict: strict, 314 | keys: make([][]byte, len(iters)), 315 | indexes: make([]int, 0, len(iters)), 316 | } 317 | } 318 | 319 | // indexHeap implements heap.Interface. 320 | type indexHeap mergedIterator 321 | 322 | func (h *indexHeap) Len() int { return len(h.indexes) } 323 | func (h *indexHeap) Less(i, j int) bool { 324 | i, j = h.indexes[i], h.indexes[j] 325 | r := h.cmp.Compare(h.keys[i], h.keys[j]) 326 | if h.reverse { 327 | return r > 0 328 | } 329 | return r < 0 330 | } 331 | 332 | func (h *indexHeap) Swap(i, j int) { 333 | h.indexes[i], h.indexes[j] = h.indexes[j], h.indexes[i] 334 | } 335 | 336 | func (h *indexHeap) Push(value interface{}) { 337 | h.indexes = append(h.indexes, value.(int)) 338 | } 339 | 340 | func (h *indexHeap) Pop() interface{} { 341 | e := len(h.indexes) - 1 342 | popped := h.indexes[e] 343 | h.indexes = h.indexes[:e] 344 | return popped 345 | } 346 | 347 | func (h *indexHeap) Reset(reverse bool) { 348 | h.reverse = reverse 349 | h.indexes = h.indexes[:0] 350 | } 351 | -------------------------------------------------------------------------------- /leveldb/iterator/merged_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | "testing" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/comparer" 16 | . "github.com/syndtr/goleveldb/leveldb/iterator" 17 | "github.com/syndtr/goleveldb/leveldb/testutil" 18 | ) 19 | 20 | var _ = testutil.Defer(func() { 21 | Describe("Merged iterator", func() { 22 | Test := func(filled int, empty int) func() { 23 | return func() { 24 | It("Should iterates and seeks correctly", func(done Done) { 25 | rnd := testutil.NewRand() 26 | 27 | // Build key/value. 28 | filledKV := make([]testutil.KeyValue, filled) 29 | kv := testutil.KeyValue_Generate(nil, 100, 1, 1, 10, 4, 4) 30 | kv.Iterate(func(i int, key, value []byte) { 31 | filledKV[rnd.Intn(filled)].Put(key, value) 32 | }) 33 | 34 | // Create itearators. 35 | iters := make([]Iterator, filled+empty) 36 | for i := range iters { 37 | if empty == 0 || (rnd.Int()%2 == 0 && filled > 0) { 38 | filled-- 39 | Expect(filledKV[filled].Len()).ShouldNot(BeZero()) 40 | iters[i] = NewArrayIterator(filledKV[filled]) 41 | } else { 42 | empty-- 43 | iters[i] = NewEmptyIterator(nil) 44 | } 45 | } 46 | 47 | // Test the iterator. 48 | t := testutil.IteratorTesting{ 49 | KeyValue: kv.Clone(), 50 | Iter: NewMergedIterator(iters, comparer.DefaultComparer, true), 51 | } 52 | testutil.DoIteratorTesting(&t) 53 | done <- true 54 | }, 15.0) 55 | } 56 | } 57 | 58 | Describe("with three, all filled iterators", Test(3, 0)) 59 | Describe("with one filled, one empty iterators", Test(1, 1)) 60 | Describe("with one filled, two empty iterators", Test(1, 2)) 61 | }) 62 | }) 63 | 64 | func BenchmarkMergedIterator(b *testing.B) { 65 | n := 11 66 | iters := make([]Iterator, n) 67 | for i := range iters { 68 | kv := testutil.KeyValue_Generate(nil, 100, 1, 1, 10, 4, 4) 69 | iters[i] = NewArrayIterator(kv) 70 | } 71 | 72 | mi := NewMergedIterator(iters, comparer.DefaultComparer, true) 73 | b.ResetTimer() 74 | 75 | for i := 0; i < b.N; i++ { 76 | mi.First() 77 | for mi.Next() { 78 | mi.Key() 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /leveldb/key.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "encoding/binary" 11 | "fmt" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/errors" 14 | "github.com/syndtr/goleveldb/leveldb/storage" 15 | ) 16 | 17 | // ErrInternalKeyCorrupted records internal key corruption. 18 | type ErrInternalKeyCorrupted struct { 19 | Ikey []byte 20 | Reason string 21 | } 22 | 23 | func (e *ErrInternalKeyCorrupted) Error() string { 24 | return fmt.Sprintf("leveldb: internal key %q corrupted: %s", e.Ikey, e.Reason) 25 | } 26 | 27 | func newErrInternalKeyCorrupted(ikey []byte, reason string) error { 28 | return errors.NewErrCorrupted(storage.FileDesc{}, &ErrInternalKeyCorrupted{append([]byte(nil), ikey...), reason}) 29 | } 30 | 31 | type keyType uint 32 | 33 | func (kt keyType) String() string { 34 | switch kt { 35 | case keyTypeDel: 36 | return "d" 37 | case keyTypeVal: 38 | return "v" 39 | } 40 | return fmt.Sprintf("", uint(kt)) 41 | } 42 | 43 | // Value types encoded as the last component of internal keys. 44 | // Don't modify; this value are saved to disk. 45 | const ( 46 | keyTypeDel = keyType(0) 47 | keyTypeVal = keyType(1) 48 | ) 49 | 50 | // keyTypeSeek defines the keyType that should be passed when constructing an 51 | // internal key for seeking to a particular sequence number (since we 52 | // sort sequence numbers in decreasing order and the value type is 53 | // embedded as the low 8 bits in the sequence number in internal keys, 54 | // we need to use the highest-numbered ValueType, not the lowest). 55 | const keyTypeSeek = keyTypeVal 56 | 57 | const ( 58 | // Maximum value possible for sequence number; the 8-bits are 59 | // used by value type, so its can packed together in single 60 | // 64-bit integer. 61 | keyMaxSeq = (uint64(1) << 56) - 1 62 | // Maximum value possible for packed sequence number and type. 63 | keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek) 64 | ) 65 | 66 | // Maximum number encoded in bytes. 67 | var keyMaxNumBytes = make([]byte, 8) 68 | 69 | func init() { 70 | binary.LittleEndian.PutUint64(keyMaxNumBytes, keyMaxNum) 71 | } 72 | 73 | type internalKey []byte 74 | 75 | func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey { 76 | if seq > keyMaxSeq { 77 | panic("leveldb: invalid sequence number") 78 | } else if kt > keyTypeVal { 79 | panic("leveldb: invalid type") 80 | } 81 | 82 | dst = ensureBuffer(dst, len(ukey)+8) 83 | copy(dst, ukey) 84 | binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt)) 85 | return internalKey(dst) 86 | } 87 | 88 | func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) { 89 | if len(ik) < 8 { 90 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length") 91 | } 92 | num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 93 | seq, kt = num>>8, keyType(num&0xff) 94 | if kt > keyTypeVal { 95 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type") 96 | } 97 | ukey = ik[:len(ik)-8] 98 | return 99 | } 100 | 101 | func validInternalKey(ik []byte) bool { 102 | _, _, _, err := parseInternalKey(ik) 103 | return err == nil 104 | } 105 | 106 | func (ik internalKey) assert() { 107 | if ik == nil { 108 | panic("leveldb: nil internalKey") 109 | } 110 | if len(ik) < 8 { 111 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid length", []byte(ik), len(ik))) 112 | } 113 | } 114 | 115 | func (ik internalKey) ukey() []byte { 116 | ik.assert() 117 | return ik[:len(ik)-8] 118 | } 119 | 120 | func (ik internalKey) num() uint64 { 121 | ik.assert() 122 | return binary.LittleEndian.Uint64(ik[len(ik)-8:]) 123 | } 124 | 125 | func (ik internalKey) parseNum() (seq uint64, kt keyType) { 126 | num := ik.num() 127 | seq, kt = num>>8, keyType(num&0xff) 128 | if kt > keyTypeVal { 129 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt)) 130 | } 131 | return 132 | } 133 | 134 | func (ik internalKey) String() string { 135 | if ik == nil { 136 | return "" 137 | } 138 | 139 | if ukey, seq, kt, err := parseInternalKey(ik); err == nil { 140 | return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq) 141 | } 142 | return fmt.Sprintf("", []byte(ik)) 143 | } 144 | -------------------------------------------------------------------------------- /leveldb/key_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/comparer" 14 | ) 15 | 16 | var defaultIComparer = &iComparer{comparer.DefaultComparer} 17 | 18 | func ikey(key string, seq uint64, kt keyType) internalKey { 19 | return makeInternalKey(nil, []byte(key), seq, kt) 20 | } 21 | 22 | func shortSep(a, b []byte) []byte { 23 | dst := make([]byte, len(a)) 24 | dst = defaultIComparer.Separator(dst[:0], a, b) 25 | if dst == nil { 26 | return a 27 | } 28 | return dst 29 | } 30 | 31 | func shortSuccessor(b []byte) []byte { 32 | dst := make([]byte, len(b)) 33 | dst = defaultIComparer.Successor(dst[:0], b) 34 | if dst == nil { 35 | return b 36 | } 37 | return dst 38 | } 39 | 40 | func testSingleKey(t *testing.T, key string, seq uint64, kt keyType) { 41 | ik := ikey(key, seq, kt) 42 | 43 | if !bytes.Equal(ik.ukey(), []byte(key)) { 44 | t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) 45 | } 46 | 47 | rseq, rt := ik.parseNum() 48 | if rseq != seq { 49 | t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) 50 | } 51 | if rt != kt { 52 | t.Errorf("type does not equal, got %v, want %v", rt, kt) 53 | } 54 | 55 | if rukey, rseq, rt, kerr := parseInternalKey(ik); kerr == nil { 56 | if !bytes.Equal(rukey, []byte(key)) { 57 | t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) 58 | } 59 | if rseq != seq { 60 | t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) 61 | } 62 | if rt != kt { 63 | t.Errorf("type does not equal, got %v, want %v", rt, kt) 64 | } 65 | } else { 66 | t.Errorf("key error: %v", kerr) 67 | } 68 | } 69 | 70 | func TestInternalKey_EncodeDecode(t *testing.T) { 71 | keys := []string{"", "k", "hello", "longggggggggggggggggggggg"} 72 | seqs := []uint64{ 73 | 1, 2, 3, 74 | (1 << 8) - 1, 1 << 8, (1 << 8) + 1, 75 | (1 << 16) - 1, 1 << 16, (1 << 16) + 1, 76 | (1 << 32) - 1, 1 << 32, (1 << 32) + 1, 77 | } 78 | for _, key := range keys { 79 | for _, seq := range seqs { 80 | testSingleKey(t, key, seq, keyTypeVal) 81 | testSingleKey(t, "hello", 1, keyTypeDel) 82 | } 83 | } 84 | } 85 | 86 | func assertBytes(t *testing.T, want, got []byte) { 87 | if !bytes.Equal(got, want) { 88 | t.Errorf("assert failed, got %v, want %v", got, want) 89 | } 90 | } 91 | 92 | func TestInternalKeyShortSeparator(t *testing.T) { 93 | // When user keys are same 94 | assertBytes(t, ikey("foo", 100, keyTypeVal), 95 | shortSep(ikey("foo", 100, keyTypeVal), 96 | ikey("foo", 99, keyTypeVal))) 97 | assertBytes(t, ikey("foo", 100, keyTypeVal), 98 | shortSep(ikey("foo", 100, keyTypeVal), 99 | ikey("foo", 101, keyTypeVal))) 100 | assertBytes(t, ikey("foo", 100, keyTypeVal), 101 | shortSep(ikey("foo", 100, keyTypeVal), 102 | ikey("foo", 100, keyTypeVal))) 103 | assertBytes(t, ikey("foo", 100, keyTypeVal), 104 | shortSep(ikey("foo", 100, keyTypeVal), 105 | ikey("foo", 100, keyTypeDel))) 106 | 107 | // When user keys are misordered 108 | assertBytes(t, ikey("foo", 100, keyTypeVal), 109 | shortSep(ikey("foo", 100, keyTypeVal), 110 | ikey("bar", 99, keyTypeVal))) 111 | 112 | // When user keys are different, but correctly ordered 113 | assertBytes(t, ikey("g", keyMaxSeq, keyTypeSeek), 114 | shortSep(ikey("foo", 100, keyTypeVal), 115 | ikey("hello", 200, keyTypeVal))) 116 | 117 | // When start user key is prefix of limit user key 118 | assertBytes(t, ikey("foo", 100, keyTypeVal), 119 | shortSep(ikey("foo", 100, keyTypeVal), 120 | ikey("foobar", 200, keyTypeVal))) 121 | 122 | // When limit user key is prefix of start user key 123 | assertBytes(t, ikey("foobar", 100, keyTypeVal), 124 | shortSep(ikey("foobar", 100, keyTypeVal), 125 | ikey("foo", 200, keyTypeVal))) 126 | } 127 | 128 | func TestInternalKeyShortestSuccessor(t *testing.T) { 129 | assertBytes(t, ikey("g", keyMaxSeq, keyTypeSeek), 130 | shortSuccessor(ikey("foo", 100, keyTypeVal))) 131 | assertBytes(t, ikey("\xff\xff", 100, keyTypeVal), 132 | shortSuccessor(ikey("\xff\xff", 100, keyTypeVal))) 133 | } 134 | -------------------------------------------------------------------------------- /leveldb/leveldb_suite_test.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/syndtr/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestLevelDB(t *testing.T) { 10 | testutil.RunSuite(t, "LevelDB Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/memdb/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package memdb 8 | 9 | import ( 10 | "encoding/binary" 11 | "math/rand" 12 | "testing" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/comparer" 15 | ) 16 | 17 | func BenchmarkPut(b *testing.B) { 18 | buf := make([][4]byte, b.N) 19 | for i := range buf { 20 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 21 | } 22 | 23 | b.ResetTimer() 24 | p := New(comparer.DefaultComparer, 0) 25 | for i := range buf { 26 | if err := p.Put(buf[i][:], nil); err != nil { 27 | b.Fatal(err) 28 | } 29 | } 30 | } 31 | 32 | func BenchmarkPutRandom(b *testing.B) { 33 | buf := make([][4]byte, b.N) 34 | for i := range buf { 35 | binary.LittleEndian.PutUint32(buf[i][:], uint32(rand.Int())) 36 | } 37 | 38 | b.ResetTimer() 39 | p := New(comparer.DefaultComparer, 0) 40 | for i := range buf { 41 | if err := p.Put(buf[i][:], nil); err != nil { 42 | b.Fatal(err) 43 | } 44 | } 45 | } 46 | 47 | func BenchmarkGet(b *testing.B) { 48 | buf := make([][4]byte, b.N) 49 | for i := range buf { 50 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 51 | } 52 | 53 | p := New(comparer.DefaultComparer, 0) 54 | for i := range buf { 55 | if err := p.Put(buf[i][:], nil); err != nil { 56 | b.Fatal(err) 57 | } 58 | } 59 | 60 | b.ResetTimer() 61 | for i := range buf { 62 | if _, err := p.Get(buf[i][:]); err != nil { 63 | b.Fatal(err) 64 | } 65 | } 66 | } 67 | 68 | func BenchmarkGetRandom(b *testing.B) { 69 | buf := make([][4]byte, b.N) 70 | for i := range buf { 71 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 72 | } 73 | 74 | p := New(comparer.DefaultComparer, 0) 75 | for i := range buf { 76 | if err := p.Put(buf[i][:], nil); err != nil { 77 | b.Fatal(err) 78 | } 79 | } 80 | 81 | b.ResetTimer() 82 | for i := 0; i < b.N; i++ { 83 | if _, err := p.Get(buf[rand.Int()%b.N][:]); err != nil { 84 | b.Fatal(err) 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /leveldb/memdb/memdb_suite_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/syndtr/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestMemDB(t *testing.T) { 10 | testutil.RunSuite(t, "MemDB Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/memdb/memdb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package memdb 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/comparer" 14 | "github.com/syndtr/goleveldb/leveldb/iterator" 15 | "github.com/syndtr/goleveldb/leveldb/testutil" 16 | "github.com/syndtr/goleveldb/leveldb/util" 17 | ) 18 | 19 | func (p *DB) TestFindLT(key []byte) (rkey, value []byte, err error) { 20 | p.mu.RLock() 21 | if node := p.findLT(key); node != 0 { 22 | n := p.nodeData[node] 23 | m := n + p.nodeData[node+nKey] 24 | rkey = p.kvData[n:m] 25 | value = p.kvData[m : m+p.nodeData[node+nVal]] 26 | } else { 27 | err = ErrNotFound 28 | } 29 | p.mu.RUnlock() 30 | return 31 | } 32 | 33 | func (p *DB) TestFindLast() (rkey, value []byte, err error) { 34 | p.mu.RLock() 35 | if node := p.findLast(); node != 0 { 36 | n := p.nodeData[node] 37 | m := n + p.nodeData[node+nKey] 38 | rkey = p.kvData[n:m] 39 | value = p.kvData[m : m+p.nodeData[node+nVal]] 40 | } else { 41 | err = ErrNotFound 42 | } 43 | p.mu.RUnlock() 44 | return 45 | } 46 | 47 | func (p *DB) TestPut(key []byte, value []byte) error { 48 | return p.Put(key, value) 49 | } 50 | 51 | func (p *DB) TestDelete(key []byte) error { 52 | return p.Delete(key) 53 | } 54 | 55 | func (p *DB) TestFind(key []byte) (rkey, rvalue []byte, err error) { 56 | return p.Find(key) 57 | } 58 | 59 | func (p *DB) TestGet(key []byte) (value []byte, err error) { 60 | return p.Get(key) 61 | } 62 | 63 | func (p *DB) TestNewIterator(slice *util.Range) iterator.Iterator { 64 | return p.NewIterator(slice) 65 | } 66 | 67 | var _ = testutil.Defer(func() { 68 | Describe("Memdb", func() { 69 | Describe("write test", func() { 70 | It("should do write correctly", func() { 71 | db := New(comparer.DefaultComparer, 0) 72 | t := testutil.DBTesting{ 73 | DB: db, 74 | Deleted: testutil.KeyValue_Generate(nil, 1000, 1, 1, 30, 5, 5).Clone(), 75 | PostFn: func(t *testutil.DBTesting) { 76 | Expect(db.Len()).Should(Equal(t.Present.Len())) 77 | Expect(db.Size()).Should(Equal(t.Present.Size())) 78 | switch t.Act { 79 | case testutil.DBPut, testutil.DBOverwrite: 80 | Expect(db.Contains(t.ActKey)).Should(BeTrue()) 81 | default: 82 | Expect(db.Contains(t.ActKey)).Should(BeFalse()) 83 | } 84 | }, 85 | } 86 | testutil.DoDBTesting(&t) 87 | }) 88 | }) 89 | 90 | Describe("read test", func() { 91 | testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB { 92 | // Building the DB. 93 | db := New(comparer.DefaultComparer, 0) 94 | kv.IterateShuffled(nil, func(i int, key, value []byte) { 95 | Expect(db.Put(key, value)).ShouldNot(HaveOccurred()) 96 | }) 97 | 98 | if kv.Len() > 1 { 99 | It("Should find correct keys with findLT", func() { 100 | testutil.ShuffledIndex(nil, kv.Len()-1, 1, func(i int) { 101 | key_, key, _ := kv.IndexInexact(i + 1) 102 | expectedKey, expectedValue := kv.Index(i) 103 | 104 | // Using key that exist. 105 | rkey, rvalue, err := db.TestFindLT(key) 106 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q -> %q", key, expectedKey) 107 | Expect(rkey).Should(Equal(expectedKey), "Key") 108 | Expect(rvalue).Should(Equal(expectedValue), "Value for key %q -> %q", key, expectedKey) 109 | 110 | // Using key that doesn't exist. 111 | rkey, rvalue, err = db.TestFindLT(key_) 112 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q) -> %q", key_, key, expectedKey) 113 | Expect(rkey).Should(Equal(expectedKey)) 114 | Expect(rvalue).Should(Equal(expectedValue), "Value for key %q (%q) -> %q", key_, key, expectedKey) 115 | }) 116 | }) 117 | } 118 | 119 | if kv.Len() > 0 { 120 | It("Should find last key with findLast", func() { 121 | key, value := kv.Index(kv.Len() - 1) 122 | rkey, rvalue, err := db.TestFindLast() 123 | Expect(err).ShouldNot(HaveOccurred()) 124 | Expect(rkey).Should(Equal(key)) 125 | Expect(rvalue).Should(Equal(value)) 126 | }) 127 | } 128 | 129 | return db 130 | }, nil, nil) 131 | }) 132 | }) 133 | }) 134 | -------------------------------------------------------------------------------- /leveldb/opt/options_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | package opt 5 | 6 | var ( 7 | DefaultOpenFilesCacheCapacity = 200 8 | ) 9 | -------------------------------------------------------------------------------- /leveldb/opt/options_default.go: -------------------------------------------------------------------------------- 1 | //go:build !darwin 2 | // +build !darwin 3 | 4 | package opt 5 | 6 | var ( 7 | DefaultOpenFilesCacheCapacity = 500 8 | ) 9 | -------------------------------------------------------------------------------- /leveldb/options.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/filter" 11 | "github.com/syndtr/goleveldb/leveldb/opt" 12 | ) 13 | 14 | func dupOptions(o *opt.Options) *opt.Options { 15 | newo := &opt.Options{} 16 | if o != nil { 17 | *newo = *o 18 | } 19 | if newo.Strict == 0 { 20 | newo.Strict = opt.DefaultStrict 21 | } 22 | return newo 23 | } 24 | 25 | func (s *session) setOptions(o *opt.Options) { 26 | no := dupOptions(o) 27 | // Alternative filters. 28 | if filters := o.GetAltFilters(); len(filters) > 0 { 29 | no.AltFilters = make([]filter.Filter, len(filters)) 30 | for i, filter := range filters { 31 | no.AltFilters[i] = &iFilter{filter} 32 | } 33 | } 34 | // Comparer. 35 | s.icmp = &iComparer{o.GetComparer()} 36 | no.Comparer = s.icmp 37 | // Filter. 38 | if filter := o.GetFilter(); filter != nil { 39 | no.Filter = &iFilter{filter} 40 | } 41 | 42 | s.o = &cachedOptions{Options: no} 43 | s.o.cache() 44 | } 45 | 46 | const optCachedLevel = 7 47 | 48 | type cachedOptions struct { 49 | *opt.Options 50 | 51 | compactionExpandLimit []int 52 | compactionGPOverlaps []int 53 | compactionSourceLimit []int 54 | compactionTableSize []int 55 | compactionTotalSize []int64 56 | } 57 | 58 | func (co *cachedOptions) cache() { 59 | co.compactionExpandLimit = make([]int, optCachedLevel) 60 | co.compactionGPOverlaps = make([]int, optCachedLevel) 61 | co.compactionSourceLimit = make([]int, optCachedLevel) 62 | co.compactionTableSize = make([]int, optCachedLevel) 63 | co.compactionTotalSize = make([]int64, optCachedLevel) 64 | 65 | for level := 0; level < optCachedLevel; level++ { 66 | co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level) 67 | co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level) 68 | co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level) 69 | co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level) 70 | co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level) 71 | } 72 | } 73 | 74 | func (co *cachedOptions) GetCompactionExpandLimit(level int) int { 75 | if level < optCachedLevel { 76 | return co.compactionExpandLimit[level] 77 | } 78 | return co.Options.GetCompactionExpandLimit(level) 79 | } 80 | 81 | func (co *cachedOptions) GetCompactionGPOverlaps(level int) int { 82 | if level < optCachedLevel { 83 | return co.compactionGPOverlaps[level] 84 | } 85 | return co.Options.GetCompactionGPOverlaps(level) 86 | } 87 | 88 | func (co *cachedOptions) GetCompactionSourceLimit(level int) int { 89 | if level < optCachedLevel { 90 | return co.compactionSourceLimit[level] 91 | } 92 | return co.Options.GetCompactionSourceLimit(level) 93 | } 94 | 95 | func (co *cachedOptions) GetCompactionTableSize(level int) int { 96 | if level < optCachedLevel { 97 | return co.compactionTableSize[level] 98 | } 99 | return co.Options.GetCompactionTableSize(level) 100 | } 101 | 102 | func (co *cachedOptions) GetCompactionTotalSize(level int) int64 { 103 | if level < optCachedLevel { 104 | return co.compactionTotalSize[level] 105 | } 106 | return co.Options.GetCompactionTotalSize(level) 107 | } 108 | -------------------------------------------------------------------------------- /leveldb/session.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "io" 12 | "os" 13 | "sync" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/errors" 16 | "github.com/syndtr/goleveldb/leveldb/journal" 17 | "github.com/syndtr/goleveldb/leveldb/opt" 18 | "github.com/syndtr/goleveldb/leveldb/storage" 19 | ) 20 | 21 | // ErrManifestCorrupted records manifest corruption. This error will be 22 | // wrapped with errors.ErrCorrupted. 23 | type ErrManifestCorrupted struct { 24 | Field string 25 | Reason string 26 | } 27 | 28 | func (e *ErrManifestCorrupted) Error() string { 29 | return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason) 30 | } 31 | 32 | func newErrManifestCorrupted(fd storage.FileDesc, field, reason string) error { 33 | return errors.NewErrCorrupted(fd, &ErrManifestCorrupted{field, reason}) 34 | } 35 | 36 | // session represent a persistent database session. 37 | type session struct { 38 | // Need 64-bit alignment. 39 | stNextFileNum int64 // current unused file number 40 | stJournalNum int64 // current journal file number; need external synchronization 41 | stPrevJournalNum int64 // prev journal file number; no longer used; for compatibility with older version of leveldb 42 | stTempFileNum int64 43 | stSeqNum uint64 // last mem compacted seq; need external synchronization 44 | 45 | stor *iStorage 46 | storLock storage.Locker 47 | o *cachedOptions 48 | icmp *iComparer 49 | tops *tOps 50 | 51 | manifest *journal.Writer 52 | manifestWriter storage.Writer 53 | manifestFd storage.FileDesc 54 | 55 | stCompPtrs []internalKey // compaction pointers; need external synchronization 56 | stVersion *version // current version 57 | ntVersionID int64 // next version id to assign 58 | refCh chan *vTask 59 | relCh chan *vTask 60 | deltaCh chan *vDelta 61 | abandon chan int64 62 | closeC chan struct{} 63 | closeW sync.WaitGroup 64 | vmu sync.Mutex 65 | 66 | // Testing fields 67 | fileRefCh chan chan map[int64]int // channel used to pass current reference stat 68 | } 69 | 70 | // Creates new initialized session instance. 71 | func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { 72 | if stor == nil { 73 | return nil, os.ErrInvalid 74 | } 75 | storLock, err := stor.Lock() 76 | if err != nil { 77 | return 78 | } 79 | s = &session{ 80 | stor: newIStorage(stor), 81 | storLock: storLock, 82 | refCh: make(chan *vTask), 83 | relCh: make(chan *vTask), 84 | deltaCh: make(chan *vDelta), 85 | abandon: make(chan int64), 86 | fileRefCh: make(chan chan map[int64]int), 87 | closeC: make(chan struct{}), 88 | } 89 | s.setOptions(o) 90 | s.tops = newTableOps(s) 91 | 92 | s.closeW.Add(1) 93 | go s.refLoop() 94 | s.setVersion(nil, newVersion(s)) 95 | s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") 96 | return 97 | } 98 | 99 | // Close session. 100 | func (s *session) close() { 101 | s.tops.close() 102 | if s.manifest != nil { 103 | s.manifest.Close() 104 | } 105 | if s.manifestWriter != nil { 106 | s.manifestWriter.Close() 107 | } 108 | s.manifest = nil 109 | s.manifestWriter = nil 110 | s.setVersion(nil, &version{s: s, closing: true, id: s.ntVersionID}) 111 | 112 | // Close all background goroutines 113 | close(s.closeC) 114 | s.closeW.Wait() 115 | } 116 | 117 | // Release session lock. 118 | func (s *session) release() { 119 | s.storLock.Unlock() 120 | } 121 | 122 | // Create a new database session; need external synchronization. 123 | func (s *session) create() error { 124 | // create manifest 125 | return s.newManifest(nil, nil) 126 | } 127 | 128 | // Recover a database session; need external synchronization. 129 | func (s *session) recover() (err error) { 130 | defer func() { 131 | if os.IsNotExist(err) { 132 | // Don't return os.ErrNotExist if the underlying storage contains 133 | // other files that belong to LevelDB. So the DB won't get trashed. 134 | if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 { 135 | err = &errors.ErrCorrupted{Err: errors.New("database entry point either missing or corrupted")} 136 | } 137 | } 138 | }() 139 | 140 | fd, err := s.stor.GetMeta() 141 | if err != nil { 142 | return 143 | } 144 | 145 | reader, err := s.stor.Open(fd) 146 | if err != nil { 147 | return 148 | } 149 | defer reader.Close() 150 | 151 | var ( 152 | // Options. 153 | strict = s.o.GetStrict(opt.StrictManifest) 154 | 155 | jr = journal.NewReader(reader, dropper{s, fd}, strict, true) 156 | rec = &sessionRecord{} 157 | staging = s.stVersion.newStaging() 158 | ) 159 | for { 160 | var r io.Reader 161 | r, err = jr.Next() 162 | if err != nil { 163 | if err == io.EOF { 164 | err = nil 165 | break 166 | } 167 | return errors.SetFd(err, fd) 168 | } 169 | 170 | err = rec.decode(r) 171 | if err == nil { 172 | // save compact pointers 173 | for _, r := range rec.compPtrs { 174 | s.setCompPtr(r.level, r.ikey) 175 | } 176 | // commit record to version staging 177 | staging.commit(rec) 178 | } else { 179 | err = errors.SetFd(err, fd) 180 | if strict || !errors.IsCorrupted(err) { 181 | return 182 | } 183 | s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd)) 184 | } 185 | rec.resetCompPtrs() 186 | rec.resetAddedTables() 187 | rec.resetDeletedTables() 188 | } 189 | 190 | switch { 191 | case !rec.has(recComparer): 192 | return newErrManifestCorrupted(fd, "comparer", "missing") 193 | case rec.comparer != s.icmp.uName(): 194 | return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer)) 195 | case !rec.has(recNextFileNum): 196 | return newErrManifestCorrupted(fd, "next-file-num", "missing") 197 | case !rec.has(recJournalNum): 198 | return newErrManifestCorrupted(fd, "journal-file-num", "missing") 199 | case !rec.has(recSeqNum): 200 | return newErrManifestCorrupted(fd, "seq-num", "missing") 201 | } 202 | 203 | s.manifestFd = fd 204 | s.setVersion(rec, staging.finish(false)) 205 | s.setNextFileNum(rec.nextFileNum) 206 | s.recordCommited(rec) 207 | return nil 208 | } 209 | 210 | // Commit session; need external synchronization. 211 | func (s *session) commit(r *sessionRecord, trivial bool) (err error) { 212 | v := s.version() 213 | defer v.release() 214 | 215 | // spawn new version based on current version 216 | nv := v.spawn(r, trivial) 217 | 218 | // abandon useless version id to prevent blocking version processing loop. 219 | defer func() { 220 | if err != nil { 221 | s.abandon <- nv.id 222 | s.logf("commit@abandon useless vid D%d", nv.id) 223 | } 224 | }() 225 | 226 | if s.manifest == nil { 227 | // manifest journal writer not yet created, create one 228 | err = s.newManifest(r, nv) 229 | } else if s.manifest.Size() >= s.o.GetMaxManifestFileSize() { 230 | // pass nil sessionRecord to avoid over-reference table file 231 | err = s.newManifest(nil, nv) 232 | } else { 233 | err = s.flushManifest(r) 234 | } 235 | 236 | // finally, apply new version if no error rise 237 | if err == nil { 238 | s.setVersion(r, nv) 239 | } 240 | 241 | return 242 | } 243 | -------------------------------------------------------------------------------- /leveldb/session_record.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bufio" 11 | "encoding/binary" 12 | "io" 13 | "strings" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/errors" 16 | "github.com/syndtr/goleveldb/leveldb/storage" 17 | ) 18 | 19 | type byteReader interface { 20 | io.Reader 21 | io.ByteReader 22 | } 23 | 24 | // These numbers are written to disk and should not be changed. 25 | const ( 26 | recComparer = 1 27 | recJournalNum = 2 28 | recNextFileNum = 3 29 | recSeqNum = 4 30 | recCompPtr = 5 31 | recDelTable = 6 32 | recAddTable = 7 33 | // 8 was used for large value refs 34 | recPrevJournalNum = 9 35 | ) 36 | 37 | type cpRecord struct { 38 | level int 39 | ikey internalKey 40 | } 41 | 42 | type atRecord struct { 43 | level int 44 | num int64 45 | size int64 46 | imin internalKey 47 | imax internalKey 48 | } 49 | 50 | type dtRecord struct { 51 | level int 52 | num int64 53 | } 54 | 55 | type sessionRecord struct { 56 | hasRec int 57 | comparer string 58 | journalNum int64 59 | prevJournalNum int64 60 | nextFileNum int64 61 | seqNum uint64 62 | compPtrs []cpRecord 63 | addedTables []atRecord 64 | deletedTables []dtRecord 65 | 66 | scratch [binary.MaxVarintLen64]byte 67 | err error 68 | } 69 | 70 | func (p *sessionRecord) has(rec int) bool { 71 | return p.hasRec&(1< 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | ) 13 | 14 | func decodeEncode(v *sessionRecord) (res bool, err error) { 15 | b := new(bytes.Buffer) 16 | err = v.encode(b) 17 | if err != nil { 18 | return 19 | } 20 | v2 := &sessionRecord{} 21 | err = v.decode(b) 22 | if err != nil { 23 | return 24 | } 25 | b2 := new(bytes.Buffer) 26 | err = v2.encode(b2) 27 | if err != nil { 28 | return 29 | } 30 | return bytes.Equal(b.Bytes(), b2.Bytes()), nil 31 | } 32 | 33 | func TestSessionRecord_EncodeDecode(t *testing.T) { 34 | big := int64(1) << 50 35 | v := &sessionRecord{} 36 | i := int64(0) 37 | test := func() { 38 | res, err := decodeEncode(v) 39 | if err != nil { 40 | t.Fatalf("error when testing encode/decode sessionRecord: %v", err) 41 | } 42 | if !res { 43 | t.Error("encode/decode test failed at iteration:", i) 44 | } 45 | } 46 | 47 | for ; i < 4; i++ { 48 | test() 49 | v.addTable(3, big+300+i, big+400+i, 50 | makeInternalKey(nil, []byte("foo"), uint64(big+500+1), keyTypeVal), 51 | makeInternalKey(nil, []byte("zoo"), uint64(big+600+1), keyTypeDel)) 52 | v.delTable(4, big+700+i) 53 | v.addCompPtr(int(i), makeInternalKey(nil, []byte("x"), uint64(big+900+1), keyTypeVal)) 54 | } 55 | 56 | v.setComparer("foo") 57 | v.setJournalNum(big + 100) 58 | v.setPrevJournalNum(big + 99) 59 | v.setNextFileNum(big + 200) 60 | v.setSeqNum(uint64(big + 1000)) 61 | test() 62 | } 63 | -------------------------------------------------------------------------------- /leveldb/storage.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "github.com/syndtr/goleveldb/leveldb/storage" 5 | "sync/atomic" 6 | ) 7 | 8 | type iStorage struct { 9 | storage.Storage 10 | read uint64 11 | write uint64 12 | } 13 | 14 | func (c *iStorage) Open(fd storage.FileDesc) (storage.Reader, error) { 15 | r, err := c.Storage.Open(fd) 16 | return &iStorageReader{r, c}, err 17 | } 18 | 19 | func (c *iStorage) Create(fd storage.FileDesc) (storage.Writer, error) { 20 | w, err := c.Storage.Create(fd) 21 | return &iStorageWriter{w, c}, err 22 | } 23 | 24 | func (c *iStorage) reads() uint64 { 25 | return atomic.LoadUint64(&c.read) 26 | } 27 | 28 | func (c *iStorage) writes() uint64 { 29 | return atomic.LoadUint64(&c.write) 30 | } 31 | 32 | // newIStorage returns the given storage wrapped by iStorage. 33 | func newIStorage(s storage.Storage) *iStorage { 34 | return &iStorage{s, 0, 0} 35 | } 36 | 37 | type iStorageReader struct { 38 | storage.Reader 39 | c *iStorage 40 | } 41 | 42 | func (r *iStorageReader) Read(p []byte) (n int, err error) { 43 | n, err = r.Reader.Read(p) 44 | atomic.AddUint64(&r.c.read, uint64(n)) 45 | return n, err 46 | } 47 | 48 | func (r *iStorageReader) ReadAt(p []byte, off int64) (n int, err error) { 49 | n, err = r.Reader.ReadAt(p, off) 50 | atomic.AddUint64(&r.c.read, uint64(n)) 51 | return n, err 52 | } 53 | 54 | type iStorageWriter struct { 55 | storage.Writer 56 | c *iStorage 57 | } 58 | 59 | func (w *iStorageWriter) Write(p []byte) (n int, err error) { 60 | n, err = w.Writer.Write(p) 61 | atomic.AddUint64(&w.c.write, uint64(n)) 62 | return n, err 63 | } 64 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_nacl.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | //go:build nacl 8 | // +build nacl 9 | 10 | package storage 11 | 12 | import ( 13 | "os" 14 | "syscall" 15 | ) 16 | 17 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 18 | return nil, syscall.ENOTSUP 19 | } 20 | 21 | func setFileLock(f *os.File, readOnly, lock bool) error { 22 | return syscall.ENOTSUP 23 | } 24 | 25 | func rename(oldpath, newpath string) error { 26 | return syscall.ENOTSUP 27 | } 28 | 29 | func isErrInvalid(err error) bool { 30 | return false 31 | } 32 | 33 | func syncDir(name string) error { 34 | return syscall.ENOTSUP 35 | } 36 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "os" 11 | ) 12 | 13 | type plan9FileLock struct { 14 | f *os.File 15 | } 16 | 17 | func (fl *plan9FileLock) release() error { 18 | return fl.f.Close() 19 | } 20 | 21 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 22 | var ( 23 | flag int 24 | perm os.FileMode 25 | ) 26 | if readOnly { 27 | flag = os.O_RDONLY 28 | } else { 29 | flag = os.O_RDWR 30 | perm = os.ModeExclusive 31 | } 32 | f, err := os.OpenFile(path, flag, perm) 33 | if os.IsNotExist(err) { 34 | f, err = os.OpenFile(path, flag|os.O_CREATE, perm|0644) 35 | } 36 | if err != nil { 37 | return 38 | } 39 | fl = &plan9FileLock{f: f} 40 | return 41 | } 42 | 43 | func rename(oldpath, newpath string) error { 44 | if _, err := os.Stat(newpath); err == nil { 45 | if err := os.Remove(newpath); err != nil { 46 | return err 47 | } 48 | } 49 | 50 | return os.Rename(oldpath, newpath) 51 | } 52 | 53 | func syncDir(name string) error { 54 | f, err := os.Open(name) 55 | if err != nil { 56 | return err 57 | } 58 | defer f.Close() 59 | if err := f.Sync(); err != nil { 60 | return err 61 | } 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_solaris.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | //go:build solaris 8 | // +build solaris 9 | 10 | package storage 11 | 12 | import ( 13 | "os" 14 | "syscall" 15 | ) 16 | 17 | type unixFileLock struct { 18 | f *os.File 19 | } 20 | 21 | func (fl *unixFileLock) release() error { 22 | if err := setFileLock(fl.f, false, false); err != nil { 23 | return err 24 | } 25 | return fl.f.Close() 26 | } 27 | 28 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 29 | var flag int 30 | if readOnly { 31 | flag = os.O_RDONLY 32 | } else { 33 | flag = os.O_RDWR 34 | } 35 | f, err := os.OpenFile(path, flag, 0) 36 | if os.IsNotExist(err) { 37 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 38 | } 39 | if err != nil { 40 | return 41 | } 42 | err = setFileLock(f, readOnly, true) 43 | if err != nil { 44 | f.Close() 45 | return 46 | } 47 | fl = &unixFileLock{f: f} 48 | return 49 | } 50 | 51 | func setFileLock(f *os.File, readOnly, lock bool) error { 52 | flock := syscall.Flock_t{ 53 | Type: syscall.F_UNLCK, 54 | Start: 0, 55 | Len: 0, 56 | Whence: 1, 57 | } 58 | if lock { 59 | if readOnly { 60 | flock.Type = syscall.F_RDLCK 61 | } else { 62 | flock.Type = syscall.F_WRLCK 63 | } 64 | } 65 | return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock) 66 | } 67 | 68 | func rename(oldpath, newpath string) error { 69 | return os.Rename(oldpath, newpath) 70 | } 71 | 72 | func syncDir(name string) error { 73 | f, err := os.Open(name) 74 | if err != nil { 75 | return err 76 | } 77 | defer f.Close() 78 | if err := f.Sync(); err != nil { 79 | return err 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd 8 | // +build darwin dragonfly freebsd linux netbsd openbsd 9 | 10 | package storage 11 | 12 | import ( 13 | "os" 14 | "syscall" 15 | ) 16 | 17 | type unixFileLock struct { 18 | f *os.File 19 | } 20 | 21 | func (fl *unixFileLock) release() error { 22 | if err := setFileLock(fl.f, false, false); err != nil { 23 | return err 24 | } 25 | return fl.f.Close() 26 | } 27 | 28 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 29 | var flag int 30 | if readOnly { 31 | flag = os.O_RDONLY 32 | } else { 33 | flag = os.O_RDWR 34 | } 35 | f, err := os.OpenFile(path, flag, 0) 36 | if os.IsNotExist(err) { 37 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 38 | } 39 | if err != nil { 40 | return 41 | } 42 | err = setFileLock(f, readOnly, true) 43 | if err != nil { 44 | f.Close() 45 | return 46 | } 47 | fl = &unixFileLock{f: f} 48 | return 49 | } 50 | 51 | func setFileLock(f *os.File, readOnly, lock bool) error { 52 | how := syscall.LOCK_UN 53 | if lock { 54 | if readOnly { 55 | how = syscall.LOCK_SH 56 | } else { 57 | how = syscall.LOCK_EX 58 | } 59 | } 60 | return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB) 61 | } 62 | 63 | func rename(oldpath, newpath string) error { 64 | return os.Rename(oldpath, newpath) 65 | } 66 | 67 | func isErrInvalid(err error) bool { 68 | if err == os.ErrInvalid { 69 | return true 70 | } 71 | // Go < 1.8 72 | if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL { 73 | return true 74 | } 75 | // Go >= 1.8 returns *os.PathError instead 76 | if patherr, ok := err.(*os.PathError); ok && patherr.Err == syscall.EINVAL { 77 | return true 78 | } 79 | return false 80 | } 81 | 82 | func syncDir(name string) error { 83 | // As per fsync manpage, Linux seems to expect fsync on directory, however 84 | // some system don't support this, so we will ignore syscall.EINVAL. 85 | // 86 | // From fsync(2): 87 | // Calling fsync() does not necessarily ensure that the entry in the 88 | // directory containing the file has also reached disk. For that an 89 | // explicit fsync() on a file descriptor for the directory is also needed. 90 | f, err := os.Open(name) 91 | if err != nil { 92 | return err 93 | } 94 | defer f.Close() 95 | if err := f.Sync(); err != nil && !isErrInvalid(err) { 96 | return err 97 | } 98 | return nil 99 | } 100 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "syscall" 11 | "unsafe" 12 | ) 13 | 14 | var ( 15 | modkernel32 = syscall.NewLazyDLL("kernel32.dll") 16 | 17 | procMoveFileExW = modkernel32.NewProc("MoveFileExW") 18 | ) 19 | 20 | const ( 21 | _MOVEFILE_REPLACE_EXISTING = 1 22 | ) 23 | 24 | type windowsFileLock struct { 25 | fd syscall.Handle 26 | } 27 | 28 | func (fl *windowsFileLock) release() error { 29 | return syscall.Close(fl.fd) 30 | } 31 | 32 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 33 | pathp, err := syscall.UTF16PtrFromString(path) 34 | if err != nil { 35 | return 36 | } 37 | var access, shareMode uint32 38 | if readOnly { 39 | access = syscall.GENERIC_READ 40 | shareMode = syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE 41 | } else { 42 | access = syscall.GENERIC_READ | syscall.GENERIC_WRITE 43 | } 44 | fd, err := syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) 45 | if err == syscall.ERROR_FILE_NOT_FOUND { 46 | fd, err = syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) 47 | } 48 | if err != nil { 49 | return 50 | } 51 | fl = &windowsFileLock{fd: fd} 52 | return 53 | } 54 | 55 | func moveFileEx(from *uint16, to *uint16, flags uint32) error { 56 | r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) 57 | if r1 == 0 { 58 | if e1 != 0 { 59 | return error(e1) 60 | } 61 | return syscall.EINVAL 62 | } 63 | return nil 64 | } 65 | 66 | func rename(oldpath, newpath string) error { 67 | from, err := syscall.UTF16PtrFromString(oldpath) 68 | if err != nil { 69 | return err 70 | } 71 | to, err := syscall.UTF16PtrFromString(newpath) 72 | if err != nil { 73 | return err 74 | } 75 | return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING) 76 | } 77 | 78 | func syncDir(name string) error { return nil } 79 | -------------------------------------------------------------------------------- /leveldb/storage/mem_storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "bytes" 11 | "os" 12 | "sync" 13 | ) 14 | 15 | const typeShift = 4 16 | 17 | // Verify at compile-time that typeShift is large enough to cover all FileType 18 | // values by confirming that 0 == 0. 19 | var _ [0]struct{} = [TypeAll >> typeShift]struct{}{} 20 | 21 | type memStorageLock struct { 22 | ms *memStorage 23 | } 24 | 25 | func (lock *memStorageLock) Unlock() { 26 | ms := lock.ms 27 | ms.mu.Lock() 28 | defer ms.mu.Unlock() 29 | if ms.slock == lock { 30 | ms.slock = nil 31 | } 32 | } 33 | 34 | // memStorage is a memory-backed storage. 35 | type memStorage struct { 36 | mu sync.Mutex 37 | slock *memStorageLock 38 | files map[uint64]*memFile 39 | meta FileDesc 40 | } 41 | 42 | // NewMemStorage returns a new memory-backed storage implementation. 43 | func NewMemStorage() Storage { 44 | return &memStorage{ 45 | files: make(map[uint64]*memFile), 46 | } 47 | } 48 | 49 | func (ms *memStorage) Lock() (Locker, error) { 50 | ms.mu.Lock() 51 | defer ms.mu.Unlock() 52 | if ms.slock != nil { 53 | return nil, ErrLocked 54 | } 55 | ms.slock = &memStorageLock{ms: ms} 56 | return ms.slock, nil 57 | } 58 | 59 | func (*memStorage) Log(str string) {} 60 | 61 | func (ms *memStorage) SetMeta(fd FileDesc) error { 62 | if !FileDescOk(fd) { 63 | return ErrInvalidFile 64 | } 65 | 66 | ms.mu.Lock() 67 | ms.meta = fd 68 | ms.mu.Unlock() 69 | return nil 70 | } 71 | 72 | func (ms *memStorage) GetMeta() (FileDesc, error) { 73 | ms.mu.Lock() 74 | defer ms.mu.Unlock() 75 | if ms.meta.Zero() { 76 | return FileDesc{}, os.ErrNotExist 77 | } 78 | return ms.meta, nil 79 | } 80 | 81 | func (ms *memStorage) List(ft FileType) ([]FileDesc, error) { 82 | ms.mu.Lock() 83 | var fds []FileDesc 84 | for x := range ms.files { 85 | fd := unpackFile(x) 86 | if fd.Type&ft != 0 { 87 | fds = append(fds, fd) 88 | } 89 | } 90 | ms.mu.Unlock() 91 | return fds, nil 92 | } 93 | 94 | func (ms *memStorage) Open(fd FileDesc) (Reader, error) { 95 | if !FileDescOk(fd) { 96 | return nil, ErrInvalidFile 97 | } 98 | 99 | ms.mu.Lock() 100 | defer ms.mu.Unlock() 101 | if m, exist := ms.files[packFile(fd)]; exist { 102 | if m.open { 103 | return nil, errFileOpen 104 | } 105 | m.open = true 106 | return &memReader{Reader: bytes.NewReader(m.Bytes()), ms: ms, m: m}, nil 107 | } 108 | return nil, os.ErrNotExist 109 | } 110 | 111 | func (ms *memStorage) Create(fd FileDesc) (Writer, error) { 112 | if !FileDescOk(fd) { 113 | return nil, ErrInvalidFile 114 | } 115 | 116 | x := packFile(fd) 117 | ms.mu.Lock() 118 | defer ms.mu.Unlock() 119 | m, exist := ms.files[x] 120 | if exist { 121 | if m.open { 122 | return nil, errFileOpen 123 | } 124 | m.Reset() 125 | } else { 126 | m = &memFile{} 127 | ms.files[x] = m 128 | } 129 | m.open = true 130 | return &memWriter{memFile: m, ms: ms}, nil 131 | } 132 | 133 | func (ms *memStorage) Remove(fd FileDesc) error { 134 | if !FileDescOk(fd) { 135 | return ErrInvalidFile 136 | } 137 | 138 | x := packFile(fd) 139 | ms.mu.Lock() 140 | defer ms.mu.Unlock() 141 | if _, exist := ms.files[x]; exist { 142 | delete(ms.files, x) 143 | return nil 144 | } 145 | return os.ErrNotExist 146 | } 147 | 148 | func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { 149 | if !FileDescOk(oldfd) || !FileDescOk(newfd) { 150 | return ErrInvalidFile 151 | } 152 | if oldfd == newfd { 153 | return nil 154 | } 155 | 156 | oldx := packFile(oldfd) 157 | newx := packFile(newfd) 158 | ms.mu.Lock() 159 | defer ms.mu.Unlock() 160 | oldm, exist := ms.files[oldx] 161 | if !exist { 162 | return os.ErrNotExist 163 | } 164 | newm, exist := ms.files[newx] 165 | if (exist && newm.open) || oldm.open { 166 | return errFileOpen 167 | } 168 | delete(ms.files, oldx) 169 | ms.files[newx] = oldm 170 | return nil 171 | } 172 | 173 | func (*memStorage) Close() error { return nil } 174 | 175 | type memFile struct { 176 | bytes.Buffer 177 | open bool 178 | } 179 | 180 | type memReader struct { 181 | *bytes.Reader 182 | ms *memStorage 183 | m *memFile 184 | closed bool 185 | } 186 | 187 | func (mr *memReader) Close() error { 188 | mr.ms.mu.Lock() 189 | defer mr.ms.mu.Unlock() 190 | if mr.closed { 191 | return ErrClosed 192 | } 193 | mr.m.open = false 194 | return nil 195 | } 196 | 197 | type memWriter struct { 198 | *memFile 199 | ms *memStorage 200 | closed bool 201 | } 202 | 203 | func (*memWriter) Sync() error { return nil } 204 | 205 | func (mw *memWriter) Close() error { 206 | mw.ms.mu.Lock() 207 | defer mw.ms.mu.Unlock() 208 | if mw.closed { 209 | return ErrClosed 210 | } 211 | mw.memFile.open = false 212 | return nil 213 | } 214 | 215 | func packFile(fd FileDesc) uint64 { 216 | return uint64(fd.Num)<> typeShift)} 221 | } 222 | -------------------------------------------------------------------------------- /leveldb/storage/mem_storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "testing" 13 | ) 14 | 15 | func TestMemStorage(t *testing.T) { 16 | m := NewMemStorage() 17 | 18 | l, err := m.Lock() 19 | if err != nil { 20 | t.Fatal("storage lock failed(1): ", err) 21 | } 22 | _, err = m.Lock() 23 | if err == nil { 24 | t.Fatal("expect error for second storage lock attempt") 25 | } else { 26 | t.Logf("storage lock got error: %s (expected)", err) 27 | } 28 | l.Unlock() 29 | _, err = m.Lock() 30 | if err != nil { 31 | t.Fatal("storage lock failed(2): ", err) 32 | } 33 | 34 | w, err := m.Create(FileDesc{TypeTable, 1}) 35 | if err != nil { 36 | t.Fatal("Storage.Create: ", err) 37 | } 38 | if _, err := w.Write([]byte("abc")); err != nil { 39 | t.Fatal("Storage.Write: ", err) 40 | } 41 | w.Close() 42 | if fds, _ := m.List(TypeAll); len(fds) != 1 { 43 | t.Fatal("invalid GetFiles len") 44 | } 45 | buf := new(bytes.Buffer) 46 | r, err := m.Open(FileDesc{TypeTable, 1}) 47 | if err != nil { 48 | t.Fatal("Open: got error: ", err) 49 | } 50 | if _, err := buf.ReadFrom(r); err != nil { 51 | t.Fatal("ReadFrom: got error: ", err) 52 | } 53 | r.Close() 54 | if got := buf.String(); got != "abc" { 55 | t.Fatalf("Read: invalid value, want=abc got=%s", got) 56 | } 57 | if _, err := m.Open(FileDesc{TypeTable, 1}); err != nil { 58 | t.Fatal("Open: got error: ", err) 59 | } 60 | if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { 61 | t.Fatal("expecting error") 62 | } 63 | if err := m.Remove(FileDesc{TypeTable, 1}); err != nil { 64 | t.Fatal("Remove: got error: ", err) 65 | } 66 | if fds, _ := m.List(TypeAll); len(fds) != 0 { 67 | t.Fatal("invalid GetFiles len", len(fds)) 68 | } 69 | if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { 70 | t.Fatal("expecting error") 71 | } 72 | } 73 | 74 | func TestMemStorageRename(t *testing.T) { 75 | fd1 := FileDesc{Type: TypeTable, Num: 1} 76 | fd2 := FileDesc{Type: TypeTable, Num: 2} 77 | 78 | m := NewMemStorage() 79 | w, err := m.Create(fd1) 80 | if err != nil { 81 | t.Fatalf("Storage.Create: %v", err) 82 | } 83 | 84 | fmt.Fprintf(w, "abc") 85 | w.Close() 86 | 87 | rd, err := m.Open(fd1) 88 | if err != nil { 89 | t.Fatalf("Storage.Open(%v): %v", fd1, err) 90 | } 91 | rd.Close() 92 | 93 | fds, err := m.List(TypeAll) 94 | if err != nil { 95 | t.Fatalf("Storage.List: %v", err) 96 | } 97 | for _, fd := range fds { 98 | if !FileDescOk(fd) { 99 | t.Errorf("Storage.List -> FileDescOk(%q)", fd) 100 | } 101 | } 102 | 103 | err = m.Rename(fd1, fd2) 104 | if err != nil { 105 | t.Fatalf("Storage.Rename: %v", err) 106 | } 107 | 108 | rd, err = m.Open(fd2) 109 | if err != nil { 110 | t.Fatalf("Storage.Open(%v): %v", fd2, err) 111 | } 112 | rd.Close() 113 | 114 | fds, err = m.List(TypeAll) 115 | if err != nil { 116 | t.Fatalf("Storage.List: %v", err) 117 | } 118 | for _, fd := range fds { 119 | if !FileDescOk(fd) { 120 | t.Errorf("Storage.List -> FileDescOk(%q)", fd) 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /leveldb/storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package storage provides storage abstraction for LevelDB. 8 | package storage 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | "io" 14 | ) 15 | 16 | // FileType represent a file type. 17 | type FileType int 18 | 19 | // File types. 20 | const ( 21 | TypeManifest FileType = 1 << iota 22 | TypeJournal 23 | TypeTable 24 | TypeTemp 25 | 26 | TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp 27 | ) 28 | 29 | func (t FileType) String() string { 30 | switch t { 31 | case TypeManifest: 32 | return "manifest" 33 | case TypeJournal: 34 | return "journal" 35 | case TypeTable: 36 | return "table" 37 | case TypeTemp: 38 | return "temp" 39 | } 40 | return fmt.Sprintf("", t) 41 | } 42 | 43 | // Common error. 44 | var ( 45 | ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument") 46 | ErrLocked = errors.New("leveldb/storage: already locked") 47 | ErrClosed = errors.New("leveldb/storage: closed") 48 | ) 49 | 50 | // ErrCorrupted is the type that wraps errors that indicate corruption of 51 | // a file. Package storage has its own type instead of using 52 | // errors.ErrCorrupted to prevent circular import. 53 | type ErrCorrupted struct { 54 | Fd FileDesc 55 | Err error 56 | } 57 | 58 | func isCorrupted(err error) bool { 59 | switch err.(type) { 60 | case *ErrCorrupted: 61 | return true 62 | default: 63 | return false 64 | } 65 | } 66 | 67 | func (e *ErrCorrupted) Error() string { 68 | if !e.Fd.Zero() { 69 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 70 | } 71 | return e.Err.Error() 72 | } 73 | 74 | // Syncer is the interface that wraps basic Sync method. 75 | type Syncer interface { 76 | // Sync commits the current contents of the file to stable storage. 77 | Sync() error 78 | } 79 | 80 | // Reader is the interface that groups the basic Read, Seek, ReadAt and Close 81 | // methods. 82 | type Reader interface { 83 | io.ReadSeeker 84 | io.ReaderAt 85 | io.Closer 86 | } 87 | 88 | // Writer is the interface that groups the basic Write, Sync and Close 89 | // methods. 90 | type Writer interface { 91 | io.WriteCloser 92 | Syncer 93 | } 94 | 95 | // Locker is the interface that wraps Unlock method. 96 | type Locker interface { 97 | Unlock() 98 | } 99 | 100 | // FileDesc is a 'file descriptor'. 101 | type FileDesc struct { 102 | Type FileType 103 | Num int64 104 | } 105 | 106 | func (fd FileDesc) String() string { 107 | switch fd.Type { 108 | case TypeManifest: 109 | return fmt.Sprintf("MANIFEST-%06d", fd.Num) 110 | case TypeJournal: 111 | return fmt.Sprintf("%06d.log", fd.Num) 112 | case TypeTable: 113 | return fmt.Sprintf("%06d.ldb", fd.Num) 114 | case TypeTemp: 115 | return fmt.Sprintf("%06d.tmp", fd.Num) 116 | default: 117 | return fmt.Sprintf("%#x-%d", fd.Type, fd.Num) 118 | } 119 | } 120 | 121 | // Zero returns true if fd == (FileDesc{}). 122 | func (fd FileDesc) Zero() bool { 123 | return fd == (FileDesc{}) 124 | } 125 | 126 | // FileDescOk returns true if fd is a valid 'file descriptor'. 127 | func FileDescOk(fd FileDesc) bool { 128 | switch fd.Type { 129 | case TypeManifest: 130 | case TypeJournal: 131 | case TypeTable: 132 | case TypeTemp: 133 | default: 134 | return false 135 | } 136 | return fd.Num >= 0 137 | } 138 | 139 | // Storage is the storage. A storage instance must be safe for concurrent use. 140 | type Storage interface { 141 | // Lock locks the storage. Any subsequent attempt to call Lock will fail 142 | // until the last lock released. 143 | // Caller should call Unlock method after use. 144 | Lock() (Locker, error) 145 | 146 | // Log logs a string. This is used for logging. 147 | // An implementation may write to a file, stdout or simply do nothing. 148 | Log(str string) 149 | 150 | // SetMeta store 'file descriptor' that can later be acquired using GetMeta 151 | // method. The 'file descriptor' should point to a valid file. 152 | // SetMeta should be implemented in such way that changes should happen 153 | // atomically. 154 | SetMeta(fd FileDesc) error 155 | 156 | // GetMeta returns 'file descriptor' stored in meta. The 'file descriptor' 157 | // can be updated using SetMeta method. 158 | // Returns os.ErrNotExist if meta doesn't store any 'file descriptor', or 159 | // 'file descriptor' point to nonexistent file. 160 | GetMeta() (FileDesc, error) 161 | 162 | // List returns file descriptors that match the given file types. 163 | // The file types may be OR'ed together. 164 | List(ft FileType) ([]FileDesc, error) 165 | 166 | // Open opens file with the given 'file descriptor' read-only. 167 | // Returns os.ErrNotExist error if the file does not exist. 168 | // Returns ErrClosed if the underlying storage is closed. 169 | Open(fd FileDesc) (Reader, error) 170 | 171 | // Create creates file with the given 'file descriptor', truncate if already 172 | // exist and opens write-only. 173 | // Returns ErrClosed if the underlying storage is closed. 174 | Create(fd FileDesc) (Writer, error) 175 | 176 | // Remove removes file with the given 'file descriptor'. 177 | // Returns ErrClosed if the underlying storage is closed. 178 | Remove(fd FileDesc) error 179 | 180 | // Rename renames file from oldfd to newfd. 181 | // Returns ErrClosed if the underlying storage is closed. 182 | Rename(oldfd, newfd FileDesc) error 183 | 184 | // Close closes the storage. 185 | // It is valid to call Close multiple times. Other methods should not be 186 | // called after the storage has been closed. 187 | Close() error 188 | } 189 | -------------------------------------------------------------------------------- /leveldb/table/block_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package table 8 | 9 | import ( 10 | "encoding/binary" 11 | "fmt" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | 16 | "github.com/syndtr/goleveldb/leveldb/comparer" 17 | "github.com/syndtr/goleveldb/leveldb/iterator" 18 | "github.com/syndtr/goleveldb/leveldb/testutil" 19 | "github.com/syndtr/goleveldb/leveldb/util" 20 | ) 21 | 22 | type blockTesting struct { 23 | tr *Reader 24 | b *block 25 | } 26 | 27 | func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator { 28 | return t.tr.newBlockIter(t.b, nil, slice, false) 29 | } 30 | 31 | var _ = testutil.Defer(func() { 32 | Describe("Block", func() { 33 | Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting { 34 | // Building the block. 35 | bw := &blockWriter{ 36 | restartInterval: restartInterval, 37 | scratch: make([]byte, 30), 38 | } 39 | kv.Iterate(func(i int, key, value []byte) { 40 | Expect(bw.append(key, value)).ShouldNot(HaveOccurred()) 41 | }) 42 | Expect(bw.finish()).ShouldNot(HaveOccurred()) 43 | 44 | // Opening the block. 45 | data := bw.buf.Bytes() 46 | restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) 47 | return &blockTesting{ 48 | tr: &Reader{cmp: comparer.DefaultComparer}, 49 | b: &block{ 50 | data: data, 51 | restartsLen: restartsLen, 52 | restartsOffset: len(data) - (restartsLen+1)*4, 53 | }, 54 | } 55 | } 56 | 57 | Describe("read test", func() { 58 | for restartInterval := 1; restartInterval <= 5; restartInterval++ { 59 | Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { 60 | kv := &testutil.KeyValue{} 61 | Text := func() string { 62 | return fmt.Sprintf("and %d keys", kv.Len()) 63 | } 64 | 65 | Test := func() { 66 | // Make block. 67 | br := Build(kv, restartInterval) 68 | // Do testing. 69 | testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil) 70 | } 71 | 72 | Describe(Text(), Test) 73 | 74 | kv.PutString("", "empty") 75 | Describe(Text(), Test) 76 | 77 | kv.PutString("a1", "foo") 78 | Describe(Text(), Test) 79 | 80 | kv.PutString("a2", "v") 81 | Describe(Text(), Test) 82 | 83 | kv.PutString("a3qqwrkks", "hello") 84 | Describe(Text(), Test) 85 | 86 | kv.PutString("a4", "bar") 87 | Describe(Text(), Test) 88 | 89 | kv.PutString("a5111111", "v5") 90 | kv.PutString("a6", "") 91 | kv.PutString("a7", "v7") 92 | kv.PutString("a8", "vvvvvvvvvvvvvvvvvvvvvv8") 93 | kv.PutString("b", "v9") 94 | kv.PutString("c9", "v9") 95 | kv.PutString("c91", "v9") 96 | kv.PutString("d0", "v9") 97 | Describe(Text(), Test) 98 | }) 99 | } 100 | }) 101 | 102 | Describe("out-of-bound slice test", func() { 103 | kv := &testutil.KeyValue{} 104 | kv.PutString("k1", "v1") 105 | kv.PutString("k2", "v2") 106 | kv.PutString("k3abcdefgg", "v3") 107 | kv.PutString("k4", "v4") 108 | kv.PutString("k5", "v5") 109 | for restartInterval := 1; restartInterval <= 5; restartInterval++ { 110 | Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { 111 | // Make block. 112 | bt := Build(kv, restartInterval) 113 | 114 | Test := func(r *util.Range) func(done Done) { 115 | return func(done Done) { 116 | iter := bt.TestNewIterator(r) 117 | Expect(iter.Error()).ShouldNot(HaveOccurred()) 118 | 119 | t := testutil.IteratorTesting{ 120 | KeyValue: kv.Clone(), 121 | Iter: iter, 122 | } 123 | 124 | testutil.DoIteratorTesting(&t) 125 | iter.Release() 126 | done <- true 127 | } 128 | } 129 | 130 | It("Should do iterations and seeks correctly #0", 131 | Test(&util.Range{Start: []byte("k0"), Limit: []byte("k6")}), 2.0) 132 | 133 | It("Should do iterations and seeks correctly #1", 134 | Test(&util.Range{Start: []byte(""), Limit: []byte("zzzzzzz")}), 2.0) 135 | }) 136 | } 137 | }) 138 | }) 139 | }) 140 | -------------------------------------------------------------------------------- /leveldb/table/table.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package table allows read and write sorted key/value. 8 | package table 9 | 10 | import ( 11 | "encoding/binary" 12 | ) 13 | 14 | /* 15 | Table: 16 | 17 | Table is consist of one or more data blocks, an optional filter block 18 | a metaindex block, an index block and a table footer. Metaindex block 19 | is a special block used to keep parameters of the table, such as filter 20 | block name and its block handle. Index block is a special block used to 21 | keep record of data blocks offset and length, index block use one as 22 | restart interval. The key used by index block are the last key of preceding 23 | block, shorter separator of adjacent blocks or shorter successor of the 24 | last key of the last block. Filter block is an optional block contains 25 | sequence of filter data generated by a filter generator. 26 | 27 | Table data structure: 28 | + optional 29 | / 30 | +--------------+--------------+--------------+------+-------+-----------------+-------------+--------+ 31 | | data block 1 | ... | data block n | filter block | metaindex block | index block | footer | 32 | +--------------+--------------+--------------+--------------+-----------------+-------------+--------+ 33 | 34 | Each block followed by a 5-bytes trailer contains compression type and checksum. 35 | 36 | Table block trailer: 37 | 38 | +---------------------------+-------------------+ 39 | | compression type (1-byte) | checksum (4-byte) | 40 | +---------------------------+-------------------+ 41 | 42 | The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression 43 | type also included in the checksum. 44 | 45 | Table footer: 46 | 47 | +------------------- 40-bytes -------------------+ 48 | / \ 49 | +------------------------+--------------------+------+-----------------+ 50 | | metaindex block handle / index block handle / ---- | magic (8-bytes) | 51 | +------------------------+--------------------+------+-----------------+ 52 | 53 | The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/". 54 | 55 | NOTE: All fixed-length integer are little-endian. 56 | */ 57 | 58 | /* 59 | Block: 60 | 61 | Block is consist of one or more key/value entries and a block trailer. 62 | Block entry shares key prefix with its preceding key until a restart 63 | point reached. A block should contains at least one restart point. 64 | First restart point are always zero. 65 | 66 | Block data structure: 67 | 68 | + restart point + restart point (depends on restart interval) 69 | / / 70 | +---------------+---------------+---------------+---------------+---------+ 71 | | block entry 1 | block entry 2 | ... | block entry n | trailer | 72 | +---------------+---------------+---------------+---------------+---------+ 73 | 74 | Key/value entry: 75 | 76 | +---- key len ----+ 77 | / \ 78 | +-------+---------+-----------+---------+--------------------+--------------+----------------+ 79 | | shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) | 80 | +-----------------+---------------------+--------------------+--------------+----------------+ 81 | 82 | Block entry shares key prefix with its preceding key: 83 | Conditions: 84 | restart_interval=2 85 | entry one : key=deck,value=v1 86 | entry two : key=dock,value=v2 87 | entry three: key=duck,value=v3 88 | The entries will be encoded as follow: 89 | 90 | + restart point (offset=0) + restart point (offset=16) 91 | / / 92 | +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ 93 | | 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" | 94 | +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ 95 | \ / \ / \ / 96 | +----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+ 97 | 98 | The block trailer will contains two restart points: 99 | 100 | +------------+-----------+--------+ 101 | | 0 | 16 | 2 | 102 | +------------+-----------+---+----+ 103 | \ / \ 104 | +-- restart points --+ + restart points length 105 | 106 | Block trailer: 107 | 108 | +-- 4-bytes --+ 109 | / \ 110 | +-----------------+-----------------+-----------------+------------------------------+ 111 | | restart point 1 | .... | restart point n | restart points len (4-bytes) | 112 | +-----------------+-----------------+-----------------+------------------------------+ 113 | 114 | 115 | NOTE: All fixed-length integer are little-endian. 116 | */ 117 | 118 | /* 119 | Filter block: 120 | 121 | Filter block consist of one or more filter data and a filter block trailer. 122 | The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg. 123 | 124 | Filter block data structure: 125 | 126 | + offset 1 + offset 2 + offset n + trailer offset 127 | / / / / 128 | +---------------+---------------+---------------+---------+ 129 | | filter data 1 | ... | filter data n | trailer | 130 | +---------------+---------------+---------------+---------+ 131 | 132 | Filter block trailer: 133 | 134 | +- 4-bytes -+ 135 | / \ 136 | +---------------+---------------+---------------+-------------------------------+------------------+ 137 | | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) | 138 | +-------------- +---------------+---------------+-------------------------------+------------------+ 139 | 140 | 141 | NOTE: All fixed-length integer are little-endian. 142 | */ 143 | 144 | const ( 145 | blockTrailerLen = 5 146 | footerLen = 48 147 | 148 | magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb" 149 | 150 | // The block type gives the per-block compression format. 151 | // These constants are part of the file format and should not be changed. 152 | blockTypeNoCompression = 0 153 | blockTypeSnappyCompression = 1 154 | ) 155 | 156 | type blockHandle struct { 157 | offset, length uint64 158 | } 159 | 160 | func decodeBlockHandle(src []byte) (blockHandle, int) { 161 | offset, n := binary.Uvarint(src) 162 | length, m := binary.Uvarint(src[n:]) 163 | if n == 0 || m == 0 { 164 | return blockHandle{}, 0 165 | } 166 | return blockHandle{offset, length}, n + m 167 | } 168 | 169 | func encodeBlockHandle(dst []byte, b blockHandle) int { 170 | n := binary.PutUvarint(dst, b.offset) 171 | m := binary.PutUvarint(dst[n:], b.length) 172 | return n + m 173 | } 174 | -------------------------------------------------------------------------------- /leveldb/table/table_suite_test.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/syndtr/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestTable(t *testing.T) { 10 | testutil.RunSuite(t, "Table Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/table/table_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package table 8 | 9 | import ( 10 | "bytes" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/iterator" 16 | "github.com/syndtr/goleveldb/leveldb/opt" 17 | "github.com/syndtr/goleveldb/leveldb/storage" 18 | "github.com/syndtr/goleveldb/leveldb/testutil" 19 | "github.com/syndtr/goleveldb/leveldb/util" 20 | ) 21 | 22 | type tableWrapper struct { 23 | *Reader 24 | } 25 | 26 | func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) { 27 | return t.Reader.Find(key, false, nil) 28 | } 29 | 30 | func (t tableWrapper) TestGet(key []byte) (value []byte, err error) { 31 | return t.Reader.Get(key, nil) 32 | } 33 | 34 | func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator { 35 | return t.Reader.NewIterator(slice, nil) 36 | } 37 | 38 | var _ = testutil.Defer(func() { 39 | Describe("Table", func() { 40 | Describe("approximate offset test", func() { 41 | var ( 42 | buf = &bytes.Buffer{} 43 | o = &opt.Options{ 44 | BlockSize: 1024, 45 | Compression: opt.NoCompression, 46 | } 47 | ) 48 | 49 | // Building the table. 50 | tw := NewWriter(buf, o, nil, 0) 51 | err := tw.Append([]byte("k01"), []byte("hello")) 52 | Expect(err).ShouldNot(HaveOccurred()) 53 | err = tw.Append([]byte("k02"), []byte("hello2")) 54 | Expect(err).ShouldNot(HaveOccurred()) 55 | err = tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000)) 56 | Expect(err).ShouldNot(HaveOccurred()) 57 | err = tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000)) 58 | Expect(err).ShouldNot(HaveOccurred()) 59 | err = tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000)) 60 | Expect(err).ShouldNot(HaveOccurred()) 61 | err = tw.Append([]byte("k06"), []byte("hello3")) 62 | Expect(err).ShouldNot(HaveOccurred()) 63 | err = tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000)) 64 | Expect(err).ShouldNot(HaveOccurred()) 65 | 66 | err = tw.Close() 67 | 68 | It("Should be able to approximate offset of a key correctly", func() { 69 | Expect(err).ShouldNot(HaveOccurred()) 70 | 71 | tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) 72 | Expect(err).ShouldNot(HaveOccurred()) 73 | CheckOffset := func(key string, expect, threshold int) { 74 | offset, err := tr.OffsetOf([]byte(key)) 75 | Expect(err).ShouldNot(HaveOccurred()) 76 | Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key) 77 | } 78 | 79 | CheckOffset("k0", 0, 0) 80 | CheckOffset("k01a", 0, 0) 81 | CheckOffset("k02", 0, 0) 82 | CheckOffset("k03", 0, 0) 83 | CheckOffset("k04", 10000, 1000) 84 | CheckOffset("k04a", 210000, 1000) 85 | CheckOffset("k05", 210000, 1000) 86 | CheckOffset("k06", 510000, 1000) 87 | CheckOffset("k07", 510000, 1000) 88 | CheckOffset("xyz", 610000, 2000) 89 | }) 90 | }) 91 | 92 | Describe("read test", func() { 93 | Build := func(kv testutil.KeyValue) testutil.DB { 94 | o := &opt.Options{ 95 | BlockSize: 512, 96 | BlockRestartInterval: 3, 97 | } 98 | buf := &bytes.Buffer{} 99 | 100 | // Building the table. 101 | tw := NewWriter(buf, o, nil, 0) 102 | kv.Iterate(func(i int, key, value []byte) { 103 | Expect(tw.Append(key, value)).ShouldNot(HaveOccurred()) 104 | }) 105 | tw.Close() 106 | 107 | // Opening the table. 108 | tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) 109 | return tableWrapper{tr} 110 | } 111 | Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() { 112 | return func() { 113 | db := Build(*kv) 114 | if body != nil { 115 | body(db.(tableWrapper).Reader) 116 | } 117 | testutil.KeyValueTesting(nil, *kv, db, nil, nil) 118 | } 119 | } 120 | 121 | testutil.AllKeyValueTesting(nil, Build, nil, nil) 122 | Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 1, 10, 512, 512), func(r *Reader) { 123 | It("should have correct blocks number", func() { 124 | indexBlock, err := r.readBlock(r.indexBH, true) 125 | Expect(err).To(BeNil()) 126 | Expect(indexBlock.restartsLen).Should(Equal(9)) 127 | }) 128 | })) 129 | }) 130 | }) 131 | }) 132 | -------------------------------------------------------------------------------- /leveldb/table_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "encoding/binary" 11 | "math/rand" 12 | "reflect" 13 | "testing" 14 | 15 | "github.com/onsi/gomega" 16 | "github.com/syndtr/goleveldb/leveldb/storage" 17 | "github.com/syndtr/goleveldb/leveldb/testutil" 18 | ) 19 | 20 | func TestGetOverlaps(t *testing.T) { 21 | gomega.RegisterTestingT(t) 22 | stor := testutil.NewStorage() 23 | defer stor.Close() 24 | s, err := newSession(stor, nil) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | v := newVersion(s) 30 | v.newStaging() 31 | 32 | tmp := make([]byte, 4) 33 | mik := func(i uint64, typ keyType, ukey bool) []byte { 34 | if i == 0 { 35 | return nil 36 | } 37 | binary.BigEndian.PutUint32(tmp, uint32(i)) 38 | if ukey { 39 | key := make([]byte, 4) 40 | copy(key, tmp) 41 | return key 42 | } 43 | return []byte(makeInternalKey(nil, tmp, 0, typ)) 44 | } 45 | 46 | rec := &sessionRecord{} 47 | for i, f := range []struct { 48 | min uint64 49 | max uint64 50 | level int 51 | }{ 52 | // Overlapped level 0 files 53 | {1, 8, 0}, 54 | {4, 5, 0}, 55 | {6, 10, 0}, 56 | // Non-overlapped level 1 files 57 | {2, 3, 1}, 58 | {8, 10, 1}, 59 | {13, 13, 1}, 60 | {20, 100, 1}, 61 | } { 62 | rec.addTable(f.level, int64(i), 1, mik(f.min, keyTypeVal, false), mik(f.max, keyTypeVal, false)) 63 | } 64 | vs := v.newStaging() 65 | vs.commit(rec) 66 | v = vs.finish(false) 67 | 68 | for i, x := range []struct { 69 | min uint64 70 | max uint64 71 | level int 72 | expected []int64 73 | }{ 74 | // Level0 cases 75 | {0, 0, 0, []int64{2, 1, 0}}, 76 | {1, 0, 0, []int64{2, 1, 0}}, 77 | {0, 10, 0, []int64{2, 1, 0}}, 78 | {2, 7, 0, []int64{2, 1, 0}}, 79 | 80 | // Level1 cases 81 | {1, 1, 1, nil}, 82 | {0, 100, 1, []int64{3, 4, 5, 6}}, 83 | {5, 0, 1, []int64{4, 5, 6}}, 84 | {5, 4, 1, nil}, // invalid search space 85 | {1, 13, 1, []int64{3, 4, 5}}, 86 | {2, 13, 1, []int64{3, 4, 5}}, 87 | {3, 13, 1, []int64{3, 4, 5}}, 88 | {4, 13, 1, []int64{4, 5}}, 89 | {4, 19, 1, []int64{4, 5}}, 90 | {4, 20, 1, []int64{4, 5, 6}}, 91 | {4, 100, 1, []int64{4, 5, 6}}, 92 | {4, 105, 1, []int64{4, 5, 6}}, 93 | } { 94 | tf := v.levels[x.level] 95 | res := tf.getOverlaps(nil, s.icmp, mik(x.min, keyTypeSeek, true), mik(x.max, keyTypeSeek, true), x.level == 0) 96 | 97 | var fnums []int64 98 | for _, f := range res { 99 | fnums = append(fnums, f.fd.Num) 100 | } 101 | if !reflect.DeepEqual(x.expected, fnums) { 102 | t.Errorf("case %d failed, expected %v, got %v", i, x.expected, fnums) 103 | } 104 | } 105 | } 106 | 107 | func BenchmarkGetOverlapLevel0(b *testing.B) { 108 | benchmarkGetOverlap(b, 0, 500000) 109 | } 110 | 111 | func BenchmarkGetOverlapNonLevel0(b *testing.B) { 112 | benchmarkGetOverlap(b, 1, 500000) 113 | } 114 | 115 | func benchmarkGetOverlap(b *testing.B, level int, size int) { 116 | stor := storage.NewMemStorage() 117 | defer stor.Close() 118 | s, err := newSession(stor, nil) 119 | if err != nil { 120 | b.Fatal(err) 121 | } 122 | 123 | v := newVersion(s) 124 | v.newStaging() 125 | 126 | tmp := make([]byte, 4) 127 | mik := func(i uint64, typ keyType, ukey bool) []byte { 128 | if i == 0 { 129 | return nil 130 | } 131 | binary.BigEndian.PutUint32(tmp, uint32(i)) 132 | if ukey { 133 | key := make([]byte, 4) 134 | copy(key, tmp) 135 | return key 136 | } 137 | return []byte(makeInternalKey(nil, tmp, 0, typ)) 138 | } 139 | 140 | rec := &sessionRecord{} 141 | for i := 1; i <= size; i++ { 142 | min := mik(uint64(2*i), keyTypeVal, false) 143 | max := mik(uint64(2*i+1), keyTypeVal, false) 144 | rec.addTable(level, int64(i), 1, min, max) 145 | } 146 | vs := v.newStaging() 147 | vs.commit(rec) 148 | v = vs.finish(false) 149 | 150 | b.ResetTimer() 151 | b.ReportAllocs() 152 | 153 | for i := 0; i < b.N; i++ { 154 | files := v.levels[level] 155 | start := rand.Intn(size) 156 | end := rand.Intn(size-start) + start 157 | files.getOverlaps(nil, s.icmp, mik(uint64(2*start), keyTypeVal, true), mik(uint64(2*end), keyTypeVal, true), level == 0) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /leveldb/testutil/db.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/errors" 16 | "github.com/syndtr/goleveldb/leveldb/iterator" 17 | "github.com/syndtr/goleveldb/leveldb/util" 18 | ) 19 | 20 | type DB interface{} 21 | 22 | type Put interface { 23 | TestPut(key []byte, value []byte) error 24 | } 25 | 26 | type Delete interface { 27 | TestDelete(key []byte) error 28 | } 29 | 30 | type Find interface { 31 | TestFind(key []byte) (rkey, rvalue []byte, err error) 32 | } 33 | 34 | type Get interface { 35 | TestGet(key []byte) (value []byte, err error) 36 | } 37 | 38 | type Has interface { 39 | TestHas(key []byte) (ret bool, err error) 40 | } 41 | 42 | type NewIterator interface { 43 | TestNewIterator(slice *util.Range) iterator.Iterator 44 | } 45 | 46 | type DBAct int 47 | 48 | func (a DBAct) String() string { 49 | switch a { 50 | case DBNone: 51 | return "none" 52 | case DBPut: 53 | return "put" 54 | case DBOverwrite: 55 | return "overwrite" 56 | case DBDelete: 57 | return "delete" 58 | case DBDeleteNA: 59 | return "delete_na" 60 | } 61 | return "unknown" 62 | } 63 | 64 | const ( 65 | DBNone DBAct = iota 66 | DBPut 67 | DBOverwrite 68 | DBDelete 69 | DBDeleteNA 70 | ) 71 | 72 | type DBTesting struct { 73 | Rand *rand.Rand 74 | DB interface { 75 | Get 76 | Put 77 | Delete 78 | } 79 | PostFn func(t *DBTesting) 80 | Deleted, Present KeyValue 81 | Act, LastAct DBAct 82 | ActKey, LastActKey []byte 83 | } 84 | 85 | func (t *DBTesting) post() { 86 | if t.PostFn != nil { 87 | t.PostFn(t) 88 | } 89 | } 90 | 91 | func (t *DBTesting) setAct(act DBAct, key []byte) { 92 | t.LastAct, t.Act = t.Act, act 93 | t.LastActKey, t.ActKey = t.ActKey, key 94 | } 95 | 96 | func (t *DBTesting) text() string { 97 | return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey) 98 | } 99 | 100 | func (t *DBTesting) Text() string { 101 | return "DBTesting " + t.text() 102 | } 103 | 104 | func (t *DBTesting) TestPresentKV(key, value []byte) { 105 | rvalue, err := t.DB.TestGet(key) 106 | Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text()) 107 | Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text()) 108 | } 109 | 110 | func (t *DBTesting) TestAllPresent() { 111 | t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) { 112 | t.TestPresentKV(key, value) 113 | }) 114 | } 115 | 116 | func (t *DBTesting) TestDeletedKey(key []byte) { 117 | _, err := t.DB.TestGet(key) 118 | Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text()) 119 | } 120 | 121 | func (t *DBTesting) TestAllDeleted() { 122 | t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) { 123 | t.TestDeletedKey(key) 124 | }) 125 | } 126 | 127 | func (t *DBTesting) TestAll() { 128 | dn := t.Deleted.Len() 129 | pn := t.Present.Len() 130 | ShuffledIndex(t.Rand, dn+pn, 1, func(i int) { 131 | if i >= dn { 132 | key, value := t.Present.Index(i - dn) 133 | t.TestPresentKV(key, value) 134 | } else { 135 | t.TestDeletedKey(t.Deleted.KeyAt(i)) 136 | } 137 | }) 138 | } 139 | 140 | func (t *DBTesting) Put(key, value []byte) { 141 | if new := t.Present.PutU(key, value); new { 142 | t.setAct(DBPut, key) 143 | } else { 144 | t.setAct(DBOverwrite, key) 145 | } 146 | t.Deleted.Delete(key) 147 | err := t.DB.TestPut(key, value) 148 | Expect(err).ShouldNot(HaveOccurred(), t.Text()) 149 | t.TestPresentKV(key, value) 150 | t.post() 151 | } 152 | 153 | func (t *DBTesting) PutRandom() bool { 154 | if t.Deleted.Len() > 0 { 155 | i := t.Rand.Intn(t.Deleted.Len()) 156 | key, value := t.Deleted.Index(i) 157 | t.Put(key, value) 158 | return true 159 | } 160 | return false 161 | } 162 | 163 | func (t *DBTesting) Delete(key []byte) { 164 | if exist, value := t.Present.Delete(key); exist { 165 | t.setAct(DBDelete, key) 166 | t.Deleted.PutU(key, value) 167 | } else { 168 | t.setAct(DBDeleteNA, key) 169 | } 170 | err := t.DB.TestDelete(key) 171 | Expect(err).ShouldNot(HaveOccurred(), t.Text()) 172 | t.TestDeletedKey(key) 173 | t.post() 174 | } 175 | 176 | func (t *DBTesting) DeleteRandom() bool { 177 | if t.Present.Len() > 0 { 178 | i := t.Rand.Intn(t.Present.Len()) 179 | t.Delete(t.Present.KeyAt(i)) 180 | return true 181 | } 182 | return false 183 | } 184 | 185 | func (t *DBTesting) RandomAct(round int) { 186 | for i := 0; i < round; i++ { 187 | if t.Rand.Int()%2 == 0 { 188 | t.PutRandom() 189 | } else { 190 | t.DeleteRandom() 191 | } 192 | } 193 | } 194 | 195 | func DoDBTesting(t *DBTesting) { 196 | if t.Rand == nil { 197 | t.Rand = NewRand() 198 | } 199 | 200 | t.DeleteRandom() 201 | t.PutRandom() 202 | t.DeleteRandom() 203 | t.DeleteRandom() 204 | for i := t.Deleted.Len() / 2; i >= 0; i-- { 205 | t.PutRandom() 206 | } 207 | t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10) 208 | 209 | // Additional iterator testing 210 | if db, ok := t.DB.(NewIterator); ok { 211 | iter := db.TestNewIterator(nil) 212 | Expect(iter.Error()).NotTo(HaveOccurred()) 213 | 214 | it := IteratorTesting{ 215 | KeyValue: t.Present, 216 | Iter: iter, 217 | } 218 | 219 | DoIteratorTesting(&it) 220 | iter.Release() 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /leveldb/testutil/ginkgo.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | ) 7 | 8 | func RunSuite(t GinkgoTestingT, name string) { 9 | RunDefer() 10 | 11 | SynchronizedBeforeSuite(func() []byte { 12 | RunDefer("setup") 13 | return nil 14 | }, func(data []byte) {}) 15 | SynchronizedAfterSuite(func() { 16 | RunDefer("teardown") 17 | }, func() {}) 18 | 19 | RegisterFailHandler(Fail) 20 | RunSpecs(t, name) 21 | } 22 | -------------------------------------------------------------------------------- /leveldb/testutil/iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/iterator" 16 | ) 17 | 18 | type IterAct int 19 | 20 | func (a IterAct) String() string { 21 | switch a { 22 | case IterNone: 23 | return "none" 24 | case IterFirst: 25 | return "first" 26 | case IterLast: 27 | return "last" 28 | case IterPrev: 29 | return "prev" 30 | case IterNext: 31 | return "next" 32 | case IterSeek: 33 | return "seek" 34 | case IterSOI: 35 | return "soi" 36 | case IterEOI: 37 | return "eoi" 38 | } 39 | return "unknown" 40 | } 41 | 42 | const ( 43 | IterNone IterAct = iota 44 | IterFirst 45 | IterLast 46 | IterPrev 47 | IterNext 48 | IterSeek 49 | IterSOI 50 | IterEOI 51 | ) 52 | 53 | type IteratorTesting struct { 54 | KeyValue 55 | Iter iterator.Iterator 56 | Rand *rand.Rand 57 | PostFn func(t *IteratorTesting) 58 | Pos int 59 | Act, LastAct IterAct 60 | 61 | once bool 62 | } 63 | 64 | func (t *IteratorTesting) init() { 65 | if !t.once { 66 | t.Pos = -1 67 | t.once = true 68 | } 69 | } 70 | 71 | func (t *IteratorTesting) post() { 72 | if t.PostFn != nil { 73 | t.PostFn(t) 74 | } 75 | } 76 | 77 | func (t *IteratorTesting) setAct(act IterAct) { 78 | t.LastAct, t.Act = t.Act, act 79 | } 80 | 81 | func (t *IteratorTesting) text() string { 82 | return fmt.Sprintf("at pos %d and last action was <%v> -> <%v>", t.Pos, t.LastAct, t.Act) 83 | } 84 | 85 | func (t *IteratorTesting) Text() string { 86 | return "IteratorTesting is " + t.text() 87 | } 88 | 89 | func (t *IteratorTesting) IsFirst() bool { 90 | t.init() 91 | return t.Len() > 0 && t.Pos == 0 92 | } 93 | 94 | func (t *IteratorTesting) IsLast() bool { 95 | t.init() 96 | return t.Len() > 0 && t.Pos == t.Len()-1 97 | } 98 | 99 | func (t *IteratorTesting) TestKV() { 100 | t.init() 101 | key, value := t.Index(t.Pos) 102 | Expect(t.Iter.Key()).NotTo(BeNil()) 103 | Expect(t.Iter.Key()).Should(Equal(key), "Key is invalid, %s", t.text()) 104 | Expect(t.Iter.Value()).Should(Equal(value), "Value for key %q, %s", key, t.text()) 105 | } 106 | 107 | func (t *IteratorTesting) First() { 108 | t.init() 109 | t.setAct(IterFirst) 110 | 111 | ok := t.Iter.First() 112 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 113 | if t.Len() > 0 { 114 | t.Pos = 0 115 | Expect(ok).Should(BeTrue(), t.Text()) 116 | t.TestKV() 117 | } else { 118 | t.Pos = -1 119 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 120 | } 121 | t.post() 122 | } 123 | 124 | func (t *IteratorTesting) Last() { 125 | t.init() 126 | t.setAct(IterLast) 127 | 128 | ok := t.Iter.Last() 129 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 130 | if t.Len() > 0 { 131 | t.Pos = t.Len() - 1 132 | Expect(ok).Should(BeTrue(), t.Text()) 133 | t.TestKV() 134 | } else { 135 | t.Pos = 0 136 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 137 | } 138 | t.post() 139 | } 140 | 141 | func (t *IteratorTesting) Next() { 142 | t.init() 143 | t.setAct(IterNext) 144 | 145 | ok := t.Iter.Next() 146 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 147 | if t.Pos < t.Len()-1 { 148 | t.Pos++ 149 | Expect(ok).Should(BeTrue(), t.Text()) 150 | t.TestKV() 151 | } else { 152 | t.Pos = t.Len() 153 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 154 | } 155 | t.post() 156 | } 157 | 158 | func (t *IteratorTesting) Prev() { 159 | t.init() 160 | t.setAct(IterPrev) 161 | 162 | ok := t.Iter.Prev() 163 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 164 | if t.Pos > 0 { 165 | t.Pos-- 166 | Expect(ok).Should(BeTrue(), t.Text()) 167 | t.TestKV() 168 | } else { 169 | t.Pos = -1 170 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 171 | } 172 | t.post() 173 | } 174 | 175 | func (t *IteratorTesting) Seek(i int) { 176 | t.init() 177 | t.setAct(IterSeek) 178 | 179 | key, _ := t.Index(i) 180 | oldKey, _ := t.IndexOrNil(t.Pos) 181 | 182 | ok := t.Iter.Seek(key) 183 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 184 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q, to pos %d, %s", oldKey, key, i, t.text())) 185 | 186 | t.Pos = i 187 | t.TestKV() 188 | t.post() 189 | } 190 | 191 | func (t *IteratorTesting) SeekInexact(i int) { 192 | t.init() 193 | t.setAct(IterSeek) 194 | var key0 []byte 195 | key1, _ := t.Index(i) 196 | if i > 0 { 197 | key0, _ = t.Index(i - 1) 198 | } 199 | key := BytesSeparator(key0, key1) 200 | oldKey, _ := t.IndexOrNil(t.Pos) 201 | 202 | ok := t.Iter.Seek(key) 203 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 204 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key1, i, t.text())) 205 | 206 | t.Pos = i 207 | t.TestKV() 208 | t.post() 209 | } 210 | 211 | func (t *IteratorTesting) SeekKey(key []byte) { 212 | t.init() 213 | t.setAct(IterSeek) 214 | oldKey, _ := t.IndexOrNil(t.Pos) 215 | i := t.Search(key) 216 | 217 | ok := t.Iter.Seek(key) 218 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 219 | if i < t.Len() { 220 | key_, _ := t.Index(i) 221 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key_, i, t.text())) 222 | t.Pos = i 223 | t.TestKV() 224 | } else { 225 | Expect(ok).ShouldNot(BeTrue(), fmt.Sprintf("Seek from key %q to %q, %s", oldKey, key, t.text())) 226 | } 227 | 228 | t.Pos = i 229 | t.post() 230 | } 231 | 232 | func (t *IteratorTesting) SOI() { 233 | t.init() 234 | t.setAct(IterSOI) 235 | Expect(t.Pos).Should(BeNumerically("<=", 0), t.Text()) 236 | for i := 0; i < 3; i++ { 237 | t.Prev() 238 | } 239 | t.post() 240 | } 241 | 242 | func (t *IteratorTesting) EOI() { 243 | t.init() 244 | t.setAct(IterEOI) 245 | Expect(t.Pos).Should(BeNumerically(">=", t.Len()-1), t.Text()) 246 | for i := 0; i < 3; i++ { 247 | t.Next() 248 | } 249 | t.post() 250 | } 251 | 252 | func (t *IteratorTesting) WalkPrev(fn func(t *IteratorTesting)) { 253 | t.init() 254 | for old := t.Pos; t.Pos > 0; old = t.Pos { 255 | fn(t) 256 | Expect(t.Pos).Should(BeNumerically("<", old), t.Text()) 257 | } 258 | } 259 | 260 | func (t *IteratorTesting) WalkNext(fn func(t *IteratorTesting)) { 261 | t.init() 262 | for old := t.Pos; t.Pos < t.Len()-1; old = t.Pos { 263 | fn(t) 264 | Expect(t.Pos).Should(BeNumerically(">", old), t.Text()) 265 | } 266 | } 267 | 268 | func (t *IteratorTesting) PrevAll() { 269 | t.WalkPrev(func(t *IteratorTesting) { 270 | t.Prev() 271 | }) 272 | } 273 | 274 | func (t *IteratorTesting) NextAll() { 275 | t.WalkNext(func(t *IteratorTesting) { 276 | t.Next() 277 | }) 278 | } 279 | 280 | func DoIteratorTesting(t *IteratorTesting) { 281 | if t.Rand == nil { 282 | t.Rand = NewRand() 283 | } 284 | t.SOI() 285 | t.NextAll() 286 | t.First() 287 | t.SOI() 288 | t.NextAll() 289 | t.EOI() 290 | t.PrevAll() 291 | t.Last() 292 | t.EOI() 293 | t.PrevAll() 294 | t.SOI() 295 | 296 | t.NextAll() 297 | t.PrevAll() 298 | t.NextAll() 299 | t.Last() 300 | t.PrevAll() 301 | t.First() 302 | t.NextAll() 303 | t.EOI() 304 | 305 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 306 | t.Seek(i) 307 | }) 308 | 309 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 310 | t.SeekInexact(i) 311 | }) 312 | 313 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 314 | t.Seek(i) 315 | if i%2 != 0 { 316 | t.PrevAll() 317 | t.SOI() 318 | } else { 319 | t.NextAll() 320 | t.EOI() 321 | } 322 | }) 323 | 324 | for _, key := range []string{"", "foo", "bar", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"} { 325 | t.SeekKey([]byte(key)) 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /leveldb/testutil/kvtest.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | 16 | "github.com/syndtr/goleveldb/leveldb/errors" 17 | "github.com/syndtr/goleveldb/leveldb/util" 18 | ) 19 | 20 | func TestFind(db Find, kv KeyValue) { 21 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 22 | key_, key, value := kv.IndexInexact(i) 23 | 24 | // Using exact key. 25 | rkey, rvalue, err := db.TestFind(key) 26 | Expect(err).ShouldNot(HaveOccurred(), "Error for exact key %q", key) 27 | Expect(rkey).Should(Equal(key), "Key") 28 | Expect(rvalue).Should(Equal(value), "Value for exact key %q", key) 29 | 30 | // Using inexact key. 31 | rkey, rvalue, err = db.TestFind(key_) 32 | Expect(err).ShouldNot(HaveOccurred(), "Error for inexact key %q (%q)", key_, key) 33 | Expect(rkey).Should(Equal(key), "Key for inexact key %q (%q)", key_, key) 34 | Expect(rvalue).Should(Equal(value), "Value for inexact key %q (%q)", key_, key) 35 | }) 36 | } 37 | 38 | func TestFindAfterLast(db Find, kv KeyValue) { 39 | var key []byte 40 | if kv.Len() > 0 { 41 | key_, _ := kv.Index(kv.Len() - 1) 42 | key = BytesAfter(key_) 43 | } 44 | rkey, _, err := db.TestFind(key) 45 | Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey) 46 | Expect(err).Should(Equal(errors.ErrNotFound)) 47 | } 48 | 49 | func TestGet(db Get, kv KeyValue) { 50 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 51 | key_, key, value := kv.IndexInexact(i) 52 | 53 | // Using exact key. 54 | rvalue, err := db.TestGet(key) 55 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) 56 | Expect(rvalue).Should(Equal(value), "Value for key %q", key) 57 | 58 | // Using inexact key. 59 | if len(key_) > 0 { 60 | _, err = db.TestGet(key_) 61 | Expect(err).Should(HaveOccurred(), "Error for key %q", key_) 62 | Expect(err).Should(Equal(errors.ErrNotFound)) 63 | } 64 | }) 65 | } 66 | 67 | func TestHas(db Has, kv KeyValue) { 68 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 69 | key_, key, _ := kv.IndexInexact(i) 70 | 71 | // Using exact key. 72 | ret, err := db.TestHas(key) 73 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) 74 | Expect(ret).Should(BeTrue(), "False for key %q", key) 75 | 76 | // Using inexact key. 77 | if len(key_) > 0 { 78 | ret, err = db.TestHas(key_) 79 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_) 80 | Expect(ret).ShouldNot(BeTrue(), "True for key %q", key) 81 | } 82 | }) 83 | } 84 | 85 | func TestIter(db NewIterator, r *util.Range, kv KeyValue) { 86 | iter := db.TestNewIterator(r) 87 | Expect(iter.Error()).ShouldNot(HaveOccurred()) 88 | 89 | t := IteratorTesting{ 90 | KeyValue: kv, 91 | Iter: iter, 92 | } 93 | 94 | DoIteratorTesting(&t) 95 | iter.Release() 96 | } 97 | 98 | func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) { 99 | if rnd == nil { 100 | rnd = NewRand() 101 | } 102 | 103 | if p == nil { 104 | BeforeEach(func() { 105 | p = setup(kv) 106 | }) 107 | if teardown != nil { 108 | AfterEach(func() { 109 | teardown(p) 110 | }) 111 | } 112 | } 113 | 114 | It("Should find all keys with Find", func() { 115 | if db, ok := p.(Find); ok { 116 | TestFind(db, kv) 117 | } 118 | }) 119 | 120 | It("Should return error if Find on key after the last", func() { 121 | if db, ok := p.(Find); ok { 122 | TestFindAfterLast(db, kv) 123 | } 124 | }) 125 | 126 | It("Should only find exact key with Get", func() { 127 | if db, ok := p.(Get); ok { 128 | TestGet(db, kv) 129 | } 130 | }) 131 | 132 | It("Should only find present key with Has", func() { 133 | if db, ok := p.(Has); ok { 134 | TestHas(db, kv) 135 | } 136 | }) 137 | 138 | It("Should iterates and seeks correctly", func(done Done) { 139 | if db, ok := p.(NewIterator); ok { 140 | TestIter(db, nil, kv.Clone()) 141 | } 142 | done <- true 143 | }, 30.0) 144 | 145 | It("Should iterates and seeks slice correctly", func(done Done) { 146 | if db, ok := p.(NewIterator); ok { 147 | RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) { 148 | type slice struct { 149 | r *util.Range 150 | start, limit int 151 | } 152 | 153 | key_, _, _ := kv.IndexInexact(i) 154 | for _, x := range []slice{ 155 | {&util.Range{Start: key_, Limit: nil}, i, kv.Len()}, 156 | {&util.Range{Start: nil, Limit: key_}, 0, i}, 157 | } { 158 | By(fmt.Sprintf("Random index of %d .. %d", x.start, x.limit), func() { 159 | TestIter(db, x.r, kv.Slice(x.start, x.limit)) 160 | }) 161 | } 162 | }) 163 | } 164 | done <- true 165 | }, 200.0) 166 | 167 | It("Should iterates and seeks slice correctly", func(done Done) { 168 | if db, ok := p.(NewIterator); ok { 169 | RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) { 170 | By(fmt.Sprintf("Random range of %d .. %d", start, limit), func() { 171 | r := kv.Range(start, limit) 172 | TestIter(db, &r, kv.Slice(start, limit)) 173 | }) 174 | }) 175 | } 176 | done <- true 177 | }, 200.0) 178 | } 179 | 180 | func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) { 181 | Test := func(kv *KeyValue) func() { 182 | return func() { 183 | var p DB 184 | if setup != nil { 185 | Defer("setup", func() { 186 | p = setup(*kv) 187 | }) 188 | } 189 | if teardown != nil { 190 | Defer("teardown", func() { 191 | teardown(p) 192 | }) 193 | } 194 | if body != nil { 195 | p = body(*kv) 196 | } 197 | KeyValueTesting(rnd, *kv, p, func(KeyValue) DB { 198 | return p 199 | }, nil) 200 | } 201 | } 202 | 203 | Describe("with no key/value (empty)", Test(&KeyValue{})) 204 | Describe("with empty key", Test(KeyValue_EmptyKey())) 205 | Describe("with empty value", Test(KeyValue_EmptyValue())) 206 | Describe("with one key/value", Test(KeyValue_OneKeyValue())) 207 | Describe("with big value", Test(KeyValue_BigValue())) 208 | Describe("with special key", Test(KeyValue_SpecialKey())) 209 | Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue())) 210 | Describe("with generated key/value 2-incr", Test(KeyValue_Generate(nil, 120, 2, 1, 50, 10, 120))) 211 | Describe("with generated key/value 3-incr", Test(KeyValue_Generate(nil, 120, 3, 1, 50, 10, 120))) 212 | } 213 | -------------------------------------------------------------------------------- /leveldb/testutil/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "bytes" 11 | "flag" 12 | "math/rand" 13 | "reflect" 14 | "sync" 15 | 16 | "github.com/onsi/ginkgo/config" 17 | 18 | "github.com/syndtr/goleveldb/leveldb/comparer" 19 | ) 20 | 21 | var ( 22 | runfn = make(map[string][]func()) 23 | runmu sync.Mutex 24 | ) 25 | 26 | func Defer(args ...interface{}) bool { 27 | var ( 28 | group string 29 | fn func() 30 | ) 31 | for _, arg := range args { 32 | v := reflect.ValueOf(arg) 33 | switch v.Kind() { 34 | case reflect.String: 35 | group = v.String() 36 | case reflect.Func: 37 | r := reflect.ValueOf(&fn).Elem() 38 | r.Set(v) 39 | } 40 | } 41 | if fn != nil { 42 | runmu.Lock() 43 | runfn[group] = append(runfn[group], fn) 44 | runmu.Unlock() 45 | } 46 | return true 47 | } 48 | 49 | func RunDefer(groups ...string) bool { 50 | if len(groups) == 0 { 51 | groups = append(groups, "") 52 | } 53 | runmu.Lock() 54 | var runfn_ []func() 55 | for _, group := range groups { 56 | runfn_ = append(runfn_, runfn[group]...) 57 | delete(runfn, group) 58 | } 59 | runmu.Unlock() 60 | for _, fn := range runfn_ { 61 | fn() 62 | } 63 | return runfn_ != nil 64 | } 65 | 66 | func RandomSeed() int64 { 67 | if !flag.Parsed() { 68 | panic("random seed not initialized") 69 | } 70 | return config.GinkgoConfig.RandomSeed 71 | } 72 | 73 | func NewRand() *rand.Rand { 74 | return rand.New(rand.NewSource(RandomSeed())) 75 | } 76 | 77 | var cmp = comparer.DefaultComparer 78 | 79 | func BytesSeparator(a, b []byte) []byte { 80 | if bytes.Equal(a, b) { 81 | return b 82 | } 83 | i, n := 0, len(a) 84 | if n > len(b) { 85 | n = len(b) 86 | } 87 | for ; i < n && (a[i] == b[i]); i++ { 88 | } 89 | x := append([]byte(nil), a[:i]...) 90 | if i < n { 91 | if c := a[i] + 1; c < b[i] { 92 | return append(x, c) 93 | } 94 | x = append(x, a[i]) 95 | i++ 96 | } 97 | for ; i < len(a); i++ { 98 | if c := a[i]; c < 0xff { 99 | return append(x, c+1) 100 | } else { 101 | x = append(x, c) 102 | } 103 | } 104 | if len(b) > i && b[i] > 0 { 105 | return append(x, b[i]-1) 106 | } 107 | return append(x, 'x') 108 | } 109 | 110 | func BytesAfter(b []byte) []byte { 111 | var x []byte 112 | for _, c := range b { 113 | if c < 0xff { 114 | return append(x, c+1) 115 | } 116 | x = append(x, c) 117 | } 118 | return append(x, 'x') 119 | } 120 | 121 | func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) { 122 | if rnd == nil { 123 | rnd = NewRand() 124 | } 125 | for x := 0; x < round; x++ { 126 | fn(rnd.Intn(n)) 127 | } 128 | } 129 | 130 | func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) { 131 | if rnd == nil { 132 | rnd = NewRand() 133 | } 134 | for x := 0; x < round; x++ { 135 | for _, i := range rnd.Perm(n) { 136 | fn(i) 137 | } 138 | } 139 | } 140 | 141 | func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) { 142 | if rnd == nil { 143 | rnd = NewRand() 144 | } 145 | for x := 0; x < round; x++ { 146 | start := rnd.Intn(n) 147 | length := 0 148 | if j := n - start; j > 0 { 149 | length = rnd.Intn(j) 150 | } 151 | fn(start, start+length) 152 | } 153 | } 154 | 155 | func Max(x, y int) int { 156 | if x > y { 157 | return x 158 | } 159 | return y 160 | } 161 | 162 | func Min(x, y int) int { 163 | if x < y { 164 | return x 165 | } 166 | return y 167 | } 168 | -------------------------------------------------------------------------------- /leveldb/testutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | . "github.com/onsi/gomega" 11 | 12 | "github.com/syndtr/goleveldb/leveldb/iterator" 13 | "github.com/syndtr/goleveldb/leveldb/opt" 14 | "github.com/syndtr/goleveldb/leveldb/testutil" 15 | "github.com/syndtr/goleveldb/leveldb/util" 16 | ) 17 | 18 | type testingDB struct { 19 | *DB 20 | ro *opt.ReadOptions 21 | wo *opt.WriteOptions 22 | stor *testutil.Storage 23 | } 24 | 25 | func (t *testingDB) TestPut(key []byte, value []byte) error { 26 | return t.Put(key, value, t.wo) 27 | } 28 | 29 | func (t *testingDB) TestDelete(key []byte) error { 30 | return t.Delete(key, t.wo) 31 | } 32 | 33 | func (t *testingDB) TestGet(key []byte) (value []byte, err error) { 34 | return t.Get(key, t.ro) 35 | } 36 | 37 | func (t *testingDB) TestHas(key []byte) (ret bool, err error) { 38 | return t.Has(key, t.ro) 39 | } 40 | 41 | func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator { 42 | return t.NewIterator(slice, t.ro) 43 | } 44 | 45 | func (t *testingDB) TestClose() { 46 | err := t.Close() 47 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 48 | err = t.stor.Close() 49 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 50 | } 51 | 52 | func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB { 53 | stor := testutil.NewStorage() 54 | db, err := Open(stor, o) 55 | // FIXME: This may be called from outside It, which may cause panic. 56 | Expect(err).NotTo(HaveOccurred()) 57 | return &testingDB{ 58 | DB: db, 59 | ro: ro, 60 | wo: wo, 61 | stor: stor, 62 | } 63 | } 64 | 65 | type testingTransaction struct { 66 | *Transaction 67 | ro *opt.ReadOptions 68 | wo *opt.WriteOptions 69 | } 70 | 71 | func (t *testingTransaction) TestPut(key []byte, value []byte) error { 72 | return t.Put(key, value, t.wo) 73 | } 74 | 75 | func (t *testingTransaction) TestDelete(key []byte) error { 76 | return t.Delete(key, t.wo) 77 | } 78 | 79 | func (t *testingTransaction) TestGet(key []byte) (value []byte, err error) { 80 | return t.Get(key, t.ro) 81 | } 82 | 83 | func (t *testingTransaction) TestHas(key []byte) (ret bool, err error) { 84 | return t.Has(key, t.ro) 85 | } 86 | 87 | func (t *testingTransaction) TestNewIterator(slice *util.Range) iterator.Iterator { 88 | return t.NewIterator(slice, t.ro) 89 | } 90 | 91 | func (t *testingTransaction) TestClose() {} 92 | -------------------------------------------------------------------------------- /leveldb/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "sort" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/storage" 14 | ) 15 | 16 | func shorten(str string) string { 17 | if len(str) <= 8 { 18 | return str 19 | } 20 | return str[:3] + ".." + str[len(str)-3:] 21 | } 22 | 23 | var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"} 24 | 25 | func shortenb(bytes int64) string { 26 | i := 0 27 | for ; bytes > 1024 && i < 4; i++ { 28 | bytes /= 1024 29 | } 30 | return fmt.Sprintf("%d%sB", bytes, bunits[i]) 31 | } 32 | 33 | func sshortenb(bytes int64) string { 34 | if bytes == 0 { 35 | return "~" 36 | } 37 | sign := "+" 38 | if bytes < 0 { 39 | sign = "-" 40 | bytes *= -1 41 | } 42 | i := 0 43 | for ; bytes > 1024 && i < 4; i++ { 44 | bytes /= 1024 45 | } 46 | return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i]) 47 | } 48 | 49 | func sint(x int) string { 50 | if x == 0 { 51 | return "~" 52 | } 53 | sign := "+" 54 | if x < 0 { 55 | sign = "-" 56 | x *= -1 57 | } 58 | return fmt.Sprintf("%s%d", sign, x) 59 | } 60 | 61 | func maxInt(a, b int) int { 62 | if a > b { 63 | return a 64 | } 65 | return b 66 | } 67 | 68 | type fdSorter []storage.FileDesc 69 | 70 | func (p fdSorter) Len() int { 71 | return len(p) 72 | } 73 | 74 | func (p fdSorter) Less(i, j int) bool { 75 | return p[i].Num < p[j].Num 76 | } 77 | 78 | func (p fdSorter) Swap(i, j int) { 79 | p[i], p[j] = p[j], p[i] 80 | } 81 | 82 | func sortFds(fds []storage.FileDesc) { 83 | sort.Sort(fdSorter(fds)) 84 | } 85 | 86 | func ensureBuffer(b []byte, n int) []byte { 87 | if cap(b) < n { 88 | return make([]byte, n) 89 | } 90 | return b[:n] 91 | } 92 | -------------------------------------------------------------------------------- /leveldb/util/buffer_pool.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "fmt" 11 | "sync" 12 | "sync/atomic" 13 | ) 14 | 15 | // BufferPool is a 'buffer pool'. 16 | type BufferPool struct { 17 | pool [6]sync.Pool 18 | baseline [5]int 19 | 20 | get uint32 21 | put uint32 22 | less uint32 23 | equal uint32 24 | greater uint32 25 | miss uint32 26 | } 27 | 28 | func (p *BufferPool) poolNum(n int) int { 29 | for i, x := range p.baseline { 30 | if n <= x { 31 | return i 32 | } 33 | } 34 | return len(p.baseline) 35 | } 36 | 37 | // Get returns buffer with length of n. 38 | func (p *BufferPool) Get(n int) []byte { 39 | if p == nil { 40 | return make([]byte, n) 41 | } 42 | atomic.AddUint32(&p.get, 1) 43 | 44 | poolNum := p.poolNum(n) 45 | 46 | b := p.pool[poolNum].Get().(*[]byte) 47 | 48 | if cap(*b) == 0 { 49 | // If we grabbed nothing, increment the miss stats. 50 | atomic.AddUint32(&p.miss, 1) 51 | 52 | if poolNum == len(p.baseline) { 53 | *b = make([]byte, n) 54 | return *b 55 | } 56 | 57 | *b = make([]byte, p.baseline[poolNum]) 58 | *b = (*b)[:n] 59 | return *b 60 | } else { 61 | // If there is enough capacity in the bytes grabbed, resize the length 62 | // to n and return. 63 | if n < cap(*b) { 64 | atomic.AddUint32(&p.less, 1) 65 | *b = (*b)[:n] 66 | return *b 67 | } else if n == cap(*b) { 68 | atomic.AddUint32(&p.equal, 1) 69 | *b = (*b)[:n] 70 | return *b 71 | } else if n > cap(*b) { 72 | atomic.AddUint32(&p.greater, 1) 73 | } 74 | } 75 | 76 | if poolNum == len(p.baseline) { 77 | *b = make([]byte, n) 78 | return *b 79 | } 80 | *b = make([]byte, p.baseline[poolNum]) 81 | *b = (*b)[:n] 82 | return *b 83 | } 84 | 85 | // Put adds given buffer to the pool. 86 | func (p *BufferPool) Put(b []byte) { 87 | if p == nil { 88 | return 89 | } 90 | 91 | poolNum := p.poolNum(cap(b)) 92 | 93 | atomic.AddUint32(&p.put, 1) 94 | p.pool[poolNum].Put(&b) 95 | } 96 | 97 | func (p *BufferPool) String() string { 98 | if p == nil { 99 | return "" 100 | } 101 | return fmt.Sprintf("BufferPool{B·%d G·%d P·%d <·%d =·%d >·%d M·%d}", 102 | p.baseline, p.get, p.put, p.less, p.equal, p.greater, p.miss) 103 | } 104 | 105 | // NewBufferPool creates a new initialized 'buffer pool'. 106 | func NewBufferPool(baseline int) *BufferPool { 107 | if baseline <= 0 { 108 | panic("baseline can't be <= 0") 109 | } 110 | bufPool := &BufferPool{ 111 | baseline: [...]int{baseline / 4, baseline / 2, baseline, baseline * 2, baseline * 4}, 112 | pool: [6]sync.Pool{ 113 | { 114 | New: func() interface{} { return new([]byte) }, 115 | }, 116 | { 117 | New: func() interface{} { return new([]byte) }, 118 | }, 119 | { 120 | New: func() interface{} { return new([]byte) }, 121 | }, 122 | { 123 | New: func() interface{} { return new([]byte) }, 124 | }, 125 | { 126 | New: func() interface{} { return new([]byte) }, 127 | }, 128 | { 129 | New: func() interface{} { return new([]byte) }, 130 | }, 131 | }, 132 | } 133 | 134 | return bufPool 135 | } 136 | -------------------------------------------------------------------------------- /leveldb/util/crc32.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The LevelDB-Go Authors. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style license that can be 4 | // found in the LICENSE file. 5 | 6 | package util 7 | 8 | import ( 9 | "hash/crc32" 10 | ) 11 | 12 | var table = crc32.MakeTable(crc32.Castagnoli) 13 | 14 | // CRC is a CRC-32 checksum computed using Castagnoli's polynomial. 15 | type CRC uint32 16 | 17 | // NewCRC creates a new crc based on the given bytes. 18 | func NewCRC(b []byte) CRC { 19 | return CRC(0).Update(b) 20 | } 21 | 22 | // Update updates the crc with the given bytes. 23 | func (c CRC) Update(b []byte) CRC { 24 | return CRC(crc32.Update(uint32(c), table, b)) 25 | } 26 | 27 | // Value returns a masked crc. 28 | func (c CRC) Value() uint32 { 29 | return uint32(c>>15|c<<17) + 0xa282ead8 30 | } 31 | -------------------------------------------------------------------------------- /leveldb/util/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "encoding/binary" 11 | ) 12 | 13 | // Hash return hash of the given data. 14 | func Hash(data []byte, seed uint32) uint32 { 15 | // Similar to murmur hash 16 | const ( 17 | m = uint32(0xc6a4a793) 18 | r = uint32(24) 19 | ) 20 | var ( 21 | h = seed ^ (uint32(len(data)) * m) 22 | i int 23 | ) 24 | 25 | for n := len(data) - len(data)%4; i < n; i += 4 { 26 | h += binary.LittleEndian.Uint32(data[i:]) 27 | h *= m 28 | h ^= (h >> 16) 29 | } 30 | 31 | switch len(data) - i { 32 | default: 33 | panic("not reached") 34 | case 3: 35 | h += uint32(data[i+2]) << 16 36 | fallthrough 37 | case 2: 38 | h += uint32(data[i+1]) << 8 39 | fallthrough 40 | case 1: 41 | h += uint32(data[i]) 42 | h *= m 43 | h ^= (h >> r) 44 | case 0: 45 | } 46 | 47 | return h 48 | } 49 | -------------------------------------------------------------------------------- /leveldb/util/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "testing" 11 | ) 12 | 13 | var hashTests = []struct { 14 | data []byte 15 | seed uint32 16 | hash uint32 17 | }{ 18 | {nil, 0xbc9f1d34, 0xbc9f1d34}, 19 | {[]byte{0x62}, 0xbc9f1d34, 0xef1345c4}, 20 | {[]byte{0xc3, 0x97}, 0xbc9f1d34, 0x5b663814}, 21 | {[]byte{0xe2, 0x99, 0xa5}, 0xbc9f1d34, 0x323c078f}, 22 | {[]byte{0xe1, 0x80, 0xb9, 0x32}, 0xbc9f1d34, 0xed21633a}, 23 | {[]byte{ 24 | 0x01, 0xc0, 0x00, 0x00, 25 | 0x00, 0x00, 0x00, 0x00, 26 | 0x00, 0x00, 0x00, 0x00, 27 | 0x00, 0x00, 0x00, 0x00, 28 | 0x14, 0x00, 0x00, 0x00, 29 | 0x00, 0x00, 0x04, 0x00, 30 | 0x00, 0x00, 0x00, 0x14, 31 | 0x00, 0x00, 0x00, 0x18, 32 | 0x28, 0x00, 0x00, 0x00, 33 | 0x00, 0x00, 0x00, 0x00, 34 | 0x02, 0x00, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | }, 0x12345678, 0xf333dabb}, 37 | } 38 | 39 | func TestHash(t *testing.T) { 40 | for i, x := range hashTests { 41 | h := Hash(x.data, x.seed) 42 | if h != x.hash { 43 | t.Fatalf("test-%d: invalid hash, %#x vs %#x", i, h, x.hash) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /leveldb/util/range.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | // Range is a key range. 10 | type Range struct { 11 | // Start of the key range, include in the range. 12 | Start []byte 13 | 14 | // Limit of the key range, not include in the range. 15 | Limit []byte 16 | } 17 | 18 | // BytesPrefix returns key range that satisfy the given prefix. 19 | // This only applicable for the standard 'bytes comparer'. 20 | func BytesPrefix(prefix []byte) *Range { 21 | var limit []byte 22 | for i := len(prefix) - 1; i >= 0; i-- { 23 | c := prefix[i] 24 | if c < 0xff { 25 | limit = make([]byte, i+1) 26 | copy(limit, prefix) 27 | limit[i] = c + 1 28 | break 29 | } 30 | } 31 | return &Range{prefix, limit} 32 | } 33 | -------------------------------------------------------------------------------- /leveldb/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package util provides utilities used throughout leveldb. 8 | package util 9 | 10 | import ( 11 | "errors" 12 | ) 13 | 14 | var ( 15 | ErrReleased = errors.New("leveldb: resource already relesed") 16 | ErrHasReleaser = errors.New("leveldb: releaser already defined") 17 | ) 18 | 19 | // Releaser is the interface that wraps the basic Release method. 20 | type Releaser interface { 21 | // Release releases associated resources. Release should always success 22 | // and can be called multiple times without causing error. 23 | Release() 24 | } 25 | 26 | // ReleaseSetter is the interface that wraps the basic SetReleaser method. 27 | type ReleaseSetter interface { 28 | // SetReleaser associates the given releaser to the resources. The 29 | // releaser will be called once coresponding resources released. 30 | // Calling SetReleaser with nil will clear the releaser. 31 | // 32 | // This will panic if a releaser already present or coresponding 33 | // resource is already released. Releaser should be cleared first 34 | // before assigned a new one. 35 | SetReleaser(releaser Releaser) 36 | } 37 | 38 | // BasicReleaser provides basic implementation of Releaser and ReleaseSetter. 39 | type BasicReleaser struct { 40 | releaser Releaser 41 | released bool 42 | } 43 | 44 | // Released returns whether Release method already called. 45 | func (r *BasicReleaser) Released() bool { 46 | return r.released 47 | } 48 | 49 | // Release implements Releaser.Release. 50 | func (r *BasicReleaser) Release() { 51 | if !r.released { 52 | if r.releaser != nil { 53 | r.releaser.Release() 54 | r.releaser = nil 55 | } 56 | r.released = true 57 | } 58 | } 59 | 60 | // SetReleaser implements ReleaseSetter.SetReleaser. 61 | func (r *BasicReleaser) SetReleaser(releaser Releaser) { 62 | if r.released { 63 | panic(ErrReleased) 64 | } 65 | if r.releaser != nil && releaser != nil { 66 | panic(ErrHasReleaser) 67 | } 68 | r.releaser = releaser 69 | } 70 | 71 | type NoopReleaser struct{} 72 | 73 | func (NoopReleaser) Release() {} 74 | -------------------------------------------------------------------------------- /manualtest/dbstress/key.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | 7 | "github.com/syndtr/goleveldb/leveldb/errors" 8 | "github.com/syndtr/goleveldb/leveldb/storage" 9 | ) 10 | 11 | type ErrIkeyCorrupted struct { 12 | Ikey []byte 13 | Reason string 14 | } 15 | 16 | func (e *ErrIkeyCorrupted) Error() string { 17 | return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason) 18 | } 19 | 20 | func newErrIkeyCorrupted(ikey []byte, reason string) error { 21 | return errors.NewErrCorrupted(storage.FileDesc{}, &ErrIkeyCorrupted{append([]byte(nil), ikey...), reason}) 22 | } 23 | 24 | type kType int 25 | 26 | func (kt kType) String() string { 27 | switch kt { 28 | case ktDel: 29 | return "d" 30 | case ktVal: 31 | return "v" 32 | } 33 | return "x" 34 | } 35 | 36 | // Value types encoded as the last component of internal keys. 37 | // Don't modify; this value are saved to disk. 38 | const ( 39 | ktDel kType = iota 40 | ktVal 41 | ) 42 | 43 | // ktSeek defines the kType that should be passed when constructing an 44 | // internal key for seeking to a particular sequence number (since we 45 | // sort sequence numbers in decreasing order and the value type is 46 | // embedded as the low 8 bits in the sequence number in internal keys, 47 | // we need to use the highest-numbered ValueType, not the lowest). 48 | const ktSeek = ktVal 49 | 50 | const ( 51 | // Maximum value possible for sequence number; the 8-bits are 52 | // used by value type, so its can packed together in single 53 | // 64-bit integer. 54 | kMaxSeq uint64 = (uint64(1) << 56) - 1 55 | // Maximum value possible for packed sequence number and type. 56 | kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek) 57 | ) 58 | 59 | // Maximum number encoded in bytes. 60 | var kMaxNumBytes = make([]byte, 8) 61 | 62 | func init() { 63 | binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum) 64 | } 65 | 66 | func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) { 67 | if len(ik) < 8 { 68 | return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length") 69 | } 70 | num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 71 | seq, kt = num>>8, kType(num&0xff) 72 | if kt > ktVal { 73 | return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type") 74 | } 75 | ukey = ik[:len(ik)-8] 76 | return 77 | } 78 | -------------------------------------------------------------------------------- /manualtest/filelock/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | 12 | "github.com/syndtr/goleveldb/leveldb/storage" 13 | ) 14 | 15 | var ( 16 | filename string 17 | child bool 18 | ) 19 | 20 | func init() { 21 | flag.StringVar(&filename, "filename", filepath.Join(os.TempDir(), "goleveldb_filelock_test"), "Filename used for testing") 22 | flag.BoolVar(&child, "child", false, "This is the child") 23 | } 24 | 25 | func runChild() error { 26 | var args []string 27 | args = append(args, os.Args[1:]...) 28 | args = append(args, "-child") 29 | cmd := exec.Command(os.Args[0], args...) 30 | var out bytes.Buffer 31 | cmd.Stdout = &out 32 | err := cmd.Run() 33 | r := bufio.NewReader(&out) 34 | for { 35 | line, _, e1 := r.ReadLine() 36 | if e1 != nil { 37 | break 38 | } 39 | fmt.Println("[Child]", string(line)) 40 | } 41 | return err 42 | } 43 | 44 | func main() { 45 | flag.Parse() 46 | 47 | fmt.Printf("Using path: %s\n", filename) 48 | if child { 49 | fmt.Println("Child flag set.") 50 | } 51 | 52 | stor, err := storage.OpenFile(filename, false) 53 | if err != nil { 54 | fmt.Printf("Could not open storage: %s", err) 55 | os.Exit(10) 56 | } 57 | 58 | if !child { 59 | fmt.Println("Executing child -- first test (expecting error)") 60 | err := runChild() 61 | if err == nil { 62 | fmt.Println("Expecting error from child") 63 | } else if err.Error() != "exit status 10" { 64 | fmt.Println("Got unexpected error from child:", err) 65 | } else { 66 | fmt.Printf("Got error from child: %s (expected)\n", err) 67 | } 68 | } 69 | 70 | err = stor.Close() 71 | if err != nil { 72 | fmt.Printf("Error when closing storage: %s", err) 73 | os.Exit(11) 74 | } 75 | 76 | if !child { 77 | fmt.Println("Executing child -- second test") 78 | err := runChild() 79 | if err != nil { 80 | fmt.Println("Got unexpected error from child:", err) 81 | } 82 | } 83 | 84 | os.RemoveAll(filename) 85 | } 86 | --------------------------------------------------------------------------------