├── .travis.yml ├── LICENSE ├── README.md ├── leveldb ├── batch.go ├── batch_test.go ├── bench_test.go ├── cache │ ├── bench_test.go │ ├── cache.go │ ├── cache_test.go │ └── lru.go ├── comparer.go ├── comparer │ ├── bytes_comparer.go │ └── comparer.go ├── corrupt_test.go ├── db.go ├── db_compaction.go ├── db_iter.go ├── db_snapshot.go ├── db_state.go ├── db_test.go ├── db_transaction.go ├── db_util.go ├── db_write.go ├── doc.go ├── errors.go ├── errors │ └── errors.go ├── external_test.go ├── filter.go ├── filter │ ├── bloom.go │ ├── bloom_test.go │ └── filter.go ├── iterator │ ├── array_iter.go │ ├── array_iter_test.go │ ├── indexed_iter.go │ ├── indexed_iter_test.go │ ├── iter.go │ ├── iter_suite_test.go │ ├── merged_iter.go │ └── merged_iter_test.go ├── journal │ ├── journal.go │ └── journal_test.go ├── key.go ├── key_test.go ├── leveldb_suite_test.go ├── memdb │ ├── bench_test.go │ ├── memdb.go │ ├── memdb_suite_test.go │ └── memdb_test.go ├── opt │ └── options.go ├── options.go ├── session.go ├── session_compaction.go ├── session_record.go ├── session_record_test.go ├── session_util.go ├── storage │ ├── file_storage.go │ ├── file_storage_nacl.go │ ├── file_storage_plan9.go │ ├── file_storage_solaris.go │ ├── file_storage_test.go │ ├── file_storage_unix.go │ ├── file_storage_wasm.go │ ├── file_storage_windows.go │ ├── mem_storage.go │ ├── mem_storage_test.go │ └── storage.go ├── table.go ├── table │ ├── block_test.go │ ├── reader.go │ ├── table.go │ ├── table_suite_test.go │ ├── table_test.go │ └── writer.go ├── testutil │ ├── db.go │ ├── ginkgo.go │ ├── iter.go │ ├── kv.go │ ├── kvtest.go │ ├── storage.go │ └── util.go ├── testutil_test.go ├── util.go ├── util │ ├── buffer.go │ ├── buffer_pool.go │ ├── buffer_test.go │ ├── crc32.go │ ├── hash.go │ ├── hash_test.go │ ├── range.go │ └── util.go ├── version.go └── version_test.go └── manualtest ├── dbstress ├── key.go └── main.go └── filelock └── main.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.4 5 | - 1.5 6 | - 1.6 7 | - 1.7 8 | - tip 9 | 10 | script: 11 | - go test -timeout 1h ./... 12 | - go test -timeout 30m -race -run "TestDB_(Concurrent|GoleveldbIssue74)" ./leveldb 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Suryandaru Triandana 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is an implementation of the [LevelDB key/value database](http:code.google.com/p/leveldb) in the [Go programming language](http:golang.org). 2 | 3 | [![Build Status](https://travis-ci.org/syndtr/goleveldb.png?branch=master)](https://travis-ci.org/syndtr/goleveldb) 4 | 5 | Installation 6 | ----------- 7 | 8 | go get github.com/pingcap/goleveldb/leveldb 9 | 10 | Requirements 11 | ----------- 12 | 13 | * Need at least `go1.4` or newer. 14 | 15 | Usage 16 | ----------- 17 | 18 | Create or open a database: 19 | ```go 20 | // The returned DB instance is safe for concurrent use. Which mean that all 21 | // DB's methods may be called concurrently from multiple goroutine. 22 | db, err := leveldb.OpenFile("path/to/db", nil) 23 | ... 24 | defer db.Close() 25 | ... 26 | ``` 27 | Read or modify the database content: 28 | ```go 29 | // Remember that the contents of the returned slice should not be modified. 30 | data, err := db.Get([]byte("key"), nil) 31 | ... 32 | err = db.Put([]byte("key"), []byte("value"), nil) 33 | ... 34 | err = db.Delete([]byte("key"), nil) 35 | ... 36 | ``` 37 | 38 | Iterate over database content: 39 | ```go 40 | iter := db.NewIterator(nil, nil) 41 | for iter.Next() { 42 | // Remember that the contents of the returned slice should not be modified, and 43 | // only valid until the next call to Next. 44 | key := iter.Key() 45 | value := iter.Value() 46 | ... 47 | } 48 | iter.Release() 49 | err = iter.Error() 50 | ... 51 | ``` 52 | Seek-then-Iterate: 53 | ```go 54 | iter := db.NewIterator(nil, nil) 55 | for ok := iter.Seek(key); ok; ok = iter.Next() { 56 | // Use key/value. 57 | ... 58 | } 59 | iter.Release() 60 | err = iter.Error() 61 | ... 62 | ``` 63 | Iterate over subset of database content: 64 | ```go 65 | iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) 66 | for iter.Next() { 67 | // Use key/value. 68 | ... 69 | } 70 | iter.Release() 71 | err = iter.Error() 72 | ... 73 | ``` 74 | Iterate over subset of database content with a particular prefix: 75 | ```go 76 | iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) 77 | for iter.Next() { 78 | // Use key/value. 79 | ... 80 | } 81 | iter.Release() 82 | err = iter.Error() 83 | ... 84 | ``` 85 | Batch writes: 86 | ```go 87 | batch := new(leveldb.Batch) 88 | batch.Put([]byte("foo"), []byte("value")) 89 | batch.Put([]byte("bar"), []byte("another value")) 90 | batch.Delete([]byte("baz")) 91 | err = db.Write(batch, nil) 92 | ... 93 | ``` 94 | Use bloom filter: 95 | ```go 96 | o := &opt.Options{ 97 | Filter: filter.NewBloomFilter(10), 98 | } 99 | db, err := leveldb.OpenFile("path/to/db", o) 100 | ... 101 | defer db.Close() 102 | ... 103 | ``` 104 | Documentation 105 | ----------- 106 | 107 | You can read package documentation [here](http:godoc.org/github.com/pingcap/goleveldb). 108 | -------------------------------------------------------------------------------- /leveldb/batch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "testing" 13 | "testing/quick" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/testutil" 16 | ) 17 | 18 | func TestBatchHeader(t *testing.T) { 19 | f := func(seq uint64, length uint32) bool { 20 | encoded := encodeBatchHeader(nil, seq, int(length)) 21 | decSeq, decLength, err := decodeBatchHeader(encoded) 22 | return err == nil && decSeq == seq && decLength == int(length) 23 | } 24 | config := &quick.Config{ 25 | Rand: testutil.NewRand(), 26 | } 27 | if err := quick.Check(f, config); err != nil { 28 | t.Error(err) 29 | } 30 | } 31 | 32 | type batchKV struct { 33 | kt keyType 34 | k, v []byte 35 | } 36 | 37 | func TestBatch(t *testing.T) { 38 | var ( 39 | kvs []batchKV 40 | internalLen int 41 | ) 42 | batch := new(Batch) 43 | rbatch := new(Batch) 44 | abatch := new(Batch) 45 | testBatch := func(i int, kt keyType, k, v []byte) error { 46 | kv := kvs[i] 47 | if kv.kt != kt { 48 | return fmt.Errorf("invalid key type, index=%d: %d vs %d", i, kv.kt, kt) 49 | } 50 | if !bytes.Equal(kv.k, k) { 51 | return fmt.Errorf("invalid key, index=%d", i) 52 | } 53 | if !bytes.Equal(kv.v, v) { 54 | return fmt.Errorf("invalid value, index=%d", i) 55 | } 56 | return nil 57 | } 58 | f := func(ktr uint8, k, v []byte) bool { 59 | kt := keyType(ktr % 2) 60 | if kt == keyTypeVal { 61 | batch.Put(k, v) 62 | rbatch.Put(k, v) 63 | kvs = append(kvs, batchKV{kt: kt, k: k, v: v}) 64 | internalLen += len(k) + len(v) + 8 65 | } else { 66 | batch.Delete(k) 67 | rbatch.Delete(k) 68 | kvs = append(kvs, batchKV{kt: kt, k: k}) 69 | internalLen += len(k) + 8 70 | } 71 | if batch.Len() != len(kvs) { 72 | t.Logf("batch.Len: %d vs %d", len(kvs), batch.Len()) 73 | return false 74 | } 75 | if batch.internalLen != internalLen { 76 | t.Logf("abatch.internalLen: %d vs %d", internalLen, batch.internalLen) 77 | return false 78 | } 79 | if len(kvs)%1000 == 0 { 80 | if err := batch.replayInternal(testBatch); err != nil { 81 | t.Logf("batch.replayInternal: %v", err) 82 | return false 83 | } 84 | 85 | abatch.append(rbatch) 86 | rbatch.Reset() 87 | if abatch.Len() != len(kvs) { 88 | t.Logf("abatch.Len: %d vs %d", len(kvs), abatch.Len()) 89 | return false 90 | } 91 | if abatch.internalLen != internalLen { 92 | t.Logf("abatch.internalLen: %d vs %d", internalLen, abatch.internalLen) 93 | return false 94 | } 95 | if err := abatch.replayInternal(testBatch); err != nil { 96 | t.Logf("abatch.replayInternal: %v", err) 97 | return false 98 | } 99 | 100 | nbatch := new(Batch) 101 | if err := nbatch.Load(batch.Dump()); err != nil { 102 | t.Logf("nbatch.Load: %v", err) 103 | return false 104 | } 105 | if nbatch.Len() != len(kvs) { 106 | t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) 107 | return false 108 | } 109 | if nbatch.internalLen != internalLen { 110 | t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) 111 | return false 112 | } 113 | if err := nbatch.replayInternal(testBatch); err != nil { 114 | t.Logf("nbatch.replayInternal: %v", err) 115 | return false 116 | } 117 | } 118 | if len(kvs)%10000 == 0 { 119 | nbatch := new(Batch) 120 | if err := batch.Replay(nbatch); err != nil { 121 | t.Logf("batch.Replay: %v", err) 122 | return false 123 | } 124 | if nbatch.Len() != len(kvs) { 125 | t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) 126 | return false 127 | } 128 | if nbatch.internalLen != internalLen { 129 | t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) 130 | return false 131 | } 132 | if err := nbatch.replayInternal(testBatch); err != nil { 133 | t.Logf("nbatch.replayInternal: %v", err) 134 | return false 135 | } 136 | } 137 | return true 138 | } 139 | config := &quick.Config{ 140 | MaxCount: 40000, 141 | Rand: testutil.NewRand(), 142 | } 143 | if err := quick.Check(f, config); err != nil { 144 | t.Error(err) 145 | } 146 | t.Logf("length=%d internalLen=%d", len(kvs), internalLen) 147 | } 148 | -------------------------------------------------------------------------------- /leveldb/cache/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package cache 8 | 9 | import ( 10 | "math/rand" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func BenchmarkLRUCache(b *testing.B) { 16 | c := NewCache(NewLRU(10000)) 17 | 18 | b.SetParallelism(10) 19 | b.RunParallel(func(pb *testing.PB) { 20 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 21 | 22 | for pb.Next() { 23 | key := uint64(r.Intn(1000000)) 24 | c.Get(0, key, func() (int, Value) { 25 | return 1, key 26 | }).Release() 27 | } 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /leveldb/cache/lru.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package cache 8 | 9 | import ( 10 | "sync" 11 | "unsafe" 12 | ) 13 | 14 | type lruNode struct { 15 | n *Node 16 | h *Handle 17 | ban bool 18 | 19 | next, prev *lruNode 20 | } 21 | 22 | func (n *lruNode) insert(at *lruNode) { 23 | x := at.next 24 | at.next = n 25 | n.prev = at 26 | n.next = x 27 | x.prev = n 28 | } 29 | 30 | func (n *lruNode) remove() { 31 | if n.prev != nil { 32 | n.prev.next = n.next 33 | n.next.prev = n.prev 34 | n.prev = nil 35 | n.next = nil 36 | } else { 37 | panic("BUG: removing removed node") 38 | } 39 | } 40 | 41 | type lru struct { 42 | mu sync.Mutex 43 | capacity int 44 | used int 45 | recent lruNode 46 | } 47 | 48 | func (r *lru) reset() { 49 | r.recent.next = &r.recent 50 | r.recent.prev = &r.recent 51 | r.used = 0 52 | } 53 | 54 | func (r *lru) Capacity() int { 55 | r.mu.Lock() 56 | defer r.mu.Unlock() 57 | return r.capacity 58 | } 59 | 60 | func (r *lru) SetCapacity(capacity int) { 61 | var evicted []*lruNode 62 | 63 | r.mu.Lock() 64 | r.capacity = capacity 65 | for r.used > r.capacity { 66 | rn := r.recent.prev 67 | if rn == nil { 68 | panic("BUG: invalid LRU used or capacity counter") 69 | } 70 | rn.remove() 71 | rn.n.CacheData = nil 72 | r.used -= rn.n.Size() 73 | evicted = append(evicted, rn) 74 | } 75 | r.mu.Unlock() 76 | 77 | for _, rn := range evicted { 78 | rn.h.Release() 79 | } 80 | } 81 | 82 | func (r *lru) Promote(n *Node) { 83 | var evicted []*lruNode 84 | 85 | r.mu.Lock() 86 | if n.CacheData == nil { 87 | if n.Size() <= r.capacity { 88 | rn := &lruNode{n: n, h: n.GetHandle()} 89 | rn.insert(&r.recent) 90 | n.CacheData = unsafe.Pointer(rn) 91 | r.used += n.Size() 92 | 93 | for r.used > r.capacity { 94 | rn := r.recent.prev 95 | if rn == nil { 96 | panic("BUG: invalid LRU used or capacity counter") 97 | } 98 | rn.remove() 99 | rn.n.CacheData = nil 100 | r.used -= rn.n.Size() 101 | evicted = append(evicted, rn) 102 | } 103 | } 104 | } else { 105 | rn := (*lruNode)(n.CacheData) 106 | if !rn.ban { 107 | rn.remove() 108 | rn.insert(&r.recent) 109 | } 110 | } 111 | r.mu.Unlock() 112 | 113 | for _, rn := range evicted { 114 | rn.h.Release() 115 | } 116 | } 117 | 118 | func (r *lru) Ban(n *Node) { 119 | r.mu.Lock() 120 | if n.CacheData == nil { 121 | n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) 122 | } else { 123 | rn := (*lruNode)(n.CacheData) 124 | if !rn.ban { 125 | rn.remove() 126 | rn.ban = true 127 | r.used -= rn.n.Size() 128 | r.mu.Unlock() 129 | 130 | rn.h.Release() 131 | rn.h = nil 132 | return 133 | } 134 | } 135 | r.mu.Unlock() 136 | } 137 | 138 | func (r *lru) Evict(n *Node) { 139 | r.mu.Lock() 140 | rn := (*lruNode)(n.CacheData) 141 | if rn == nil || rn.ban { 142 | r.mu.Unlock() 143 | return 144 | } 145 | n.CacheData = nil 146 | r.mu.Unlock() 147 | 148 | rn.h.Release() 149 | } 150 | 151 | func (r *lru) EvictNS(ns uint64) { 152 | var evicted []*lruNode 153 | 154 | r.mu.Lock() 155 | for e := r.recent.prev; e != &r.recent; { 156 | rn := e 157 | e = e.prev 158 | if rn.n.NS() == ns { 159 | rn.remove() 160 | rn.n.CacheData = nil 161 | r.used -= rn.n.Size() 162 | evicted = append(evicted, rn) 163 | } 164 | } 165 | r.mu.Unlock() 166 | 167 | for _, rn := range evicted { 168 | rn.h.Release() 169 | } 170 | } 171 | 172 | func (r *lru) EvictAll() { 173 | r.mu.Lock() 174 | back := r.recent.prev 175 | for rn := back; rn != &r.recent; rn = rn.prev { 176 | rn.n.CacheData = nil 177 | } 178 | r.reset() 179 | r.mu.Unlock() 180 | 181 | for rn := back; rn != &r.recent; rn = rn.prev { 182 | rn.h.Release() 183 | } 184 | } 185 | 186 | func (r *lru) Close() error { 187 | return nil 188 | } 189 | 190 | // NewLRU create a new LRU-cache. 191 | func NewLRU(capacity int) Cacher { 192 | r := &lru{capacity: capacity} 193 | r.reset() 194 | return r 195 | } 196 | -------------------------------------------------------------------------------- /leveldb/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/comparer" 11 | ) 12 | 13 | type iComparer struct { 14 | ucmp comparer.Comparer 15 | } 16 | 17 | func (icmp *iComparer) uName() string { 18 | return icmp.ucmp.Name() 19 | } 20 | 21 | func (icmp *iComparer) uCompare(a, b []byte) int { 22 | return icmp.ucmp.Compare(a, b) 23 | } 24 | 25 | func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte { 26 | return icmp.ucmp.Separator(dst, a, b) 27 | } 28 | 29 | func (icmp *iComparer) uSuccessor(dst, b []byte) []byte { 30 | return icmp.ucmp.Successor(dst, b) 31 | } 32 | 33 | func (icmp *iComparer) Name() string { 34 | return icmp.uName() 35 | } 36 | 37 | func (icmp *iComparer) Compare(a, b []byte) int { 38 | x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) 39 | if x == 0 { 40 | if m, n := internalKey(a).num(), internalKey(b).num(); m > n { 41 | return -1 42 | } else if m < n { 43 | return 1 44 | } 45 | } 46 | return x 47 | } 48 | 49 | func (icmp *iComparer) Separator(dst, a, b []byte) []byte { 50 | ua, ub := internalKey(a).ukey(), internalKey(b).ukey() 51 | dst = icmp.uSeparator(dst, ua, ub) 52 | if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 { 53 | // Append earliest possible number. 54 | return append(dst, keyMaxNumBytes...) 55 | } 56 | return nil 57 | } 58 | 59 | func (icmp *iComparer) Successor(dst, b []byte) []byte { 60 | ub := internalKey(b).ukey() 61 | dst = icmp.uSuccessor(dst, ub) 62 | if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 { 63 | // Append earliest possible number. 64 | return append(dst, keyMaxNumBytes...) 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /leveldb/comparer/bytes_comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package comparer 8 | 9 | import "bytes" 10 | 11 | type bytesComparer struct{} 12 | 13 | func (bytesComparer) Compare(a, b []byte) int { 14 | return bytes.Compare(a, b) 15 | } 16 | 17 | func (bytesComparer) Name() string { 18 | return "leveldb.BytewiseComparator" 19 | } 20 | 21 | func (bytesComparer) Separator(dst, a, b []byte) []byte { 22 | i, n := 0, len(a) 23 | if n > len(b) { 24 | n = len(b) 25 | } 26 | for ; i < n && a[i] == b[i]; i++ { 27 | } 28 | if i >= n { 29 | // Do not shorten if one string is a prefix of the other 30 | } else if c := a[i]; c < 0xff && c+1 < b[i] { 31 | dst = append(dst, a[:i+1]...) 32 | dst[i]++ 33 | return dst 34 | } 35 | return nil 36 | } 37 | 38 | func (bytesComparer) Successor(dst, b []byte) []byte { 39 | for i, c := range b { 40 | if c != 0xff { 41 | dst = append(dst, b[:i+1]...) 42 | dst[i]++ 43 | return dst 44 | } 45 | } 46 | return nil 47 | } 48 | 49 | // DefaultComparer are default implementation of the Comparer interface. 50 | // It uses the natural ordering, consistent with bytes.Compare. 51 | var DefaultComparer = bytesComparer{} 52 | -------------------------------------------------------------------------------- /leveldb/comparer/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package comparer provides interface and implementation for ordering 8 | // sets of data. 9 | package comparer 10 | 11 | // BasicComparer is the interface that wraps the basic Compare method. 12 | type BasicComparer interface { 13 | // Compare returns -1, 0, or +1 depending on whether a is 'less than', 14 | // 'equal to' or 'greater than' b. The two arguments can only be 'equal' 15 | // if their contents are exactly equal. Furthermore, the empty slice 16 | // must be 'less than' any non-empty slice. 17 | Compare(a, b []byte) int 18 | } 19 | 20 | // Comparer defines a total ordering over the space of []byte keys: a 'less 21 | // than' relationship. 22 | type Comparer interface { 23 | BasicComparer 24 | 25 | // Name returns name of the comparer. 26 | // 27 | // The Level-DB on-disk format stores the comparer name, and opening a 28 | // database with a different comparer from the one it was created with 29 | // will result in an error. 30 | // 31 | // An implementation to a new name whenever the comparer implementation 32 | // changes in a way that will cause the relative ordering of any two keys 33 | // to change. 34 | // 35 | // Names starting with "leveldb." are reserved and should not be used 36 | // by any users of this package. 37 | Name() string 38 | 39 | // Bellow are advanced functions used used to reduce the space requirements 40 | // for internal data structures such as index blocks. 41 | 42 | // Separator appends a sequence of bytes x to dst such that a <= x && x < b, 43 | // where 'less than' is consistent with Compare. An implementation should 44 | // return nil if x equal to a. 45 | // 46 | // Either contents of a or b should not by any means modified. Doing so 47 | // may cause corruption on the internal state. 48 | Separator(dst, a, b []byte) []byte 49 | 50 | // Successor appends a sequence of bytes x to dst such that x >= b, where 51 | // 'less than' is consistent with Compare. An implementation should return 52 | // nil if x equal to b. 53 | // 54 | // Contents of b should not by any means modified. Doing so may cause 55 | // corruption on the internal state. 56 | Successor(dst, b []byte) []byte 57 | } 58 | -------------------------------------------------------------------------------- /leveldb/db_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "errors" 11 | "math/rand" 12 | "runtime" 13 | "sync" 14 | "sync/atomic" 15 | 16 | "github.com/pingcap/goleveldb/leveldb/iterator" 17 | "github.com/pingcap/goleveldb/leveldb/opt" 18 | "github.com/pingcap/goleveldb/leveldb/util" 19 | ) 20 | 21 | var ( 22 | errInvalidInternalKey = errors.New("leveldb: Iterator: invalid internal key") 23 | ) 24 | 25 | type memdbReleaser struct { 26 | once sync.Once 27 | m *memDB 28 | } 29 | 30 | func (mr *memdbReleaser) Release() { 31 | mr.once.Do(func() { 32 | mr.m.decref() 33 | }) 34 | } 35 | 36 | func (db *DB) newRawIterator(auxm *memDB, auxt tFiles, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 37 | strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader) 38 | em, fm := db.getMems() 39 | v := db.s.version() 40 | 41 | tableIts := v.getIterators(slice, ro) 42 | n := len(tableIts) + len(auxt) + 3 43 | its := make([]iterator.Iterator, 0, n) 44 | 45 | if auxm != nil { 46 | ami := auxm.NewIterator(slice) 47 | ami.SetReleaser(&memdbReleaser{m: auxm}) 48 | its = append(its, ami) 49 | } 50 | for _, t := range auxt { 51 | its = append(its, v.s.tops.newIterator(t, slice, ro)) 52 | } 53 | 54 | emi := em.NewIterator(slice) 55 | emi.SetReleaser(&memdbReleaser{m: em}) 56 | its = append(its, emi) 57 | if fm != nil { 58 | fmi := fm.NewIterator(slice) 59 | fmi.SetReleaser(&memdbReleaser{m: fm}) 60 | its = append(its, fmi) 61 | } 62 | its = append(its, tableIts...) 63 | mi := iterator.NewMergedIterator(its, db.s.icmp, strict) 64 | mi.SetReleaser(&versionReleaser{v: v}) 65 | return mi 66 | } 67 | 68 | func (db *DB) newIterator(auxm *memDB, auxt tFiles, seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter { 69 | var islice *util.Range 70 | if slice != nil { 71 | islice = &util.Range{} 72 | if slice.Start != nil { 73 | islice.Start = makeInternalKey(nil, slice.Start, keyMaxSeq, keyTypeSeek) 74 | } 75 | if slice.Limit != nil { 76 | islice.Limit = makeInternalKey(nil, slice.Limit, keyMaxSeq, keyTypeSeek) 77 | } 78 | } 79 | rawIter := db.newRawIterator(auxm, auxt, islice, ro) 80 | iter := &dbIter{ 81 | db: db, 82 | icmp: db.s.icmp, 83 | iter: rawIter, 84 | seq: seq, 85 | strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader), 86 | key: make([]byte, 0), 87 | value: make([]byte, 0), 88 | } 89 | atomic.AddInt32(&db.aliveIters, 1) 90 | runtime.SetFinalizer(iter, (*dbIter).Release) 91 | return iter 92 | } 93 | 94 | func (db *DB) iterSamplingRate() int { 95 | return rand.Intn(2 * db.s.o.GetIteratorSamplingRate()) 96 | } 97 | 98 | type dir int 99 | 100 | const ( 101 | dirReleased dir = iota - 1 102 | dirSOI 103 | dirEOI 104 | dirBackward 105 | dirForward 106 | ) 107 | 108 | // dbIter represent an interator states over a database session. 109 | type dbIter struct { 110 | db *DB 111 | icmp *iComparer 112 | iter iterator.Iterator 113 | seq uint64 114 | strict bool 115 | 116 | smaplingGap int 117 | dir dir 118 | key []byte 119 | value []byte 120 | err error 121 | releaser util.Releaser 122 | } 123 | 124 | func (i *dbIter) sampleSeek() { 125 | ikey := i.iter.Key() 126 | i.smaplingGap -= len(ikey) + len(i.iter.Value()) 127 | for i.smaplingGap < 0 { 128 | i.smaplingGap += i.db.iterSamplingRate() 129 | i.db.sampleSeek(ikey) 130 | } 131 | } 132 | 133 | func (i *dbIter) setErr(err error) { 134 | i.err = err 135 | i.key = nil 136 | i.value = nil 137 | } 138 | 139 | func (i *dbIter) iterErr() { 140 | if err := i.iter.Error(); err != nil { 141 | i.setErr(err) 142 | } 143 | } 144 | 145 | func (i *dbIter) Valid() bool { 146 | return i.err == nil && i.dir > dirEOI 147 | } 148 | 149 | func (i *dbIter) First() bool { 150 | if i.err != nil { 151 | return false 152 | } else if i.dir == dirReleased { 153 | i.err = ErrIterReleased 154 | return false 155 | } 156 | 157 | if i.iter.First() { 158 | i.dir = dirSOI 159 | return i.next() 160 | } 161 | i.dir = dirEOI 162 | i.iterErr() 163 | return false 164 | } 165 | 166 | func (i *dbIter) Last() bool { 167 | if i.err != nil { 168 | return false 169 | } else if i.dir == dirReleased { 170 | i.err = ErrIterReleased 171 | return false 172 | } 173 | 174 | if i.iter.Last() { 175 | return i.prev() 176 | } 177 | i.dir = dirSOI 178 | i.iterErr() 179 | return false 180 | } 181 | 182 | func (i *dbIter) Seek(key []byte) bool { 183 | if i.err != nil { 184 | return false 185 | } else if i.dir == dirReleased { 186 | i.err = ErrIterReleased 187 | return false 188 | } 189 | 190 | ikey := makeInternalKey(nil, key, i.seq, keyTypeSeek) 191 | if i.iter.Seek(ikey) { 192 | i.dir = dirSOI 193 | return i.next() 194 | } 195 | i.dir = dirEOI 196 | i.iterErr() 197 | return false 198 | } 199 | 200 | func (i *dbIter) next() bool { 201 | for { 202 | if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 203 | i.sampleSeek() 204 | if seq <= i.seq { 205 | switch kt { 206 | case keyTypeDel: 207 | // Skip deleted key. 208 | i.key = append(i.key[:0], ukey...) 209 | i.dir = dirForward 210 | case keyTypeVal: 211 | if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 { 212 | i.key = append(i.key[:0], ukey...) 213 | i.value = append(i.value[:0], i.iter.Value()...) 214 | i.dir = dirForward 215 | return true 216 | } 217 | } 218 | } 219 | } else if i.strict { 220 | i.setErr(kerr) 221 | break 222 | } 223 | if !i.iter.Next() { 224 | i.dir = dirEOI 225 | i.iterErr() 226 | break 227 | } 228 | } 229 | return false 230 | } 231 | 232 | func (i *dbIter) Next() bool { 233 | if i.dir == dirEOI || i.err != nil { 234 | return false 235 | } else if i.dir == dirReleased { 236 | i.err = ErrIterReleased 237 | return false 238 | } 239 | 240 | if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) { 241 | i.dir = dirEOI 242 | i.iterErr() 243 | return false 244 | } 245 | return i.next() 246 | } 247 | 248 | func (i *dbIter) prev() bool { 249 | i.dir = dirBackward 250 | del := true 251 | if i.iter.Valid() { 252 | for { 253 | if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 254 | i.sampleSeek() 255 | if seq <= i.seq { 256 | if !del && i.icmp.uCompare(ukey, i.key) < 0 { 257 | return true 258 | } 259 | del = (kt == keyTypeDel) 260 | if !del { 261 | i.key = append(i.key[:0], ukey...) 262 | i.value = append(i.value[:0], i.iter.Value()...) 263 | } 264 | } 265 | } else if i.strict { 266 | i.setErr(kerr) 267 | return false 268 | } 269 | if !i.iter.Prev() { 270 | break 271 | } 272 | } 273 | } 274 | if del { 275 | i.dir = dirSOI 276 | i.iterErr() 277 | return false 278 | } 279 | return true 280 | } 281 | 282 | func (i *dbIter) Prev() bool { 283 | if i.dir == dirSOI || i.err != nil { 284 | return false 285 | } else if i.dir == dirReleased { 286 | i.err = ErrIterReleased 287 | return false 288 | } 289 | 290 | switch i.dir { 291 | case dirEOI: 292 | return i.Last() 293 | case dirForward: 294 | for i.iter.Prev() { 295 | if ukey, _, _, kerr := parseInternalKey(i.iter.Key()); kerr == nil { 296 | i.sampleSeek() 297 | if i.icmp.uCompare(ukey, i.key) < 0 { 298 | goto cont 299 | } 300 | } else if i.strict { 301 | i.setErr(kerr) 302 | return false 303 | } 304 | } 305 | i.dir = dirSOI 306 | i.iterErr() 307 | return false 308 | } 309 | 310 | cont: 311 | return i.prev() 312 | } 313 | 314 | func (i *dbIter) Key() []byte { 315 | if i.err != nil || i.dir <= dirEOI { 316 | return nil 317 | } 318 | return i.key 319 | } 320 | 321 | func (i *dbIter) Value() []byte { 322 | if i.err != nil || i.dir <= dirEOI { 323 | return nil 324 | } 325 | return i.value 326 | } 327 | 328 | func (i *dbIter) Release() { 329 | if i.dir != dirReleased { 330 | // Clear the finalizer. 331 | runtime.SetFinalizer(i, nil) 332 | 333 | if i.releaser != nil { 334 | i.releaser.Release() 335 | i.releaser = nil 336 | } 337 | 338 | i.dir = dirReleased 339 | i.key = nil 340 | i.value = nil 341 | i.iter.Release() 342 | i.iter = nil 343 | atomic.AddInt32(&i.db.aliveIters, -1) 344 | i.db = nil 345 | } 346 | } 347 | 348 | func (i *dbIter) SetReleaser(releaser util.Releaser) { 349 | if i.dir == dirReleased { 350 | panic(util.ErrReleased) 351 | } 352 | if i.releaser != nil && releaser != nil { 353 | panic(util.ErrHasReleaser) 354 | } 355 | i.releaser = releaser 356 | } 357 | 358 | func (i *dbIter) Error() error { 359 | return i.err 360 | } 361 | -------------------------------------------------------------------------------- /leveldb/db_snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "container/list" 11 | "fmt" 12 | "runtime" 13 | "sync" 14 | "sync/atomic" 15 | 16 | "github.com/pingcap/goleveldb/leveldb/iterator" 17 | "github.com/pingcap/goleveldb/leveldb/opt" 18 | "github.com/pingcap/goleveldb/leveldb/util" 19 | ) 20 | 21 | type snapshotElement struct { 22 | seq uint64 23 | ref int 24 | e *list.Element 25 | } 26 | 27 | // Acquires a snapshot, based on latest sequence. 28 | func (db *DB) acquireSnapshot() *snapshotElement { 29 | db.snapsMu.Lock() 30 | defer db.snapsMu.Unlock() 31 | 32 | seq := db.getSeq() 33 | 34 | if e := db.snapsList.Back(); e != nil { 35 | se := e.Value.(*snapshotElement) 36 | if se.seq == seq { 37 | se.ref++ 38 | return se 39 | } else if seq < se.seq { 40 | panic("leveldb: sequence number is not increasing") 41 | } 42 | } 43 | se := &snapshotElement{seq: seq, ref: 1} 44 | se.e = db.snapsList.PushBack(se) 45 | return se 46 | } 47 | 48 | // Releases given snapshot element. 49 | func (db *DB) releaseSnapshot(se *snapshotElement) { 50 | db.snapsMu.Lock() 51 | defer db.snapsMu.Unlock() 52 | 53 | se.ref-- 54 | if se.ref == 0 { 55 | db.snapsList.Remove(se.e) 56 | se.e = nil 57 | } else if se.ref < 0 { 58 | panic("leveldb: Snapshot: negative element reference") 59 | } 60 | } 61 | 62 | // Gets minimum sequence that not being snapshotted. 63 | func (db *DB) minSeq() uint64 { 64 | db.snapsMu.Lock() 65 | defer db.snapsMu.Unlock() 66 | 67 | if e := db.snapsList.Front(); e != nil { 68 | return e.Value.(*snapshotElement).seq 69 | } 70 | 71 | return db.getSeq() 72 | } 73 | 74 | // Snapshot is a DB snapshot. 75 | type Snapshot struct { 76 | db *DB 77 | elem *snapshotElement 78 | mu sync.RWMutex 79 | released bool 80 | } 81 | 82 | // Creates new snapshot object. 83 | func (db *DB) newSnapshot() *Snapshot { 84 | snap := &Snapshot{ 85 | db: db, 86 | elem: db.acquireSnapshot(), 87 | } 88 | atomic.AddInt32(&db.aliveSnaps, 1) 89 | runtime.SetFinalizer(snap, (*Snapshot).Release) 90 | return snap 91 | } 92 | 93 | func (snap *Snapshot) String() string { 94 | return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq) 95 | } 96 | 97 | // Get gets the value for the given key. It returns ErrNotFound if 98 | // the DB does not contains the key. 99 | // 100 | // The caller should not modify the contents of the returned slice, but 101 | // it is safe to modify the contents of the argument after Get returns. 102 | func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { 103 | err = snap.db.ok() 104 | if err != nil { 105 | return 106 | } 107 | snap.mu.RLock() 108 | defer snap.mu.RUnlock() 109 | if snap.released { 110 | err = ErrSnapshotReleased 111 | return 112 | } 113 | return snap.db.get(nil, nil, key, snap.elem.seq, ro) 114 | } 115 | 116 | // Has returns true if the DB does contains the given key. 117 | // 118 | // It is safe to modify the contents of the argument after Get returns. 119 | func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { 120 | err = snap.db.ok() 121 | if err != nil { 122 | return 123 | } 124 | snap.mu.RLock() 125 | defer snap.mu.RUnlock() 126 | if snap.released { 127 | err = ErrSnapshotReleased 128 | return 129 | } 130 | return snap.db.has(nil, nil, key, snap.elem.seq, ro) 131 | } 132 | 133 | // NewIterator returns an iterator for the snapshot of the underlying DB. 134 | // The returned iterator is not safe for concurrent use, but it is safe to use 135 | // multiple iterators concurrently, with each in a dedicated goroutine. 136 | // It is also safe to use an iterator concurrently with modifying its 137 | // underlying DB. The resultant key/value pairs are guaranteed to be 138 | // consistent. 139 | // 140 | // Slice allows slicing the iterator to only contains keys in the given 141 | // range. A nil Range.Start is treated as a key before all keys in the 142 | // DB. And a nil Range.Limit is treated as a key after all keys in 143 | // the DB. 144 | // 145 | // The iterator must be released after use, by calling Release method. 146 | // Releasing the snapshot doesn't mean releasing the iterator too, the 147 | // iterator would be still valid until released. 148 | // 149 | // Also read Iterator documentation of the leveldb/iterator package. 150 | func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 151 | if err := snap.db.ok(); err != nil { 152 | return iterator.NewEmptyIterator(err) 153 | } 154 | snap.mu.Lock() 155 | defer snap.mu.Unlock() 156 | if snap.released { 157 | return iterator.NewEmptyIterator(ErrSnapshotReleased) 158 | } 159 | // Since iterator already hold version ref, it doesn't need to 160 | // hold snapshot ref. 161 | return snap.db.newIterator(nil, nil, snap.elem.seq, slice, ro) 162 | } 163 | 164 | // Release releases the snapshot. This will not release any returned 165 | // iterators, the iterators would still be valid until released or the 166 | // underlying DB is closed. 167 | // 168 | // Other methods should not be called after the snapshot has been released. 169 | func (snap *Snapshot) Release() { 170 | snap.mu.Lock() 171 | defer snap.mu.Unlock() 172 | 173 | if !snap.released { 174 | // Clear the finalizer. 175 | runtime.SetFinalizer(snap, nil) 176 | 177 | snap.released = true 178 | snap.db.releaseSnapshot(snap.elem) 179 | atomic.AddInt32(&snap.db.aliveSnaps, -1) 180 | snap.db = nil 181 | snap.elem = nil 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /leveldb/db_state.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "errors" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/pingcap/goleveldb/leveldb/journal" 15 | "github.com/pingcap/goleveldb/leveldb/memdb" 16 | "github.com/pingcap/goleveldb/leveldb/storage" 17 | ) 18 | 19 | var ( 20 | errHasFrozenMem = errors.New("has frozen mem") 21 | ) 22 | 23 | type memDB struct { 24 | db *DB 25 | *memdb.DB 26 | ref int32 27 | } 28 | 29 | func (m *memDB) getref() int32 { 30 | return atomic.LoadInt32(&m.ref) 31 | } 32 | 33 | func (m *memDB) incref() { 34 | atomic.AddInt32(&m.ref, 1) 35 | } 36 | 37 | func (m *memDB) decref() { 38 | if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { 39 | // Only put back memdb with std capacity. 40 | if m.Capacity() == m.db.s.o.GetWriteBuffer() { 41 | m.Reset() 42 | m.db.mpoolPut(m.DB) 43 | } 44 | m.db = nil 45 | m.DB = nil 46 | } else if ref < 0 { 47 | panic("negative memdb ref") 48 | } 49 | } 50 | 51 | // Get latest sequence number. 52 | func (db *DB) getSeq() uint64 { 53 | return atomic.LoadUint64(&db.seq) 54 | } 55 | 56 | // Atomically adds delta to seq. 57 | func (db *DB) addSeq(delta uint64) { 58 | atomic.AddUint64(&db.seq, delta) 59 | } 60 | 61 | func (db *DB) setSeq(seq uint64) { 62 | atomic.StoreUint64(&db.seq, seq) 63 | } 64 | 65 | func (db *DB) sampleSeek(ikey internalKey) { 66 | v := db.s.version() 67 | if v.sampleSeek(ikey) { 68 | // Trigger table compaction. 69 | db.compTrigger(db.tcompCmdC) 70 | } 71 | v.release() 72 | } 73 | 74 | func (db *DB) mpoolPut(mem *memdb.DB) { 75 | if !db.isClosed() { 76 | select { 77 | case db.memPool <- mem: 78 | default: 79 | } 80 | } 81 | } 82 | 83 | func (db *DB) mpoolGet(n int) *memDB { 84 | var mdb *memdb.DB 85 | select { 86 | case mdb = <-db.memPool: 87 | default: 88 | } 89 | if mdb == nil || mdb.Capacity() < n { 90 | mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)) 91 | } 92 | return &memDB{ 93 | db: db, 94 | DB: mdb, 95 | } 96 | } 97 | 98 | func (db *DB) mpoolDrain() { 99 | ticker := time.NewTicker(30 * time.Second) 100 | for { 101 | select { 102 | case <-ticker.C: 103 | select { 104 | case <-db.memPool: 105 | default: 106 | } 107 | case <-db.closeC: 108 | ticker.Stop() 109 | // Make sure the pool is drained. 110 | select { 111 | case <-db.memPool: 112 | case <-time.After(time.Second): 113 | } 114 | close(db.memPool) 115 | return 116 | } 117 | } 118 | } 119 | 120 | // Create new memdb and froze the old one; need external synchronization. 121 | // newMem only called synchronously by the writer. 122 | func (db *DB) newMem(n int) (mem *memDB, err error) { 123 | fd := storage.FileDesc{Type: storage.TypeJournal, Num: db.s.allocFileNum()} 124 | w, err := db.s.stor.Create(fd) 125 | if err != nil { 126 | db.s.reuseFileNum(fd.Num) 127 | return 128 | } 129 | 130 | db.memMu.Lock() 131 | defer db.memMu.Unlock() 132 | 133 | if db.frozenMem != nil { 134 | return nil, errHasFrozenMem 135 | } 136 | 137 | if db.journal == nil { 138 | db.journal = journal.NewWriter(w) 139 | } else { 140 | db.journal.Reset(w) 141 | db.journalWriter.Close() 142 | db.frozenJournalFd = db.journalFd 143 | } 144 | db.journalWriter = w 145 | db.journalFd = fd 146 | db.frozenMem = db.mem 147 | mem = db.mpoolGet(n) 148 | mem.incref() // for self 149 | mem.incref() // for caller 150 | db.mem = mem 151 | // The seq only incremented by the writer. And whoever called newMem 152 | // should hold write lock, so no need additional synchronization here. 153 | db.frozenSeq = db.seq 154 | return 155 | } 156 | 157 | // Get all memdbs. 158 | func (db *DB) getMems() (e, f *memDB) { 159 | db.memMu.RLock() 160 | defer db.memMu.RUnlock() 161 | if db.mem != nil { 162 | db.mem.incref() 163 | } else if !db.isClosed() { 164 | panic("nil effective mem") 165 | } 166 | if db.frozenMem != nil { 167 | db.frozenMem.incref() 168 | } 169 | return db.mem, db.frozenMem 170 | } 171 | 172 | // Get effective memdb. 173 | func (db *DB) getEffectiveMem() *memDB { 174 | db.memMu.RLock() 175 | defer db.memMu.RUnlock() 176 | if db.mem != nil { 177 | db.mem.incref() 178 | } else if !db.isClosed() { 179 | panic("nil effective mem") 180 | } 181 | return db.mem 182 | } 183 | 184 | // Check whether we has frozen memdb. 185 | func (db *DB) hasFrozenMem() bool { 186 | db.memMu.RLock() 187 | defer db.memMu.RUnlock() 188 | return db.frozenMem != nil 189 | } 190 | 191 | // Get frozen memdb. 192 | func (db *DB) getFrozenMem() *memDB { 193 | db.memMu.RLock() 194 | defer db.memMu.RUnlock() 195 | if db.frozenMem != nil { 196 | db.frozenMem.incref() 197 | } 198 | return db.frozenMem 199 | } 200 | 201 | // Drop frozen memdb; assume that frozen memdb isn't nil. 202 | func (db *DB) dropFrozenMem() { 203 | db.memMu.Lock() 204 | if err := db.s.stor.Remove(db.frozenJournalFd); err != nil { 205 | db.logf("journal@remove removing @%d %q", db.frozenJournalFd.Num, err) 206 | } else { 207 | db.logf("journal@remove removed @%d", db.frozenJournalFd.Num) 208 | } 209 | db.frozenJournalFd = storage.FileDesc{} 210 | db.frozenMem.decref() 211 | db.frozenMem = nil 212 | db.memMu.Unlock() 213 | } 214 | 215 | // Clear mems ptr; used by DB.Close(). 216 | func (db *DB) clearMems() { 217 | db.memMu.Lock() 218 | db.mem = nil 219 | db.frozenMem = nil 220 | db.memMu.Unlock() 221 | } 222 | 223 | // Set closed flag; return true if not already closed. 224 | func (db *DB) setClosed() bool { 225 | return atomic.CompareAndSwapUint32(&db.closed, 0, 1) 226 | } 227 | 228 | // Check whether DB was closed. 229 | func (db *DB) isClosed() bool { 230 | return atomic.LoadUint32(&db.closed) != 0 231 | } 232 | 233 | // Check read ok status. 234 | func (db *DB) ok() error { 235 | if db.isClosed() { 236 | return ErrClosed 237 | } 238 | return nil 239 | } 240 | -------------------------------------------------------------------------------- /leveldb/db_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/errors" 11 | "github.com/pingcap/goleveldb/leveldb/iterator" 12 | "github.com/pingcap/goleveldb/leveldb/opt" 13 | "github.com/pingcap/goleveldb/leveldb/storage" 14 | "github.com/pingcap/goleveldb/leveldb/util" 15 | ) 16 | 17 | // Reader is the interface that wraps basic Get and NewIterator methods. 18 | // This interface implemented by both DB and Snapshot. 19 | type Reader interface { 20 | Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) 21 | NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator 22 | } 23 | 24 | // Sizes is list of size. 25 | type Sizes []int64 26 | 27 | // Sum returns sum of the sizes. 28 | func (sizes Sizes) Sum() int64 { 29 | var sum int64 30 | for _, size := range sizes { 31 | sum += size 32 | } 33 | return sum 34 | } 35 | 36 | // Logging. 37 | func (db *DB) log(v ...interface{}) { db.s.log(v...) } 38 | func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) } 39 | 40 | // Check and clean files. 41 | func (db *DB) checkAndCleanFiles() error { 42 | v := db.s.version() 43 | defer v.release() 44 | 45 | tmap := make(map[int64]bool) 46 | for _, tables := range v.levels { 47 | for _, t := range tables { 48 | tmap[t.fd.Num] = false 49 | } 50 | } 51 | 52 | fds, err := db.s.stor.List(storage.TypeAll) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | var nt int 58 | var rem []storage.FileDesc 59 | for _, fd := range fds { 60 | keep := true 61 | switch fd.Type { 62 | case storage.TypeManifest: 63 | keep = fd.Num >= db.s.manifestFd.Num 64 | case storage.TypeJournal: 65 | if !db.frozenJournalFd.Zero() { 66 | keep = fd.Num >= db.frozenJournalFd.Num 67 | } else { 68 | keep = fd.Num >= db.journalFd.Num 69 | } 70 | case storage.TypeTable: 71 | _, keep = tmap[fd.Num] 72 | if keep { 73 | tmap[fd.Num] = true 74 | nt++ 75 | } 76 | } 77 | 78 | if !keep { 79 | rem = append(rem, fd) 80 | } 81 | } 82 | 83 | if nt != len(tmap) { 84 | var mfds []storage.FileDesc 85 | for num, present := range tmap { 86 | if !present { 87 | mfds = append(mfds, storage.FileDesc{storage.TypeTable, num}) 88 | db.logf("db@janitor table missing @%d", num) 89 | } 90 | } 91 | return errors.NewErrCorrupted(storage.FileDesc{}, &errors.ErrMissingFiles{Fds: mfds}) 92 | } 93 | 94 | db.logf("db@janitor F·%d G·%d", len(fds), len(rem)) 95 | for _, fd := range rem { 96 | db.logf("db@janitor removing %s-%d", fd.Type, fd.Num) 97 | if err := db.s.stor.Remove(fd); err != nil { 98 | return err 99 | } 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /leveldb/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package leveldb provides implementation of LevelDB key/value database. 8 | // 9 | // Create or open a database: 10 | // 11 | // // The returned DB instance is safe for concurrent use. Which mean that all 12 | // // DB's methods may be called concurrently from multiple goroutine. 13 | // db, err := leveldb.OpenFile("path/to/db", nil) 14 | // ... 15 | // defer db.Close() 16 | // ... 17 | // 18 | // Read or modify the database content: 19 | // 20 | // // Remember that the contents of the returned slice should not be modified. 21 | // data, err := db.Get([]byte("key"), nil) 22 | // ... 23 | // err = db.Put([]byte("key"), []byte("value"), nil) 24 | // ... 25 | // err = db.Delete([]byte("key"), nil) 26 | // ... 27 | // 28 | // Iterate over database content: 29 | // 30 | // iter := db.NewIterator(nil, nil) 31 | // for iter.Next() { 32 | // // Remember that the contents of the returned slice should not be modified, and 33 | // // only valid until the next call to Next. 34 | // key := iter.Key() 35 | // value := iter.Value() 36 | // ... 37 | // } 38 | // iter.Release() 39 | // err = iter.Error() 40 | // ... 41 | // 42 | // Iterate over subset of database content with a particular prefix: 43 | // iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) 44 | // for iter.Next() { 45 | // // Use key/value. 46 | // ... 47 | // } 48 | // iter.Release() 49 | // err = iter.Error() 50 | // ... 51 | // 52 | // Seek-then-Iterate: 53 | // 54 | // iter := db.NewIterator(nil, nil) 55 | // for ok := iter.Seek(key); ok; ok = iter.Next() { 56 | // // Use key/value. 57 | // ... 58 | // } 59 | // iter.Release() 60 | // err = iter.Error() 61 | // ... 62 | // 63 | // Iterate over subset of database content: 64 | // 65 | // iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) 66 | // for iter.Next() { 67 | // // Use key/value. 68 | // ... 69 | // } 70 | // iter.Release() 71 | // err = iter.Error() 72 | // ... 73 | // 74 | // Batch writes: 75 | // 76 | // batch := new(leveldb.Batch) 77 | // batch.Put([]byte("foo"), []byte("value")) 78 | // batch.Put([]byte("bar"), []byte("another value")) 79 | // batch.Delete([]byte("baz")) 80 | // err = db.Write(batch, nil) 81 | // ... 82 | // 83 | // Use bloom filter: 84 | // 85 | // o := &opt.Options{ 86 | // Filter: filter.NewBloomFilter(10), 87 | // } 88 | // db, err := leveldb.OpenFile("path/to/db", o) 89 | // ... 90 | // defer db.Close() 91 | // ... 92 | package leveldb 93 | -------------------------------------------------------------------------------- /leveldb/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/errors" 11 | ) 12 | 13 | // Common errors. 14 | var ( 15 | ErrNotFound = errors.ErrNotFound 16 | ErrReadOnly = errors.New("leveldb: read-only mode") 17 | ErrSnapshotReleased = errors.New("leveldb: snapshot released") 18 | ErrIterReleased = errors.New("leveldb: iterator released") 19 | ErrClosed = errors.New("leveldb: closed") 20 | ) 21 | -------------------------------------------------------------------------------- /leveldb/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package errors provides common error types used throughout leveldb. 8 | package errors 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | 14 | "github.com/pingcap/goleveldb/leveldb/storage" 15 | "github.com/pingcap/goleveldb/leveldb/util" 16 | ) 17 | 18 | // Common errors. 19 | var ( 20 | ErrNotFound = New("leveldb: not found") 21 | ErrReleased = util.ErrReleased 22 | ErrHasReleaser = util.ErrHasReleaser 23 | ) 24 | 25 | // New returns an error that formats as the given text. 26 | func New(text string) error { 27 | return errors.New(text) 28 | } 29 | 30 | // ErrCorrupted is the type that wraps errors that indicate corruption in 31 | // the database. 32 | type ErrCorrupted struct { 33 | Fd storage.FileDesc 34 | Err error 35 | } 36 | 37 | func (e *ErrCorrupted) Error() string { 38 | if !e.Fd.Zero() { 39 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 40 | } 41 | return e.Err.Error() 42 | } 43 | 44 | // NewErrCorrupted creates new ErrCorrupted error. 45 | func NewErrCorrupted(fd storage.FileDesc, err error) error { 46 | return &ErrCorrupted{fd, err} 47 | } 48 | 49 | // IsCorrupted returns a boolean indicating whether the error is indicating 50 | // a corruption. 51 | func IsCorrupted(err error) bool { 52 | switch err.(type) { 53 | case *ErrCorrupted: 54 | return true 55 | case *storage.ErrCorrupted: 56 | return true 57 | } 58 | return false 59 | } 60 | 61 | // ErrMissingFiles is the type that indicating a corruption due to missing 62 | // files. ErrMissingFiles always wrapped with ErrCorrupted. 63 | type ErrMissingFiles struct { 64 | Fds []storage.FileDesc 65 | } 66 | 67 | func (e *ErrMissingFiles) Error() string { return "file missing" } 68 | 69 | // SetFd sets 'file info' of the given error with the given file. 70 | // Currently only ErrCorrupted is supported, otherwise will do nothing. 71 | func SetFd(err error, fd storage.FileDesc) error { 72 | switch x := err.(type) { 73 | case *ErrCorrupted: 74 | x.Fd = fd 75 | return x 76 | } 77 | return err 78 | } 79 | -------------------------------------------------------------------------------- /leveldb/external_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/opt" 14 | "github.com/pingcap/goleveldb/leveldb/testutil" 15 | ) 16 | 17 | var _ = testutil.Defer(func() { 18 | Describe("Leveldb external", func() { 19 | o := &opt.Options{ 20 | DisableBlockCache: true, 21 | BlockRestartInterval: 5, 22 | BlockSize: 80, 23 | Compression: opt.NoCompression, 24 | OpenFilesCacheCapacity: -1, 25 | Strict: opt.StrictAll, 26 | WriteBuffer: 1000, 27 | CompactionTableSize: 2000, 28 | } 29 | 30 | Describe("write test", func() { 31 | It("should do write correctly", func(done Done) { 32 | db := newTestingDB(o, nil, nil) 33 | t := testutil.DBTesting{ 34 | DB: db, 35 | Deleted: testutil.KeyValue_Generate(nil, 500, 1, 1, 50, 5, 5).Clone(), 36 | } 37 | testutil.DoDBTesting(&t) 38 | db.TestClose() 39 | done <- true 40 | }, 80.0) 41 | }) 42 | 43 | Describe("read test", func() { 44 | testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB { 45 | // Building the DB. 46 | db := newTestingDB(o, nil, nil) 47 | kv.IterateShuffled(nil, func(i int, key, value []byte) { 48 | err := db.TestPut(key, value) 49 | Expect(err).NotTo(HaveOccurred()) 50 | }) 51 | 52 | return db 53 | }, func(db testutil.DB) { 54 | db.(*testingDB).TestClose() 55 | }) 56 | }) 57 | 58 | Describe("transaction test", func() { 59 | It("should do transaction correctly", func(done Done) { 60 | db := newTestingDB(o, nil, nil) 61 | 62 | By("creating first transaction") 63 | var err error 64 | tr := &testingTransaction{} 65 | tr.Transaction, err = db.OpenTransaction() 66 | Expect(err).NotTo(HaveOccurred()) 67 | t0 := &testutil.DBTesting{ 68 | DB: tr, 69 | Deleted: testutil.KeyValue_Generate(nil, 200, 1, 1, 50, 5, 5).Clone(), 70 | } 71 | testutil.DoDBTesting(t0) 72 | testutil.TestGet(tr, t0.Present) 73 | testutil.TestHas(tr, t0.Present) 74 | 75 | By("committing first transaction") 76 | err = tr.Commit() 77 | Expect(err).NotTo(HaveOccurred()) 78 | testutil.TestIter(db, nil, t0.Present) 79 | testutil.TestGet(db, t0.Present) 80 | testutil.TestHas(db, t0.Present) 81 | 82 | By("manipulating DB without transaction") 83 | t0.DB = db 84 | testutil.DoDBTesting(t0) 85 | 86 | By("creating second transaction") 87 | tr.Transaction, err = db.OpenTransaction() 88 | Expect(err).NotTo(HaveOccurred()) 89 | t1 := &testutil.DBTesting{ 90 | DB: tr, 91 | Deleted: t0.Deleted.Clone(), 92 | Present: t0.Present.Clone(), 93 | } 94 | testutil.DoDBTesting(t1) 95 | testutil.TestIter(db, nil, t0.Present) 96 | 97 | By("discarding second transaction") 98 | tr.Discard() 99 | testutil.TestIter(db, nil, t0.Present) 100 | 101 | By("creating third transaction") 102 | tr.Transaction, err = db.OpenTransaction() 103 | Expect(err).NotTo(HaveOccurred()) 104 | t0.DB = tr 105 | testutil.DoDBTesting(t0) 106 | 107 | By("committing third transaction") 108 | err = tr.Commit() 109 | Expect(err).NotTo(HaveOccurred()) 110 | testutil.TestIter(db, nil, t0.Present) 111 | 112 | db.TestClose() 113 | done <- true 114 | }, 240.0) 115 | }) 116 | }) 117 | }) 118 | -------------------------------------------------------------------------------- /leveldb/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/filter" 11 | ) 12 | 13 | type iFilter struct { 14 | filter.Filter 15 | } 16 | 17 | func (f iFilter) Contains(filter, key []byte) bool { 18 | return f.Filter.Contains(filter, internalKey(key).ukey()) 19 | } 20 | 21 | func (f iFilter) NewGenerator() filter.FilterGenerator { 22 | return iFilterGenerator{f.Filter.NewGenerator()} 23 | } 24 | 25 | type iFilterGenerator struct { 26 | filter.FilterGenerator 27 | } 28 | 29 | func (g iFilterGenerator) Add(key []byte) { 30 | g.FilterGenerator.Add(internalKey(key).ukey()) 31 | } 32 | -------------------------------------------------------------------------------- /leveldb/filter/bloom.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package filter 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/util" 11 | ) 12 | 13 | func bloomHash(key []byte) uint32 { 14 | return util.Hash(key, 0xbc9f1d34) 15 | } 16 | 17 | type bloomFilter int 18 | 19 | // The bloom filter serializes its parameters and is backward compatible 20 | // with respect to them. Therefor, its parameters are not added to its 21 | // name. 22 | func (bloomFilter) Name() string { 23 | return "leveldb.BuiltinBloomFilter" 24 | } 25 | 26 | func (f bloomFilter) Contains(filter, key []byte) bool { 27 | nBytes := len(filter) - 1 28 | if nBytes < 1 { 29 | return false 30 | } 31 | nBits := uint32(nBytes * 8) 32 | 33 | // Use the encoded k so that we can read filters generated by 34 | // bloom filters created using different parameters. 35 | k := filter[nBytes] 36 | if k > 30 { 37 | // Reserved for potentially new encodings for short bloom filters. 38 | // Consider it a match. 39 | return true 40 | } 41 | 42 | kh := bloomHash(key) 43 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 44 | for j := uint8(0); j < k; j++ { 45 | bitpos := kh % nBits 46 | if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 { 47 | return false 48 | } 49 | kh += delta 50 | } 51 | return true 52 | } 53 | 54 | func (f bloomFilter) NewGenerator() FilterGenerator { 55 | // Round down to reduce probing cost a little bit. 56 | k := uint8(f * 69 / 100) // 0.69 =~ ln(2) 57 | if k < 1 { 58 | k = 1 59 | } else if k > 30 { 60 | k = 30 61 | } 62 | return &bloomFilterGenerator{ 63 | n: int(f), 64 | k: k, 65 | } 66 | } 67 | 68 | type bloomFilterGenerator struct { 69 | n int 70 | k uint8 71 | 72 | keyHashes []uint32 73 | } 74 | 75 | func (g *bloomFilterGenerator) Add(key []byte) { 76 | // Use double-hashing to generate a sequence of hash values. 77 | // See analysis in [Kirsch,Mitzenmacher 2006]. 78 | g.keyHashes = append(g.keyHashes, bloomHash(key)) 79 | } 80 | 81 | func (g *bloomFilterGenerator) Generate(b Buffer) { 82 | // Compute bloom filter size (in both bits and bytes) 83 | nBits := uint32(len(g.keyHashes) * g.n) 84 | // For small n, we can see a very high false positive rate. Fix it 85 | // by enforcing a minimum bloom filter length. 86 | if nBits < 64 { 87 | nBits = 64 88 | } 89 | nBytes := (nBits + 7) / 8 90 | nBits = nBytes * 8 91 | 92 | dest := b.Alloc(int(nBytes) + 1) 93 | dest[nBytes] = g.k 94 | for _, kh := range g.keyHashes { 95 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 96 | for j := uint8(0); j < g.k; j++ { 97 | bitpos := kh % nBits 98 | dest[bitpos/8] |= (1 << (bitpos % 8)) 99 | kh += delta 100 | } 101 | } 102 | 103 | g.keyHashes = g.keyHashes[:0] 104 | } 105 | 106 | // NewBloomFilter creates a new initialized bloom filter for given 107 | // bitsPerKey. 108 | // 109 | // Since bitsPerKey is persisted individually for each bloom filter 110 | // serialization, bloom filters are backwards compatible with respect to 111 | // changing bitsPerKey. This means that no big performance penalty will 112 | // be experienced when changing the parameter. See documentation for 113 | // opt.Options.Filter for more information. 114 | func NewBloomFilter(bitsPerKey int) Filter { 115 | return bloomFilter(bitsPerKey) 116 | } 117 | -------------------------------------------------------------------------------- /leveldb/filter/bloom_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package filter 8 | 9 | import ( 10 | "encoding/binary" 11 | "github.com/pingcap/goleveldb/leveldb/util" 12 | "testing" 13 | ) 14 | 15 | type harness struct { 16 | t *testing.T 17 | 18 | bloom Filter 19 | generator FilterGenerator 20 | filter []byte 21 | } 22 | 23 | func newHarness(t *testing.T) *harness { 24 | bloom := NewBloomFilter(10) 25 | return &harness{ 26 | t: t, 27 | bloom: bloom, 28 | generator: bloom.NewGenerator(), 29 | } 30 | } 31 | 32 | func (h *harness) add(key []byte) { 33 | h.generator.Add(key) 34 | } 35 | 36 | func (h *harness) addNum(key uint32) { 37 | var b [4]byte 38 | binary.LittleEndian.PutUint32(b[:], key) 39 | h.add(b[:]) 40 | } 41 | 42 | func (h *harness) build() { 43 | b := &util.Buffer{} 44 | h.generator.Generate(b) 45 | h.filter = b.Bytes() 46 | } 47 | 48 | func (h *harness) reset() { 49 | h.filter = nil 50 | } 51 | 52 | func (h *harness) filterLen() int { 53 | return len(h.filter) 54 | } 55 | 56 | func (h *harness) assert(key []byte, want, silent bool) bool { 57 | got := h.bloom.Contains(h.filter, key) 58 | if !silent && got != want { 59 | h.t.Errorf("assert on '%v' failed got '%v', want '%v'", key, got, want) 60 | } 61 | return got 62 | } 63 | 64 | func (h *harness) assertNum(key uint32, want, silent bool) bool { 65 | var b [4]byte 66 | binary.LittleEndian.PutUint32(b[:], key) 67 | return h.assert(b[:], want, silent) 68 | } 69 | 70 | func TestBloomFilter_Empty(t *testing.T) { 71 | h := newHarness(t) 72 | h.build() 73 | h.assert([]byte("hello"), false, false) 74 | h.assert([]byte("world"), false, false) 75 | } 76 | 77 | func TestBloomFilter_Small(t *testing.T) { 78 | h := newHarness(t) 79 | h.add([]byte("hello")) 80 | h.add([]byte("world")) 81 | h.build() 82 | h.assert([]byte("hello"), true, false) 83 | h.assert([]byte("world"), true, false) 84 | h.assert([]byte("x"), false, false) 85 | h.assert([]byte("foo"), false, false) 86 | } 87 | 88 | func nextN(n int) int { 89 | switch { 90 | case n < 10: 91 | n += 1 92 | case n < 100: 93 | n += 10 94 | case n < 1000: 95 | n += 100 96 | default: 97 | n += 1000 98 | } 99 | return n 100 | } 101 | 102 | func TestBloomFilter_VaryingLengths(t *testing.T) { 103 | h := newHarness(t) 104 | var mediocre, good int 105 | for n := 1; n < 10000; n = nextN(n) { 106 | h.reset() 107 | for i := 0; i < n; i++ { 108 | h.addNum(uint32(i)) 109 | } 110 | h.build() 111 | 112 | got := h.filterLen() 113 | want := (n * 10 / 8) + 40 114 | if got > want { 115 | t.Errorf("filter len test failed, '%d' > '%d'", got, want) 116 | } 117 | 118 | for i := 0; i < n; i++ { 119 | h.assertNum(uint32(i), true, false) 120 | } 121 | 122 | var rate float32 123 | for i := 0; i < 10000; i++ { 124 | if h.assertNum(uint32(i+1000000000), true, true) { 125 | rate++ 126 | } 127 | } 128 | rate /= 10000 129 | if rate > 0.02 { 130 | t.Errorf("false positive rate is more than 2%%, got %v, at len %d", rate, n) 131 | } 132 | if rate > 0.0125 { 133 | mediocre++ 134 | } else { 135 | good++ 136 | } 137 | } 138 | t.Logf("false positive rate: %d good, %d mediocre", good, mediocre) 139 | if mediocre > good/5 { 140 | t.Error("mediocre false positive rate is more than expected") 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /leveldb/filter/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package filter provides interface and implementation of probabilistic 8 | // data structure. 9 | // 10 | // The filter is resposible for creating small filter from a set of keys. 11 | // These filter will then used to test whether a key is a member of the set. 12 | // In many cases, a filter can cut down the number of disk seeks from a 13 | // handful to a single disk seek per DB.Get call. 14 | package filter 15 | 16 | // Buffer is the interface that wraps basic Alloc, Write and WriteByte methods. 17 | type Buffer interface { 18 | // Alloc allocs n bytes of slice from the buffer. This also advancing 19 | // write offset. 20 | Alloc(n int) []byte 21 | 22 | // Write appends the contents of p to the buffer. 23 | Write(p []byte) (n int, err error) 24 | 25 | // WriteByte appends the byte c to the buffer. 26 | WriteByte(c byte) error 27 | } 28 | 29 | // Filter is the filter. 30 | type Filter interface { 31 | // Name returns the name of this policy. 32 | // 33 | // Note that if the filter encoding changes in an incompatible way, 34 | // the name returned by this method must be changed. Otherwise, old 35 | // incompatible filters may be passed to methods of this type. 36 | Name() string 37 | 38 | // NewGenerator creates a new filter generator. 39 | NewGenerator() FilterGenerator 40 | 41 | // Contains returns true if the filter contains the given key. 42 | // 43 | // The filter are filters generated by the filter generator. 44 | Contains(filter, key []byte) bool 45 | } 46 | 47 | // FilterGenerator is the filter generator. 48 | type FilterGenerator interface { 49 | // Add adds a key to the filter generator. 50 | // 51 | // The key may become invalid after call to this method end, therefor 52 | // key must be copied if implementation require keeping key for later 53 | // use. The key should not modified directly, doing so may cause 54 | // undefined results. 55 | Add(key []byte) 56 | 57 | // Generate generates filters based on keys passed so far. After call 58 | // to Generate the filter generator maybe resetted, depends on implementation. 59 | Generate(b Buffer) 60 | } 61 | -------------------------------------------------------------------------------- /leveldb/iterator/array_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/util" 11 | ) 12 | 13 | // BasicArray is the interface that wraps basic Len and Search method. 14 | type BasicArray interface { 15 | // Len returns length of the array. 16 | Len() int 17 | 18 | // Search finds smallest index that point to a key that is greater 19 | // than or equal to the given key. 20 | Search(key []byte) int 21 | } 22 | 23 | // Array is the interface that wraps BasicArray and basic Index method. 24 | type Array interface { 25 | BasicArray 26 | 27 | // Index returns key/value pair with index of i. 28 | Index(i int) (key, value []byte) 29 | } 30 | 31 | // Array is the interface that wraps BasicArray and basic Get method. 32 | type ArrayIndexer interface { 33 | BasicArray 34 | 35 | // Get returns a new data iterator with index of i. 36 | Get(i int) Iterator 37 | } 38 | 39 | type basicArrayIterator struct { 40 | util.BasicReleaser 41 | array BasicArray 42 | pos int 43 | err error 44 | } 45 | 46 | func (i *basicArrayIterator) Valid() bool { 47 | return i.pos >= 0 && i.pos < i.array.Len() && !i.Released() 48 | } 49 | 50 | func (i *basicArrayIterator) First() bool { 51 | if i.Released() { 52 | i.err = ErrIterReleased 53 | return false 54 | } 55 | 56 | if i.array.Len() == 0 { 57 | i.pos = -1 58 | return false 59 | } 60 | i.pos = 0 61 | return true 62 | } 63 | 64 | func (i *basicArrayIterator) Last() bool { 65 | if i.Released() { 66 | i.err = ErrIterReleased 67 | return false 68 | } 69 | 70 | n := i.array.Len() 71 | if n == 0 { 72 | i.pos = 0 73 | return false 74 | } 75 | i.pos = n - 1 76 | return true 77 | } 78 | 79 | func (i *basicArrayIterator) Seek(key []byte) bool { 80 | if i.Released() { 81 | i.err = ErrIterReleased 82 | return false 83 | } 84 | 85 | n := i.array.Len() 86 | if n == 0 { 87 | i.pos = 0 88 | return false 89 | } 90 | i.pos = i.array.Search(key) 91 | if i.pos >= n { 92 | return false 93 | } 94 | return true 95 | } 96 | 97 | func (i *basicArrayIterator) Next() bool { 98 | if i.Released() { 99 | i.err = ErrIterReleased 100 | return false 101 | } 102 | 103 | i.pos++ 104 | if n := i.array.Len(); i.pos >= n { 105 | i.pos = n 106 | return false 107 | } 108 | return true 109 | } 110 | 111 | func (i *basicArrayIterator) Prev() bool { 112 | if i.Released() { 113 | i.err = ErrIterReleased 114 | return false 115 | } 116 | 117 | i.pos-- 118 | if i.pos < 0 { 119 | i.pos = -1 120 | return false 121 | } 122 | return true 123 | } 124 | 125 | func (i *basicArrayIterator) Error() error { return i.err } 126 | 127 | type arrayIterator struct { 128 | basicArrayIterator 129 | array Array 130 | pos int 131 | key, value []byte 132 | } 133 | 134 | func (i *arrayIterator) updateKV() { 135 | if i.pos == i.basicArrayIterator.pos { 136 | return 137 | } 138 | i.pos = i.basicArrayIterator.pos 139 | if i.Valid() { 140 | i.key, i.value = i.array.Index(i.pos) 141 | } else { 142 | i.key = nil 143 | i.value = nil 144 | } 145 | } 146 | 147 | func (i *arrayIterator) Key() []byte { 148 | i.updateKV() 149 | return i.key 150 | } 151 | 152 | func (i *arrayIterator) Value() []byte { 153 | i.updateKV() 154 | return i.value 155 | } 156 | 157 | type arrayIteratorIndexer struct { 158 | basicArrayIterator 159 | array ArrayIndexer 160 | } 161 | 162 | func (i *arrayIteratorIndexer) Get() Iterator { 163 | if i.Valid() { 164 | return i.array.Get(i.basicArrayIterator.pos) 165 | } 166 | return nil 167 | } 168 | 169 | // NewArrayIterator returns an iterator from the given array. 170 | func NewArrayIterator(array Array) Iterator { 171 | return &arrayIterator{ 172 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 173 | array: array, 174 | pos: -1, 175 | } 176 | } 177 | 178 | // NewArrayIndexer returns an index iterator from the given array. 179 | func NewArrayIndexer(array ArrayIndexer) IteratorIndexer { 180 | return &arrayIteratorIndexer{ 181 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 182 | array: array, 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /leveldb/iterator/array_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | 12 | . "github.com/pingcap/goleveldb/leveldb/iterator" 13 | "github.com/pingcap/goleveldb/leveldb/testutil" 14 | ) 15 | 16 | var _ = testutil.Defer(func() { 17 | Describe("Array iterator", func() { 18 | It("Should iterates and seeks correctly", func() { 19 | // Build key/value. 20 | kv := testutil.KeyValue_Generate(nil, 70, 1, 1, 5, 3, 3) 21 | 22 | // Test the iterator. 23 | t := testutil.IteratorTesting{ 24 | KeyValue: kv.Clone(), 25 | Iter: NewArrayIterator(kv), 26 | } 27 | testutil.DoIteratorTesting(&t) 28 | }) 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /leveldb/iterator/indexed_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/errors" 11 | "github.com/pingcap/goleveldb/leveldb/util" 12 | ) 13 | 14 | // IteratorIndexer is the interface that wraps CommonIterator and basic Get 15 | // method. IteratorIndexer provides index for indexed iterator. 16 | type IteratorIndexer interface { 17 | CommonIterator 18 | 19 | // Get returns a new data iterator for the current position, or nil if 20 | // done. 21 | Get() Iterator 22 | } 23 | 24 | type indexedIterator struct { 25 | util.BasicReleaser 26 | index IteratorIndexer 27 | strict bool 28 | 29 | data Iterator 30 | err error 31 | errf func(err error) 32 | closed bool 33 | } 34 | 35 | func (i *indexedIterator) setData() { 36 | if i.data != nil { 37 | i.data.Release() 38 | } 39 | i.data = i.index.Get() 40 | } 41 | 42 | func (i *indexedIterator) clearData() { 43 | if i.data != nil { 44 | i.data.Release() 45 | } 46 | i.data = nil 47 | } 48 | 49 | func (i *indexedIterator) indexErr() { 50 | if err := i.index.Error(); err != nil { 51 | if i.errf != nil { 52 | i.errf(err) 53 | } 54 | i.err = err 55 | } 56 | } 57 | 58 | func (i *indexedIterator) dataErr() bool { 59 | if err := i.data.Error(); err != nil { 60 | if i.errf != nil { 61 | i.errf(err) 62 | } 63 | if i.strict || !errors.IsCorrupted(err) { 64 | i.err = err 65 | return true 66 | } 67 | } 68 | return false 69 | } 70 | 71 | func (i *indexedIterator) Valid() bool { 72 | return i.data != nil && i.data.Valid() 73 | } 74 | 75 | func (i *indexedIterator) First() bool { 76 | if i.err != nil { 77 | return false 78 | } else if i.Released() { 79 | i.err = ErrIterReleased 80 | return false 81 | } 82 | 83 | if !i.index.First() { 84 | i.indexErr() 85 | i.clearData() 86 | return false 87 | } 88 | i.setData() 89 | return i.Next() 90 | } 91 | 92 | func (i *indexedIterator) Last() bool { 93 | if i.err != nil { 94 | return false 95 | } else if i.Released() { 96 | i.err = ErrIterReleased 97 | return false 98 | } 99 | 100 | if !i.index.Last() { 101 | i.indexErr() 102 | i.clearData() 103 | return false 104 | } 105 | i.setData() 106 | if !i.data.Last() { 107 | if i.dataErr() { 108 | return false 109 | } 110 | i.clearData() 111 | return i.Prev() 112 | } 113 | return true 114 | } 115 | 116 | func (i *indexedIterator) Seek(key []byte) bool { 117 | if i.err != nil { 118 | return false 119 | } else if i.Released() { 120 | i.err = ErrIterReleased 121 | return false 122 | } 123 | 124 | if !i.index.Seek(key) { 125 | i.indexErr() 126 | i.clearData() 127 | return false 128 | } 129 | i.setData() 130 | if !i.data.Seek(key) { 131 | if i.dataErr() { 132 | return false 133 | } 134 | i.clearData() 135 | return i.Next() 136 | } 137 | return true 138 | } 139 | 140 | func (i *indexedIterator) Next() bool { 141 | if i.err != nil { 142 | return false 143 | } else if i.Released() { 144 | i.err = ErrIterReleased 145 | return false 146 | } 147 | 148 | switch { 149 | case i.data != nil && !i.data.Next(): 150 | if i.dataErr() { 151 | return false 152 | } 153 | i.clearData() 154 | fallthrough 155 | case i.data == nil: 156 | if !i.index.Next() { 157 | i.indexErr() 158 | return false 159 | } 160 | i.setData() 161 | return i.Next() 162 | } 163 | return true 164 | } 165 | 166 | func (i *indexedIterator) Prev() bool { 167 | if i.err != nil { 168 | return false 169 | } else if i.Released() { 170 | i.err = ErrIterReleased 171 | return false 172 | } 173 | 174 | switch { 175 | case i.data != nil && !i.data.Prev(): 176 | if i.dataErr() { 177 | return false 178 | } 179 | i.clearData() 180 | fallthrough 181 | case i.data == nil: 182 | if !i.index.Prev() { 183 | i.indexErr() 184 | return false 185 | } 186 | i.setData() 187 | if !i.data.Last() { 188 | if i.dataErr() { 189 | return false 190 | } 191 | i.clearData() 192 | return i.Prev() 193 | } 194 | } 195 | return true 196 | } 197 | 198 | func (i *indexedIterator) Key() []byte { 199 | if i.data == nil { 200 | return nil 201 | } 202 | return i.data.Key() 203 | } 204 | 205 | func (i *indexedIterator) Value() []byte { 206 | if i.data == nil { 207 | return nil 208 | } 209 | return i.data.Value() 210 | } 211 | 212 | func (i *indexedIterator) Release() { 213 | i.clearData() 214 | i.index.Release() 215 | i.BasicReleaser.Release() 216 | } 217 | 218 | func (i *indexedIterator) Error() error { 219 | if i.err != nil { 220 | return i.err 221 | } 222 | if err := i.index.Error(); err != nil { 223 | return err 224 | } 225 | return nil 226 | } 227 | 228 | func (i *indexedIterator) SetErrorCallback(f func(err error)) { 229 | i.errf = f 230 | } 231 | 232 | // NewIndexedIterator returns an 'indexed iterator'. An index is iterator 233 | // that returns another iterator, a 'data iterator'. A 'data iterator' is the 234 | // iterator that contains actual key/value pairs. 235 | // 236 | // If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) 237 | // won't be ignored and will halt 'indexed iterator', otherwise the iterator will 238 | // continue to the next 'data iterator'. Corruption on 'index iterator' will not be 239 | // ignored and will halt the iterator. 240 | func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator { 241 | return &indexedIterator{index: index, strict: strict} 242 | } 243 | -------------------------------------------------------------------------------- /leveldb/iterator/indexed_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | "sort" 11 | 12 | . "github.com/onsi/ginkgo" 13 | 14 | "github.com/pingcap/goleveldb/leveldb/comparer" 15 | . "github.com/pingcap/goleveldb/leveldb/iterator" 16 | "github.com/pingcap/goleveldb/leveldb/testutil" 17 | ) 18 | 19 | type keyValue struct { 20 | key []byte 21 | testutil.KeyValue 22 | } 23 | 24 | type keyValueIndex []keyValue 25 | 26 | func (x keyValueIndex) Search(key []byte) int { 27 | return sort.Search(x.Len(), func(i int) bool { 28 | return comparer.DefaultComparer.Compare(x[i].key, key) >= 0 29 | }) 30 | } 31 | 32 | func (x keyValueIndex) Len() int { return len(x) } 33 | func (x keyValueIndex) Index(i int) (key, value []byte) { return x[i].key, nil } 34 | func (x keyValueIndex) Get(i int) Iterator { return NewArrayIterator(x[i]) } 35 | 36 | var _ = testutil.Defer(func() { 37 | Describe("Indexed iterator", func() { 38 | Test := func(n ...int) func() { 39 | if len(n) == 0 { 40 | rnd := testutil.NewRand() 41 | n = make([]int, rnd.Intn(17)+3) 42 | for i := range n { 43 | n[i] = rnd.Intn(19) + 1 44 | } 45 | } 46 | 47 | return func() { 48 | It("Should iterates and seeks correctly", func(done Done) { 49 | // Build key/value. 50 | index := make(keyValueIndex, len(n)) 51 | sum := 0 52 | for _, x := range n { 53 | sum += x 54 | } 55 | kv := testutil.KeyValue_Generate(nil, sum, 1, 1, 10, 4, 4) 56 | for i, j := 0, 0; i < len(n); i++ { 57 | for x := n[i]; x > 0; x-- { 58 | key, value := kv.Index(j) 59 | index[i].key = key 60 | index[i].Put(key, value) 61 | j++ 62 | } 63 | } 64 | 65 | // Test the iterator. 66 | t := testutil.IteratorTesting{ 67 | KeyValue: kv.Clone(), 68 | Iter: NewIndexedIterator(NewArrayIndexer(index), true), 69 | } 70 | testutil.DoIteratorTesting(&t) 71 | done <- true 72 | }, 15.0) 73 | } 74 | } 75 | 76 | Describe("with 100 keys", Test(100)) 77 | Describe("with 50-50 keys", Test(50, 50)) 78 | Describe("with 50-1 keys", Test(50, 1)) 79 | Describe("with 50-1-50 keys", Test(50, 1, 50)) 80 | Describe("with 1-50 keys", Test(1, 50)) 81 | Describe("with random N-keys", Test()) 82 | }) 83 | }) 84 | -------------------------------------------------------------------------------- /leveldb/iterator/iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package iterator provides interface and implementation to traverse over 8 | // contents of a database. 9 | package iterator 10 | 11 | import ( 12 | "errors" 13 | 14 | "github.com/pingcap/goleveldb/leveldb/util" 15 | ) 16 | 17 | var ( 18 | ErrIterReleased = errors.New("leveldb/iterator: iterator released") 19 | ) 20 | 21 | // IteratorSeeker is the interface that wraps the 'seeks method'. 22 | type IteratorSeeker interface { 23 | // First moves the iterator to the first key/value pair. If the iterator 24 | // only contains one key/value pair then First and Last would moves 25 | // to the same key/value pair. 26 | // It returns whether such pair exist. 27 | First() bool 28 | 29 | // Last moves the iterator to the last key/value pair. If the iterator 30 | // only contains one key/value pair then First and Last would moves 31 | // to the same key/value pair. 32 | // It returns whether such pair exist. 33 | Last() bool 34 | 35 | // Seek moves the iterator to the first key/value pair whose key is greater 36 | // than or equal to the given key. 37 | // It returns whether such pair exist. 38 | // 39 | // It is safe to modify the contents of the argument after Seek returns. 40 | Seek(key []byte) bool 41 | 42 | // Next moves the iterator to the next key/value pair. 43 | // It returns whether the iterator is exhausted. 44 | Next() bool 45 | 46 | // Prev moves the iterator to the previous key/value pair. 47 | // It returns whether the iterator is exhausted. 48 | Prev() bool 49 | } 50 | 51 | // CommonIterator is the interface that wraps common iterator methods. 52 | type CommonIterator interface { 53 | IteratorSeeker 54 | 55 | // util.Releaser is the interface that wraps basic Release method. 56 | // When called Release will releases any resources associated with the 57 | // iterator. 58 | util.Releaser 59 | 60 | // util.ReleaseSetter is the interface that wraps the basic SetReleaser 61 | // method. 62 | util.ReleaseSetter 63 | 64 | // TODO: Remove this when ready. 65 | Valid() bool 66 | 67 | // Error returns any accumulated error. Exhausting all the key/value pairs 68 | // is not considered to be an error. 69 | Error() error 70 | } 71 | 72 | // Iterator iterates over a DB's key/value pairs in key order. 73 | // 74 | // When encounter an error any 'seeks method' will return false and will 75 | // yield no key/value pairs. The error can be queried by calling the Error 76 | // method. Calling Release is still necessary. 77 | // 78 | // An iterator must be released after use, but it is not necessary to read 79 | // an iterator until exhaustion. 80 | // Also, an iterator is not necessarily safe for concurrent use, but it is 81 | // safe to use multiple iterators concurrently, with each in a dedicated 82 | // goroutine. 83 | type Iterator interface { 84 | CommonIterator 85 | 86 | // Key returns the key of the current key/value pair, or nil if done. 87 | // The caller should not modify the contents of the returned slice, and 88 | // its contents may change on the next call to any 'seeks method'. 89 | Key() []byte 90 | 91 | // Value returns the key of the current key/value pair, or nil if done. 92 | // The caller should not modify the contents of the returned slice, and 93 | // its contents may change on the next call to any 'seeks method'. 94 | Value() []byte 95 | } 96 | 97 | // ErrorCallbackSetter is the interface that wraps basic SetErrorCallback 98 | // method. 99 | // 100 | // ErrorCallbackSetter implemented by indexed and merged iterator. 101 | type ErrorCallbackSetter interface { 102 | // SetErrorCallback allows set an error callback of the corresponding 103 | // iterator. Use nil to clear the callback. 104 | SetErrorCallback(f func(err error)) 105 | } 106 | 107 | type emptyIterator struct { 108 | util.BasicReleaser 109 | err error 110 | } 111 | 112 | func (i *emptyIterator) rErr() { 113 | if i.err == nil && i.Released() { 114 | i.err = ErrIterReleased 115 | } 116 | } 117 | 118 | func (*emptyIterator) Valid() bool { return false } 119 | func (i *emptyIterator) First() bool { i.rErr(); return false } 120 | func (i *emptyIterator) Last() bool { i.rErr(); return false } 121 | func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false } 122 | func (i *emptyIterator) Next() bool { i.rErr(); return false } 123 | func (i *emptyIterator) Prev() bool { i.rErr(); return false } 124 | func (*emptyIterator) Key() []byte { return nil } 125 | func (*emptyIterator) Value() []byte { return nil } 126 | func (i *emptyIterator) Error() error { return i.err } 127 | 128 | // NewEmptyIterator creates an empty iterator. The err parameter can be 129 | // nil, but if not nil the given err will be returned by Error method. 130 | func NewEmptyIterator(err error) Iterator { 131 | return &emptyIterator{err: err} 132 | } 133 | -------------------------------------------------------------------------------- /leveldb/iterator/iter_suite_test.go: -------------------------------------------------------------------------------- 1 | package iterator_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pingcap/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestIterator(t *testing.T) { 10 | testutil.RunSuite(t, "Iterator Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/iterator/merged_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/comparer" 11 | "github.com/pingcap/goleveldb/leveldb/errors" 12 | "github.com/pingcap/goleveldb/leveldb/util" 13 | ) 14 | 15 | type dir int 16 | 17 | const ( 18 | dirReleased dir = iota - 1 19 | dirSOI 20 | dirEOI 21 | dirBackward 22 | dirForward 23 | ) 24 | 25 | type mergedIterator struct { 26 | cmp comparer.Comparer 27 | iters []Iterator 28 | strict bool 29 | 30 | keys [][]byte 31 | index int 32 | dir dir 33 | err error 34 | errf func(err error) 35 | releaser util.Releaser 36 | } 37 | 38 | func assertKey(key []byte) []byte { 39 | if key == nil { 40 | panic("leveldb/iterator: nil key") 41 | } 42 | return key 43 | } 44 | 45 | func (i *mergedIterator) iterErr(iter Iterator) bool { 46 | if err := iter.Error(); err != nil { 47 | if i.errf != nil { 48 | i.errf(err) 49 | } 50 | if i.strict || !errors.IsCorrupted(err) { 51 | i.err = err 52 | return true 53 | } 54 | } 55 | return false 56 | } 57 | 58 | func (i *mergedIterator) Valid() bool { 59 | return i.err == nil && i.dir > dirEOI 60 | } 61 | 62 | func (i *mergedIterator) First() bool { 63 | if i.err != nil { 64 | return false 65 | } else if i.dir == dirReleased { 66 | i.err = ErrIterReleased 67 | return false 68 | } 69 | 70 | for x, iter := range i.iters { 71 | switch { 72 | case iter.First(): 73 | i.keys[x] = assertKey(iter.Key()) 74 | case i.iterErr(iter): 75 | return false 76 | default: 77 | i.keys[x] = nil 78 | } 79 | } 80 | i.dir = dirSOI 81 | return i.next() 82 | } 83 | 84 | func (i *mergedIterator) Last() bool { 85 | if i.err != nil { 86 | return false 87 | } else if i.dir == dirReleased { 88 | i.err = ErrIterReleased 89 | return false 90 | } 91 | 92 | for x, iter := range i.iters { 93 | switch { 94 | case iter.Last(): 95 | i.keys[x] = assertKey(iter.Key()) 96 | case i.iterErr(iter): 97 | return false 98 | default: 99 | i.keys[x] = nil 100 | } 101 | } 102 | i.dir = dirEOI 103 | return i.prev() 104 | } 105 | 106 | func (i *mergedIterator) Seek(key []byte) bool { 107 | if i.err != nil { 108 | return false 109 | } else if i.dir == dirReleased { 110 | i.err = ErrIterReleased 111 | return false 112 | } 113 | 114 | for x, iter := range i.iters { 115 | switch { 116 | case iter.Seek(key): 117 | i.keys[x] = assertKey(iter.Key()) 118 | case i.iterErr(iter): 119 | return false 120 | default: 121 | i.keys[x] = nil 122 | } 123 | } 124 | i.dir = dirSOI 125 | return i.next() 126 | } 127 | 128 | func (i *mergedIterator) next() bool { 129 | var key []byte 130 | if i.dir == dirForward { 131 | key = i.keys[i.index] 132 | } 133 | for x, tkey := range i.keys { 134 | if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) < 0) { 135 | key = tkey 136 | i.index = x 137 | } 138 | } 139 | if key == nil { 140 | i.dir = dirEOI 141 | return false 142 | } 143 | i.dir = dirForward 144 | return true 145 | } 146 | 147 | func (i *mergedIterator) Next() bool { 148 | if i.dir == dirEOI || i.err != nil { 149 | return false 150 | } else if i.dir == dirReleased { 151 | i.err = ErrIterReleased 152 | return false 153 | } 154 | 155 | switch i.dir { 156 | case dirSOI: 157 | return i.First() 158 | case dirBackward: 159 | key := append([]byte{}, i.keys[i.index]...) 160 | if !i.Seek(key) { 161 | return false 162 | } 163 | return i.Next() 164 | } 165 | 166 | x := i.index 167 | iter := i.iters[x] 168 | switch { 169 | case iter.Next(): 170 | i.keys[x] = assertKey(iter.Key()) 171 | case i.iterErr(iter): 172 | return false 173 | default: 174 | i.keys[x] = nil 175 | } 176 | return i.next() 177 | } 178 | 179 | func (i *mergedIterator) prev() bool { 180 | var key []byte 181 | if i.dir == dirBackward { 182 | key = i.keys[i.index] 183 | } 184 | for x, tkey := range i.keys { 185 | if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) > 0) { 186 | key = tkey 187 | i.index = x 188 | } 189 | } 190 | if key == nil { 191 | i.dir = dirSOI 192 | return false 193 | } 194 | i.dir = dirBackward 195 | return true 196 | } 197 | 198 | func (i *mergedIterator) Prev() bool { 199 | if i.dir == dirSOI || i.err != nil { 200 | return false 201 | } else if i.dir == dirReleased { 202 | i.err = ErrIterReleased 203 | return false 204 | } 205 | 206 | switch i.dir { 207 | case dirEOI: 208 | return i.Last() 209 | case dirForward: 210 | key := append([]byte{}, i.keys[i.index]...) 211 | for x, iter := range i.iters { 212 | if x == i.index { 213 | continue 214 | } 215 | seek := iter.Seek(key) 216 | switch { 217 | case seek && iter.Prev(), !seek && iter.Last(): 218 | i.keys[x] = assertKey(iter.Key()) 219 | case i.iterErr(iter): 220 | return false 221 | default: 222 | i.keys[x] = nil 223 | } 224 | } 225 | } 226 | 227 | x := i.index 228 | iter := i.iters[x] 229 | switch { 230 | case iter.Prev(): 231 | i.keys[x] = assertKey(iter.Key()) 232 | case i.iterErr(iter): 233 | return false 234 | default: 235 | i.keys[x] = nil 236 | } 237 | return i.prev() 238 | } 239 | 240 | func (i *mergedIterator) Key() []byte { 241 | if i.err != nil || i.dir <= dirEOI { 242 | return nil 243 | } 244 | return i.keys[i.index] 245 | } 246 | 247 | func (i *mergedIterator) Value() []byte { 248 | if i.err != nil || i.dir <= dirEOI { 249 | return nil 250 | } 251 | return i.iters[i.index].Value() 252 | } 253 | 254 | func (i *mergedIterator) Release() { 255 | if i.dir != dirReleased { 256 | i.dir = dirReleased 257 | for _, iter := range i.iters { 258 | iter.Release() 259 | } 260 | i.iters = nil 261 | i.keys = nil 262 | if i.releaser != nil { 263 | i.releaser.Release() 264 | i.releaser = nil 265 | } 266 | } 267 | } 268 | 269 | func (i *mergedIterator) SetReleaser(releaser util.Releaser) { 270 | if i.dir == dirReleased { 271 | panic(util.ErrReleased) 272 | } 273 | if i.releaser != nil && releaser != nil { 274 | panic(util.ErrHasReleaser) 275 | } 276 | i.releaser = releaser 277 | } 278 | 279 | func (i *mergedIterator) Error() error { 280 | return i.err 281 | } 282 | 283 | func (i *mergedIterator) SetErrorCallback(f func(err error)) { 284 | i.errf = f 285 | } 286 | 287 | // NewMergedIterator returns an iterator that merges its input. Walking the 288 | // resultant iterator will return all key/value pairs of all input iterators 289 | // in strictly increasing key order, as defined by cmp. 290 | // The input's key ranges may overlap, but there are assumed to be no duplicate 291 | // keys: if iters[i] contains a key k then iters[j] will not contain that key k. 292 | // None of the iters may be nil. 293 | // 294 | // If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) 295 | // won't be ignored and will halt 'merged iterator', otherwise the iterator will 296 | // continue to the next 'input iterator'. 297 | func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator { 298 | return &mergedIterator{ 299 | iters: iters, 300 | cmp: cmp, 301 | strict: strict, 302 | keys: make([][]byte, len(iters)), 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /leveldb/iterator/merged_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator_test 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/comparer" 14 | . "github.com/pingcap/goleveldb/leveldb/iterator" 15 | "github.com/pingcap/goleveldb/leveldb/testutil" 16 | ) 17 | 18 | var _ = testutil.Defer(func() { 19 | Describe("Merged iterator", func() { 20 | Test := func(filled int, empty int) func() { 21 | return func() { 22 | It("Should iterates and seeks correctly", func(done Done) { 23 | rnd := testutil.NewRand() 24 | 25 | // Build key/value. 26 | filledKV := make([]testutil.KeyValue, filled) 27 | kv := testutil.KeyValue_Generate(nil, 100, 1, 1, 10, 4, 4) 28 | kv.Iterate(func(i int, key, value []byte) { 29 | filledKV[rnd.Intn(filled)].Put(key, value) 30 | }) 31 | 32 | // Create itearators. 33 | iters := make([]Iterator, filled+empty) 34 | for i := range iters { 35 | if empty == 0 || (rnd.Int()%2 == 0 && filled > 0) { 36 | filled-- 37 | Expect(filledKV[filled].Len()).ShouldNot(BeZero()) 38 | iters[i] = NewArrayIterator(filledKV[filled]) 39 | } else { 40 | empty-- 41 | iters[i] = NewEmptyIterator(nil) 42 | } 43 | } 44 | 45 | // Test the iterator. 46 | t := testutil.IteratorTesting{ 47 | KeyValue: kv.Clone(), 48 | Iter: NewMergedIterator(iters, comparer.DefaultComparer, true), 49 | } 50 | testutil.DoIteratorTesting(&t) 51 | done <- true 52 | }, 15.0) 53 | } 54 | } 55 | 56 | Describe("with three, all filled iterators", Test(3, 0)) 57 | Describe("with one filled, one empty iterators", Test(1, 1)) 58 | Describe("with one filled, two empty iterators", Test(1, 2)) 59 | }) 60 | }) 61 | -------------------------------------------------------------------------------- /leveldb/key.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "encoding/binary" 11 | "fmt" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/errors" 14 | "github.com/pingcap/goleveldb/leveldb/storage" 15 | ) 16 | 17 | // ErrInternalKeyCorrupted records internal key corruption. 18 | type ErrInternalKeyCorrupted struct { 19 | Ikey []byte 20 | Reason string 21 | } 22 | 23 | func (e *ErrInternalKeyCorrupted) Error() string { 24 | return fmt.Sprintf("leveldb: internal key %q corrupted: %s", e.Ikey, e.Reason) 25 | } 26 | 27 | func newErrInternalKeyCorrupted(ikey []byte, reason string) error { 28 | return errors.NewErrCorrupted(storage.FileDesc{}, &ErrInternalKeyCorrupted{append([]byte{}, ikey...), reason}) 29 | } 30 | 31 | type keyType uint 32 | 33 | func (kt keyType) String() string { 34 | switch kt { 35 | case keyTypeDel: 36 | return "d" 37 | case keyTypeVal: 38 | return "v" 39 | } 40 | return fmt.Sprintf("", uint(kt)) 41 | } 42 | 43 | // Value types encoded as the last component of internal keys. 44 | // Don't modify; this value are saved to disk. 45 | const ( 46 | keyTypeDel = keyType(0) 47 | keyTypeVal = keyType(1) 48 | ) 49 | 50 | // keyTypeSeek defines the keyType that should be passed when constructing an 51 | // internal key for seeking to a particular sequence number (since we 52 | // sort sequence numbers in decreasing order and the value type is 53 | // embedded as the low 8 bits in the sequence number in internal keys, 54 | // we need to use the highest-numbered ValueType, not the lowest). 55 | const keyTypeSeek = keyTypeVal 56 | 57 | const ( 58 | // Maximum value possible for sequence number; the 8-bits are 59 | // used by value type, so its can packed together in single 60 | // 64-bit integer. 61 | keyMaxSeq = (uint64(1) << 56) - 1 62 | // Maximum value possible for packed sequence number and type. 63 | keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek) 64 | ) 65 | 66 | // Maximum number encoded in bytes. 67 | var keyMaxNumBytes = make([]byte, 8) 68 | 69 | func init() { 70 | binary.LittleEndian.PutUint64(keyMaxNumBytes, keyMaxNum) 71 | } 72 | 73 | type internalKey []byte 74 | 75 | func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey { 76 | if seq > keyMaxSeq { 77 | panic("leveldb: invalid sequence number") 78 | } else if kt > keyTypeVal { 79 | panic("leveldb: invalid type") 80 | } 81 | 82 | dst = ensureBuffer(dst, len(ukey)+8) 83 | copy(dst, ukey) 84 | binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt)) 85 | return internalKey(dst) 86 | } 87 | 88 | func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) { 89 | if len(ik) < 8 { 90 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length") 91 | } 92 | num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 93 | seq, kt = uint64(num>>8), keyType(num&0xff) 94 | if kt > keyTypeVal { 95 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type") 96 | } 97 | ukey = ik[:len(ik)-8] 98 | return 99 | } 100 | 101 | func validInternalKey(ik []byte) bool { 102 | _, _, _, err := parseInternalKey(ik) 103 | return err == nil 104 | } 105 | 106 | func (ik internalKey) assert() { 107 | if ik == nil { 108 | panic("leveldb: nil internalKey") 109 | } 110 | if len(ik) < 8 { 111 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid length", []byte(ik), len(ik))) 112 | } 113 | } 114 | 115 | func (ik internalKey) ukey() []byte { 116 | ik.assert() 117 | return ik[:len(ik)-8] 118 | } 119 | 120 | func (ik internalKey) num() uint64 { 121 | ik.assert() 122 | return binary.LittleEndian.Uint64(ik[len(ik)-8:]) 123 | } 124 | 125 | func (ik internalKey) parseNum() (seq uint64, kt keyType) { 126 | num := ik.num() 127 | seq, kt = uint64(num>>8), keyType(num&0xff) 128 | if kt > keyTypeVal { 129 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt)) 130 | } 131 | return 132 | } 133 | 134 | func (ik internalKey) String() string { 135 | if ik == nil { 136 | return "" 137 | } 138 | 139 | if ukey, seq, kt, err := parseInternalKey(ik); err == nil { 140 | return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq) 141 | } 142 | return fmt.Sprintf("", []byte(ik)) 143 | } 144 | -------------------------------------------------------------------------------- /leveldb/key_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/comparer" 14 | ) 15 | 16 | var defaultIComparer = &iComparer{comparer.DefaultComparer} 17 | 18 | func ikey(key string, seq uint64, kt keyType) internalKey { 19 | return makeInternalKey(nil, []byte(key), uint64(seq), kt) 20 | } 21 | 22 | func shortSep(a, b []byte) []byte { 23 | dst := make([]byte, len(a)) 24 | dst = defaultIComparer.Separator(dst[:0], a, b) 25 | if dst == nil { 26 | return a 27 | } 28 | return dst 29 | } 30 | 31 | func shortSuccessor(b []byte) []byte { 32 | dst := make([]byte, len(b)) 33 | dst = defaultIComparer.Successor(dst[:0], b) 34 | if dst == nil { 35 | return b 36 | } 37 | return dst 38 | } 39 | 40 | func testSingleKey(t *testing.T, key string, seq uint64, kt keyType) { 41 | ik := ikey(key, seq, kt) 42 | 43 | if !bytes.Equal(ik.ukey(), []byte(key)) { 44 | t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) 45 | } 46 | 47 | rseq, rt := ik.parseNum() 48 | if rseq != seq { 49 | t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) 50 | } 51 | if rt != kt { 52 | t.Errorf("type does not equal, got %v, want %v", rt, kt) 53 | } 54 | 55 | if rukey, rseq, rt, kerr := parseInternalKey(ik); kerr == nil { 56 | if !bytes.Equal(rukey, []byte(key)) { 57 | t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) 58 | } 59 | if rseq != seq { 60 | t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) 61 | } 62 | if rt != kt { 63 | t.Errorf("type does not equal, got %v, want %v", rt, kt) 64 | } 65 | } else { 66 | t.Errorf("key error: %v", kerr) 67 | } 68 | } 69 | 70 | func TestInternalKey_EncodeDecode(t *testing.T) { 71 | keys := []string{"", "k", "hello", "longggggggggggggggggggggg"} 72 | seqs := []uint64{ 73 | 1, 2, 3, 74 | (1 << 8) - 1, 1 << 8, (1 << 8) + 1, 75 | (1 << 16) - 1, 1 << 16, (1 << 16) + 1, 76 | (1 << 32) - 1, 1 << 32, (1 << 32) + 1, 77 | } 78 | for _, key := range keys { 79 | for _, seq := range seqs { 80 | testSingleKey(t, key, seq, keyTypeVal) 81 | testSingleKey(t, "hello", 1, keyTypeDel) 82 | } 83 | } 84 | } 85 | 86 | func assertBytes(t *testing.T, want, got []byte) { 87 | if !bytes.Equal(got, want) { 88 | t.Errorf("assert failed, got %v, want %v", got, want) 89 | } 90 | } 91 | 92 | func TestInternalKeyShortSeparator(t *testing.T) { 93 | // When user keys are same 94 | assertBytes(t, ikey("foo", 100, keyTypeVal), 95 | shortSep(ikey("foo", 100, keyTypeVal), 96 | ikey("foo", 99, keyTypeVal))) 97 | assertBytes(t, ikey("foo", 100, keyTypeVal), 98 | shortSep(ikey("foo", 100, keyTypeVal), 99 | ikey("foo", 101, keyTypeVal))) 100 | assertBytes(t, ikey("foo", 100, keyTypeVal), 101 | shortSep(ikey("foo", 100, keyTypeVal), 102 | ikey("foo", 100, keyTypeVal))) 103 | assertBytes(t, ikey("foo", 100, keyTypeVal), 104 | shortSep(ikey("foo", 100, keyTypeVal), 105 | ikey("foo", 100, keyTypeDel))) 106 | 107 | // When user keys are misordered 108 | assertBytes(t, ikey("foo", 100, keyTypeVal), 109 | shortSep(ikey("foo", 100, keyTypeVal), 110 | ikey("bar", 99, keyTypeVal))) 111 | 112 | // When user keys are different, but correctly ordered 113 | assertBytes(t, ikey("g", uint64(keyMaxSeq), keyTypeSeek), 114 | shortSep(ikey("foo", 100, keyTypeVal), 115 | ikey("hello", 200, keyTypeVal))) 116 | 117 | // When start user key is prefix of limit user key 118 | assertBytes(t, ikey("foo", 100, keyTypeVal), 119 | shortSep(ikey("foo", 100, keyTypeVal), 120 | ikey("foobar", 200, keyTypeVal))) 121 | 122 | // When limit user key is prefix of start user key 123 | assertBytes(t, ikey("foobar", 100, keyTypeVal), 124 | shortSep(ikey("foobar", 100, keyTypeVal), 125 | ikey("foo", 200, keyTypeVal))) 126 | } 127 | 128 | func TestInternalKeyShortestSuccessor(t *testing.T) { 129 | assertBytes(t, ikey("g", uint64(keyMaxSeq), keyTypeSeek), 130 | shortSuccessor(ikey("foo", 100, keyTypeVal))) 131 | assertBytes(t, ikey("\xff\xff", 100, keyTypeVal), 132 | shortSuccessor(ikey("\xff\xff", 100, keyTypeVal))) 133 | } 134 | -------------------------------------------------------------------------------- /leveldb/leveldb_suite_test.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pingcap/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestLevelDB(t *testing.T) { 10 | testutil.RunSuite(t, "LevelDB Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/memdb/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package memdb 8 | 9 | import ( 10 | "encoding/binary" 11 | "math/rand" 12 | "sync" 13 | "testing" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/comparer" 16 | ) 17 | 18 | func BenchmarkPut(b *testing.B) { 19 | buf := make([][4]byte, b.N) 20 | for i := range buf { 21 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 22 | } 23 | 24 | b.ResetTimer() 25 | p := New(comparer.DefaultComparer, 0) 26 | for i := range buf { 27 | p.Put(buf[i][:], nil) 28 | } 29 | } 30 | 31 | func BenchmarkConcurrentPut(b *testing.B) { 32 | buf := make([][4]byte, b.N) 33 | for i := range buf { 34 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 35 | } 36 | 37 | b.ResetTimer() 38 | var wg sync.WaitGroup 39 | wg.Add(100) 40 | for i := 0; i < 100; i++ { 41 | go func() { 42 | db := New(comparer.DefaultComparer, 0) 43 | for i := 0; i < len(buf); i++ { 44 | db.Put(buf[i][:], nil) 45 | } 46 | db.Free() 47 | wg.Done() 48 | }() 49 | } 50 | wg.Wait() 51 | } 52 | 53 | func BenchmarkPutRandom(b *testing.B) { 54 | buf := make([][4]byte, b.N) 55 | for i := range buf { 56 | binary.LittleEndian.PutUint32(buf[i][:], uint32(rand.Int())) 57 | } 58 | 59 | b.ResetTimer() 60 | p := New(comparer.DefaultComparer, 0) 61 | for i := range buf { 62 | p.Put(buf[i][:], nil) 63 | } 64 | } 65 | 66 | func BenchmarkGet(b *testing.B) { 67 | buf := make([][4]byte, b.N) 68 | for i := range buf { 69 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 70 | } 71 | 72 | p := New(comparer.DefaultComparer, 0) 73 | for i := range buf { 74 | p.Put(buf[i][:], nil) 75 | } 76 | 77 | b.ResetTimer() 78 | for i := range buf { 79 | p.Get(buf[i][:]) 80 | } 81 | } 82 | 83 | func BenchmarkGetRandom(b *testing.B) { 84 | buf := make([][4]byte, b.N) 85 | for i := range buf { 86 | binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) 87 | } 88 | 89 | p := New(comparer.DefaultComparer, 0) 90 | for i := range buf { 91 | p.Put(buf[i][:], nil) 92 | } 93 | 94 | b.ResetTimer() 95 | for i := 0; i < b.N; i++ { 96 | p.Get(buf[rand.Int()%b.N][:]) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /leveldb/memdb/memdb_suite_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/pingcap/goleveldb/leveldb/comparer" 10 | "github.com/pingcap/goleveldb/leveldb/testutil" 11 | ) 12 | 13 | func TestMemDB(t *testing.T) { 14 | testutil.RunSuite(t, "MemDB Suite") 15 | } 16 | 17 | func TestRace(t *testing.T) { 18 | var wg sync.WaitGroup 19 | db := New(comparer.DefaultComparer, 0) 20 | 21 | for i := 0; i < 5000; i++ { 22 | wg.Add(1) 23 | go func(db *DB, wg *sync.WaitGroup) { 24 | defer wg.Done() 25 | 26 | for i := 0; i < 2000; i++ { 27 | if db.rnd.src.Int63()%5 == 0 { 28 | db.rnd.src.Seed(db.rnd.src.Int63()) 29 | } 30 | } 31 | 32 | }(db, &wg) 33 | } 34 | wg.Wait() 35 | } 36 | 37 | func TestBitRand(t *testing.T) { 38 | src := rand.NewSource(int64(time.Now().Nanosecond())) 39 | rnd := &bitRand{ 40 | src: src, 41 | } 42 | var slot [4]int 43 | 44 | for i := 0; i < 100000; i++ { 45 | slot[rnd.bitN(2)]++ 46 | } 47 | 48 | sum := 0 49 | for i := 0; i < 4; i++ { 50 | x := slot[i] - 25000 51 | sum += x * x 52 | 53 | if sum >= 200000 { 54 | t.Fatalf("not so random %d! %d %d %d %d", sum, slot[0], slot[1], slot[2], slot[3]) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /leveldb/memdb/memdb_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package memdb 8 | 9 | import ( 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/comparer" 14 | "github.com/pingcap/goleveldb/leveldb/iterator" 15 | "github.com/pingcap/goleveldb/leveldb/testutil" 16 | "github.com/pingcap/goleveldb/leveldb/util" 17 | ) 18 | 19 | func (p *DB) TestFindLT(key []byte) (rkey, value []byte, err error) { 20 | p.mu.RLock() 21 | if node := p.findLT(key); node != 0 { 22 | n := p.nodeData[node] 23 | m := n + p.nodeData[node+nKey] 24 | rkey = p.kvData[n:m] 25 | value = p.kvData[m : m+p.nodeData[node+nVal]] 26 | } else { 27 | err = ErrNotFound 28 | } 29 | p.mu.RUnlock() 30 | return 31 | } 32 | 33 | func (p *DB) TestFindLast() (rkey, value []byte, err error) { 34 | p.mu.RLock() 35 | if node := p.findLast(); node != 0 { 36 | n := p.nodeData[node] 37 | m := n + p.nodeData[node+nKey] 38 | rkey = p.kvData[n:m] 39 | value = p.kvData[m : m+p.nodeData[node+nVal]] 40 | } else { 41 | err = ErrNotFound 42 | } 43 | p.mu.RUnlock() 44 | return 45 | } 46 | 47 | func (p *DB) TestPut(key []byte, value []byte) error { 48 | p.Put(key, value) 49 | return nil 50 | } 51 | 52 | func (p *DB) TestDelete(key []byte) error { 53 | p.Delete(key) 54 | return nil 55 | } 56 | 57 | func (p *DB) TestFind(key []byte) (rkey, rvalue []byte, err error) { 58 | return p.Find(key) 59 | } 60 | 61 | func (p *DB) TestGet(key []byte) (value []byte, err error) { 62 | return p.Get(key) 63 | } 64 | 65 | func (p *DB) TestNewIterator(slice *util.Range) iterator.Iterator { 66 | return p.NewIterator(slice) 67 | } 68 | 69 | var _ = testutil.Defer(func() { 70 | Describe("Memdb", func() { 71 | Describe("write test", func() { 72 | It("should do write correctly", func() { 73 | db := New(comparer.DefaultComparer, 0) 74 | t := testutil.DBTesting{ 75 | DB: db, 76 | Deleted: testutil.KeyValue_Generate(nil, 1000, 1, 1, 30, 5, 5).Clone(), 77 | PostFn: func(t *testutil.DBTesting) { 78 | Expect(db.Len()).Should(Equal(t.Present.Len())) 79 | Expect(db.Size()).Should(Equal(t.Present.Size())) 80 | switch t.Act { 81 | case testutil.DBPut, testutil.DBOverwrite: 82 | Expect(db.Contains(t.ActKey)).Should(BeTrue()) 83 | default: 84 | Expect(db.Contains(t.ActKey)).Should(BeFalse()) 85 | } 86 | }, 87 | } 88 | testutil.DoDBTesting(&t) 89 | }) 90 | }) 91 | 92 | Describe("read test", func() { 93 | testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB { 94 | // Building the DB. 95 | db := New(comparer.DefaultComparer, 0) 96 | kv.IterateShuffled(nil, func(i int, key, value []byte) { 97 | db.Put(key, value) 98 | }) 99 | 100 | if kv.Len() > 1 { 101 | It("Should find correct keys with findLT", func() { 102 | testutil.ShuffledIndex(nil, kv.Len()-1, 1, func(i int) { 103 | key_, key, _ := kv.IndexInexact(i + 1) 104 | expectedKey, expectedValue := kv.Index(i) 105 | 106 | // Using key that exist. 107 | rkey, rvalue, err := db.TestFindLT(key) 108 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q -> %q", key, expectedKey) 109 | Expect(rkey).Should(Equal(expectedKey), "Key") 110 | Expect(rvalue).Should(Equal(expectedValue), "Value for key %q -> %q", key, expectedKey) 111 | 112 | // Using key that doesn't exist. 113 | rkey, rvalue, err = db.TestFindLT(key_) 114 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q) -> %q", key_, key, expectedKey) 115 | Expect(rkey).Should(Equal(expectedKey)) 116 | Expect(rvalue).Should(Equal(expectedValue), "Value for key %q (%q) -> %q", key_, key, expectedKey) 117 | }) 118 | }) 119 | } 120 | 121 | if kv.Len() > 0 { 122 | It("Should find last key with findLast", func() { 123 | key, value := kv.Index(kv.Len() - 1) 124 | rkey, rvalue, err := db.TestFindLast() 125 | Expect(err).ShouldNot(HaveOccurred()) 126 | Expect(rkey).Should(Equal(key)) 127 | Expect(rvalue).Should(Equal(value)) 128 | }) 129 | } 130 | 131 | return db 132 | }, nil, nil) 133 | }) 134 | }) 135 | }) 136 | -------------------------------------------------------------------------------- /leveldb/options.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/pingcap/goleveldb/leveldb/filter" 11 | "github.com/pingcap/goleveldb/leveldb/opt" 12 | ) 13 | 14 | func dupOptions(o *opt.Options) *opt.Options { 15 | newo := &opt.Options{} 16 | if o != nil { 17 | *newo = *o 18 | } 19 | if newo.Strict == 0 { 20 | newo.Strict = opt.DefaultStrict 21 | } 22 | return newo 23 | } 24 | 25 | func (s *session) setOptions(o *opt.Options) { 26 | no := dupOptions(o) 27 | // Alternative filters. 28 | if filters := o.GetAltFilters(); len(filters) > 0 { 29 | no.AltFilters = make([]filter.Filter, len(filters)) 30 | for i, filter := range filters { 31 | no.AltFilters[i] = &iFilter{filter} 32 | } 33 | } 34 | // Comparer. 35 | s.icmp = &iComparer{o.GetComparer()} 36 | no.Comparer = s.icmp 37 | // Filter. 38 | if filter := o.GetFilter(); filter != nil { 39 | no.Filter = &iFilter{filter} 40 | } 41 | 42 | s.o = &cachedOptions{Options: no} 43 | s.o.cache() 44 | } 45 | 46 | const optCachedLevel = 7 47 | 48 | type cachedOptions struct { 49 | *opt.Options 50 | 51 | compactionExpandLimit []int 52 | compactionGPOverlaps []int 53 | compactionSourceLimit []int 54 | compactionTableSize []int 55 | compactionTotalSize []int64 56 | } 57 | 58 | func (co *cachedOptions) cache() { 59 | co.compactionExpandLimit = make([]int, optCachedLevel) 60 | co.compactionGPOverlaps = make([]int, optCachedLevel) 61 | co.compactionSourceLimit = make([]int, optCachedLevel) 62 | co.compactionTableSize = make([]int, optCachedLevel) 63 | co.compactionTotalSize = make([]int64, optCachedLevel) 64 | 65 | for level := 0; level < optCachedLevel; level++ { 66 | co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level) 67 | co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level) 68 | co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level) 69 | co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level) 70 | co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level) 71 | } 72 | } 73 | 74 | func (co *cachedOptions) GetCompactionExpandLimit(level int) int { 75 | if level < optCachedLevel { 76 | return co.compactionExpandLimit[level] 77 | } 78 | return co.Options.GetCompactionExpandLimit(level) 79 | } 80 | 81 | func (co *cachedOptions) GetCompactionGPOverlaps(level int) int { 82 | if level < optCachedLevel { 83 | return co.compactionGPOverlaps[level] 84 | } 85 | return co.Options.GetCompactionGPOverlaps(level) 86 | } 87 | 88 | func (co *cachedOptions) GetCompactionSourceLimit(level int) int { 89 | if level < optCachedLevel { 90 | return co.compactionSourceLimit[level] 91 | } 92 | return co.Options.GetCompactionSourceLimit(level) 93 | } 94 | 95 | func (co *cachedOptions) GetCompactionTableSize(level int) int { 96 | if level < optCachedLevel { 97 | return co.compactionTableSize[level] 98 | } 99 | return co.Options.GetCompactionTableSize(level) 100 | } 101 | 102 | func (co *cachedOptions) GetCompactionTotalSize(level int) int64 { 103 | if level < optCachedLevel { 104 | return co.compactionTotalSize[level] 105 | } 106 | return co.Options.GetCompactionTotalSize(level) 107 | } 108 | -------------------------------------------------------------------------------- /leveldb/session.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "io" 12 | "os" 13 | "sync" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/errors" 16 | "github.com/pingcap/goleveldb/leveldb/journal" 17 | "github.com/pingcap/goleveldb/leveldb/opt" 18 | "github.com/pingcap/goleveldb/leveldb/storage" 19 | ) 20 | 21 | // ErrManifestCorrupted records manifest corruption. This error will be 22 | // wrapped with errors.ErrCorrupted. 23 | type ErrManifestCorrupted struct { 24 | Field string 25 | Reason string 26 | } 27 | 28 | func (e *ErrManifestCorrupted) Error() string { 29 | return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason) 30 | } 31 | 32 | func newErrManifestCorrupted(fd storage.FileDesc, field, reason string) error { 33 | return errors.NewErrCorrupted(fd, &ErrManifestCorrupted{field, reason}) 34 | } 35 | 36 | // session represent a persistent database session. 37 | type session struct { 38 | // Need 64-bit alignment. 39 | stNextFileNum int64 // current unused file number 40 | stJournalNum int64 // current journal file number; need external synchronization 41 | stPrevJournalNum int64 // prev journal file number; no longer used; for compatibility with older version of leveldb 42 | stTempFileNum int64 43 | stSeqNum uint64 // last mem compacted seq; need external synchronization 44 | 45 | stor storage.Storage 46 | storLock storage.Locker 47 | o *cachedOptions 48 | icmp *iComparer 49 | tops *tOps 50 | fileRef map[int64]int 51 | 52 | manifest *journal.Writer 53 | manifestWriter storage.Writer 54 | manifestFd storage.FileDesc 55 | 56 | stCompPtrs []internalKey // compaction pointers; need external synchronization 57 | stVersion *version // current version 58 | vmu sync.Mutex 59 | } 60 | 61 | // Creates new initialized session instance. 62 | func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { 63 | if stor == nil { 64 | return nil, os.ErrInvalid 65 | } 66 | storLock, err := stor.Lock() 67 | if err != nil { 68 | return 69 | } 70 | s = &session{ 71 | stor: stor, 72 | storLock: storLock, 73 | fileRef: make(map[int64]int), 74 | } 75 | s.setOptions(o) 76 | s.tops = newTableOps(s) 77 | s.setVersion(newVersion(s)) 78 | s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") 79 | return 80 | } 81 | 82 | // Close session. 83 | func (s *session) close() { 84 | s.tops.close() 85 | if s.manifest != nil { 86 | s.manifest.Close() 87 | } 88 | if s.manifestWriter != nil { 89 | s.manifestWriter.Close() 90 | } 91 | s.manifest = nil 92 | s.manifestWriter = nil 93 | s.setVersion(&version{s: s, closing: true}) 94 | } 95 | 96 | // Release session lock. 97 | func (s *session) release() { 98 | s.storLock.Unlock() 99 | } 100 | 101 | // Create a new database session; need external synchronization. 102 | func (s *session) create() error { 103 | // create manifest 104 | return s.newManifest(nil, nil) 105 | } 106 | 107 | // Recover a database session; need external synchronization. 108 | func (s *session) recover() (err error) { 109 | defer func() { 110 | if os.IsNotExist(err) { 111 | // Don't return os.ErrNotExist if the underlying storage contains 112 | // other files that belong to LevelDB. So the DB won't get trashed. 113 | if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 { 114 | err = &errors.ErrCorrupted{Fd: storage.FileDesc{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}} 115 | } 116 | } 117 | }() 118 | 119 | fd, err := s.stor.GetMeta() 120 | if err != nil { 121 | return 122 | } 123 | 124 | reader, err := s.stor.Open(fd) 125 | if err != nil { 126 | return 127 | } 128 | defer reader.Close() 129 | 130 | var ( 131 | // Options. 132 | strict = s.o.GetStrict(opt.StrictManifest) 133 | 134 | jr = journal.NewReader(reader, dropper{s, fd}, strict, true) 135 | rec = &sessionRecord{} 136 | staging = s.stVersion.newStaging() 137 | ) 138 | for { 139 | var r io.Reader 140 | r, err = jr.Next() 141 | if err != nil { 142 | if err == io.EOF { 143 | err = nil 144 | break 145 | } 146 | return errors.SetFd(err, fd) 147 | } 148 | 149 | err = rec.decode(r) 150 | if err == nil { 151 | // save compact pointers 152 | for _, r := range rec.compPtrs { 153 | s.setCompPtr(r.level, internalKey(r.ikey)) 154 | } 155 | // commit record to version staging 156 | staging.commit(rec) 157 | } else { 158 | err = errors.SetFd(err, fd) 159 | if strict || !errors.IsCorrupted(err) { 160 | return 161 | } 162 | s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd)) 163 | } 164 | rec.resetCompPtrs() 165 | rec.resetAddedTables() 166 | rec.resetDeletedTables() 167 | } 168 | 169 | switch { 170 | case !rec.has(recComparer): 171 | return newErrManifestCorrupted(fd, "comparer", "missing") 172 | case rec.comparer != s.icmp.uName(): 173 | return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer)) 174 | case !rec.has(recNextFileNum): 175 | return newErrManifestCorrupted(fd, "next-file-num", "missing") 176 | case !rec.has(recJournalNum): 177 | return newErrManifestCorrupted(fd, "journal-file-num", "missing") 178 | case !rec.has(recSeqNum): 179 | return newErrManifestCorrupted(fd, "seq-num", "missing") 180 | } 181 | 182 | s.manifestFd = fd 183 | s.setVersion(staging.finish()) 184 | s.setNextFileNum(rec.nextFileNum) 185 | s.recordCommited(rec) 186 | return nil 187 | } 188 | 189 | // Commit session; need external synchronization. 190 | func (s *session) commit(r *sessionRecord) (err error) { 191 | v := s.version() 192 | defer v.release() 193 | 194 | // spawn new version based on current version 195 | nv := v.spawn(r) 196 | 197 | if s.manifest == nil { 198 | // manifest journal writer not yet created, create one 199 | err = s.newManifest(r, nv) 200 | } else { 201 | err = s.flushManifest(r) 202 | } 203 | 204 | // finally, apply new version if no error rise 205 | if err == nil { 206 | s.setVersion(nv) 207 | } 208 | 209 | return 210 | } 211 | -------------------------------------------------------------------------------- /leveldb/session_record.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bufio" 11 | "encoding/binary" 12 | "io" 13 | "strings" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/errors" 16 | "github.com/pingcap/goleveldb/leveldb/storage" 17 | ) 18 | 19 | type byteReader interface { 20 | io.Reader 21 | io.ByteReader 22 | } 23 | 24 | // These numbers are written to disk and should not be changed. 25 | const ( 26 | recComparer = 1 27 | recJournalNum = 2 28 | recNextFileNum = 3 29 | recSeqNum = 4 30 | recCompPtr = 5 31 | recDelTable = 6 32 | recAddTable = 7 33 | // 8 was used for large value refs 34 | recPrevJournalNum = 9 35 | ) 36 | 37 | type cpRecord struct { 38 | level int 39 | ikey internalKey 40 | } 41 | 42 | type atRecord struct { 43 | level int 44 | num int64 45 | size int64 46 | imin internalKey 47 | imax internalKey 48 | } 49 | 50 | type dtRecord struct { 51 | level int 52 | num int64 53 | } 54 | 55 | type sessionRecord struct { 56 | hasRec int 57 | comparer string 58 | journalNum int64 59 | prevJournalNum int64 60 | nextFileNum int64 61 | seqNum uint64 62 | compPtrs []cpRecord 63 | addedTables []atRecord 64 | deletedTables []dtRecord 65 | 66 | scratch [binary.MaxVarintLen64]byte 67 | err error 68 | } 69 | 70 | func (p *sessionRecord) has(rec int) bool { 71 | return p.hasRec&(1< 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | ) 13 | 14 | func decodeEncode(v *sessionRecord) (res bool, err error) { 15 | b := new(bytes.Buffer) 16 | err = v.encode(b) 17 | if err != nil { 18 | return 19 | } 20 | v2 := &sessionRecord{} 21 | err = v.decode(b) 22 | if err != nil { 23 | return 24 | } 25 | b2 := new(bytes.Buffer) 26 | err = v2.encode(b2) 27 | if err != nil { 28 | return 29 | } 30 | return bytes.Equal(b.Bytes(), b2.Bytes()), nil 31 | } 32 | 33 | func TestSessionRecord_EncodeDecode(t *testing.T) { 34 | big := int64(1) << 50 35 | v := &sessionRecord{} 36 | i := int64(0) 37 | test := func() { 38 | res, err := decodeEncode(v) 39 | if err != nil { 40 | t.Fatalf("error when testing encode/decode sessionRecord: %v", err) 41 | } 42 | if !res { 43 | t.Error("encode/decode test failed at iteration:", i) 44 | } 45 | } 46 | 47 | for ; i < 4; i++ { 48 | test() 49 | v.addTable(3, big+300+i, big+400+i, 50 | makeInternalKey(nil, []byte("foo"), uint64(big+500+1), keyTypeVal), 51 | makeInternalKey(nil, []byte("zoo"), uint64(big+600+1), keyTypeDel)) 52 | v.delTable(4, big+700+i) 53 | v.addCompPtr(int(i), makeInternalKey(nil, []byte("x"), uint64(big+900+1), keyTypeVal)) 54 | } 55 | 56 | v.setComparer("foo") 57 | v.setJournalNum(big + 100) 58 | v.setPrevJournalNum(big + 99) 59 | v.setNextFileNum(big + 200) 60 | v.setSeqNum(uint64(big + 1000)) 61 | test() 62 | } 63 | -------------------------------------------------------------------------------- /leveldb/session_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "sync/atomic" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/journal" 14 | "github.com/pingcap/goleveldb/leveldb/storage" 15 | ) 16 | 17 | // Logging. 18 | 19 | type dropper struct { 20 | s *session 21 | fd storage.FileDesc 22 | } 23 | 24 | func (d dropper) Drop(err error) { 25 | if e, ok := err.(*journal.ErrCorrupted); ok { 26 | d.s.logf("journal@drop %s-%d S·%s %q", d.fd.Type, d.fd.Num, shortenb(e.Size), e.Reason) 27 | } else { 28 | d.s.logf("journal@drop %s-%d %q", d.fd.Type, d.fd.Num, err) 29 | } 30 | } 31 | 32 | func (s *session) log(v ...interface{}) { s.stor.Log(fmt.Sprint(v...)) } 33 | func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) } 34 | 35 | // File utils. 36 | 37 | func (s *session) newTemp() storage.FileDesc { 38 | num := atomic.AddInt64(&s.stTempFileNum, 1) - 1 39 | return storage.FileDesc{storage.TypeTemp, num} 40 | } 41 | 42 | func (s *session) addFileRef(fd storage.FileDesc, ref int) int { 43 | ref += s.fileRef[fd.Num] 44 | if ref > 0 { 45 | s.fileRef[fd.Num] = ref 46 | } else if ref == 0 { 47 | delete(s.fileRef, fd.Num) 48 | } else { 49 | panic(fmt.Sprintf("negative ref: %v", fd)) 50 | } 51 | return ref 52 | } 53 | 54 | // Session state. 55 | 56 | // Get current version. This will incr version ref, must call 57 | // version.release (exactly once) after use. 58 | func (s *session) version() *version { 59 | s.vmu.Lock() 60 | defer s.vmu.Unlock() 61 | s.stVersion.incref() 62 | return s.stVersion 63 | } 64 | 65 | func (s *session) tLen(level int) int { 66 | s.vmu.Lock() 67 | defer s.vmu.Unlock() 68 | return s.stVersion.tLen(level) 69 | } 70 | 71 | // Set current version to v. 72 | func (s *session) setVersion(v *version) { 73 | s.vmu.Lock() 74 | defer s.vmu.Unlock() 75 | // Hold by session. It is important to call this first before releasing 76 | // current version, otherwise the still used files might get released. 77 | v.incref() 78 | if s.stVersion != nil { 79 | // Release current version. 80 | s.stVersion.releaseNB() 81 | } 82 | s.stVersion = v 83 | } 84 | 85 | // Get current unused file number. 86 | func (s *session) nextFileNum() int64 { 87 | return atomic.LoadInt64(&s.stNextFileNum) 88 | } 89 | 90 | // Set current unused file number to num. 91 | func (s *session) setNextFileNum(num int64) { 92 | atomic.StoreInt64(&s.stNextFileNum, num) 93 | } 94 | 95 | // Mark file number as used. 96 | func (s *session) markFileNum(num int64) { 97 | nextFileNum := num + 1 98 | for { 99 | old, x := s.stNextFileNum, nextFileNum 100 | if old > x { 101 | x = old 102 | } 103 | if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) { 104 | break 105 | } 106 | } 107 | } 108 | 109 | // Allocate a file number. 110 | func (s *session) allocFileNum() int64 { 111 | return atomic.AddInt64(&s.stNextFileNum, 1) - 1 112 | } 113 | 114 | // Reuse given file number. 115 | func (s *session) reuseFileNum(num int64) { 116 | for { 117 | old, x := s.stNextFileNum, num 118 | if old != x+1 { 119 | x = old 120 | } 121 | if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) { 122 | break 123 | } 124 | } 125 | } 126 | 127 | // Set compaction ptr at given level; need external synchronization. 128 | func (s *session) setCompPtr(level int, ik internalKey) { 129 | if level >= len(s.stCompPtrs) { 130 | newCompPtrs := make([]internalKey, level+1) 131 | copy(newCompPtrs, s.stCompPtrs) 132 | s.stCompPtrs = newCompPtrs 133 | } 134 | s.stCompPtrs[level] = append(internalKey{}, ik...) 135 | } 136 | 137 | // Get compaction ptr at given level; need external synchronization. 138 | func (s *session) getCompPtr(level int) internalKey { 139 | if level >= len(s.stCompPtrs) { 140 | return nil 141 | } 142 | return s.stCompPtrs[level] 143 | } 144 | 145 | // Manifest related utils. 146 | 147 | // Fill given session record obj with current states; need external 148 | // synchronization. 149 | func (s *session) fillRecord(r *sessionRecord, snapshot bool) { 150 | r.setNextFileNum(s.nextFileNum()) 151 | 152 | if snapshot { 153 | if !r.has(recJournalNum) { 154 | r.setJournalNum(s.stJournalNum) 155 | } 156 | 157 | if !r.has(recSeqNum) { 158 | r.setSeqNum(s.stSeqNum) 159 | } 160 | 161 | for level, ik := range s.stCompPtrs { 162 | if ik != nil { 163 | r.addCompPtr(level, ik) 164 | } 165 | } 166 | 167 | r.setComparer(s.icmp.uName()) 168 | } 169 | } 170 | 171 | // Mark if record has been committed, this will update session state; 172 | // need external synchronization. 173 | func (s *session) recordCommited(rec *sessionRecord) { 174 | if rec.has(recJournalNum) { 175 | s.stJournalNum = rec.journalNum 176 | } 177 | 178 | if rec.has(recPrevJournalNum) { 179 | s.stPrevJournalNum = rec.prevJournalNum 180 | } 181 | 182 | if rec.has(recSeqNum) { 183 | s.stSeqNum = rec.seqNum 184 | } 185 | 186 | for _, r := range rec.compPtrs { 187 | s.setCompPtr(r.level, internalKey(r.ikey)) 188 | } 189 | } 190 | 191 | // Create a new manifest file; need external synchronization. 192 | func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { 193 | fd := storage.FileDesc{storage.TypeManifest, s.allocFileNum()} 194 | writer, err := s.stor.Create(fd) 195 | if err != nil { 196 | return 197 | } 198 | jw := journal.NewWriter(writer) 199 | 200 | if v == nil { 201 | v = s.version() 202 | defer v.release() 203 | } 204 | if rec == nil { 205 | rec = &sessionRecord{} 206 | } 207 | s.fillRecord(rec, true) 208 | v.fillRecord(rec) 209 | 210 | defer func() { 211 | if err == nil { 212 | s.recordCommited(rec) 213 | if s.manifest != nil { 214 | s.manifest.Close() 215 | } 216 | if s.manifestWriter != nil { 217 | s.manifestWriter.Close() 218 | } 219 | if !s.manifestFd.Zero() { 220 | s.stor.Remove(s.manifestFd) 221 | } 222 | s.manifestFd = fd 223 | s.manifestWriter = writer 224 | s.manifest = jw 225 | } else { 226 | writer.Close() 227 | s.stor.Remove(fd) 228 | s.reuseFileNum(fd.Num) 229 | } 230 | }() 231 | 232 | w, err := jw.Next() 233 | if err != nil { 234 | return 235 | } 236 | err = rec.encode(w) 237 | if err != nil { 238 | return 239 | } 240 | err = jw.Flush() 241 | if err != nil { 242 | return 243 | } 244 | err = s.stor.SetMeta(fd) 245 | return 246 | } 247 | 248 | // Flush record to disk. 249 | func (s *session) flushManifest(rec *sessionRecord) (err error) { 250 | s.fillRecord(rec, false) 251 | w, err := s.manifest.Next() 252 | if err != nil { 253 | return 254 | } 255 | err = rec.encode(w) 256 | if err != nil { 257 | return 258 | } 259 | err = s.manifest.Flush() 260 | if err != nil { 261 | return 262 | } 263 | if !s.o.GetNoSync() { 264 | err = s.manifestWriter.Sync() 265 | if err != nil { 266 | return 267 | } 268 | } 269 | s.recordCommited(rec) 270 | return 271 | } 272 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_nacl.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build nacl 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 17 | return nil, syscall.ENOTSUP 18 | } 19 | 20 | func setFileLock(f *os.File, readOnly, lock bool) error { 21 | return syscall.ENOTSUP 22 | } 23 | 24 | func rename(oldpath, newpath string) error { 25 | return syscall.ENOTSUP 26 | } 27 | 28 | func isErrInvalid(err error) bool { 29 | return false 30 | } 31 | 32 | func syncDir(name string) error { 33 | return syscall.ENOTSUP 34 | } 35 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | type plan9FileLock struct { 15 | f *os.File 16 | } 17 | 18 | func (fl *plan9FileLock) release() error { 19 | return fl.f.Close() 20 | } 21 | 22 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 23 | var ( 24 | flag int 25 | perm os.FileMode 26 | ) 27 | if readOnly { 28 | flag = os.O_RDONLY 29 | } else { 30 | flag = os.O_RDWR 31 | perm = os.ModeExclusive 32 | } 33 | f, err := os.OpenFile(path, flag, perm) 34 | if os.IsNotExist(err) { 35 | f, err = os.OpenFile(path, flag|os.O_CREATE, perm|0644) 36 | } 37 | if err != nil { 38 | return 39 | } 40 | fl = &plan9FileLock{f: f} 41 | return 42 | } 43 | 44 | func rename(oldpath, newpath string) error { 45 | if _, err := os.Stat(newpath); err == nil { 46 | if err := os.Remove(newpath); err != nil { 47 | return err 48 | } 49 | } 50 | 51 | _, fname := filepath.Split(newpath) 52 | return os.Rename(oldpath, fname) 53 | } 54 | 55 | func syncDir(name string) error { 56 | f, err := os.Open(name) 57 | if err != nil { 58 | return err 59 | } 60 | defer f.Close() 61 | if err := f.Sync(); err != nil { 62 | return err 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_solaris.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build solaris 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | type unixFileLock struct { 17 | f *os.File 18 | } 19 | 20 | func (fl *unixFileLock) release() error { 21 | if err := setFileLock(fl.f, false, false); err != nil { 22 | return err 23 | } 24 | return fl.f.Close() 25 | } 26 | 27 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 28 | var flag int 29 | if readOnly { 30 | flag = os.O_RDONLY 31 | } else { 32 | flag = os.O_RDWR 33 | } 34 | f, err := os.OpenFile(path, flag, 0) 35 | if os.IsNotExist(err) { 36 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 37 | } 38 | if err != nil { 39 | return 40 | } 41 | err = setFileLock(f, readOnly, true) 42 | if err != nil { 43 | f.Close() 44 | return 45 | } 46 | fl = &unixFileLock{f: f} 47 | return 48 | } 49 | 50 | func setFileLock(f *os.File, readOnly, lock bool) error { 51 | flock := syscall.Flock_t{ 52 | Type: syscall.F_UNLCK, 53 | Start: 0, 54 | Len: 0, 55 | Whence: 1, 56 | } 57 | if lock { 58 | if readOnly { 59 | flock.Type = syscall.F_RDLCK 60 | } else { 61 | flock.Type = syscall.F_WRLCK 62 | } 63 | } 64 | return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock) 65 | } 66 | 67 | func rename(oldpath, newpath string) error { 68 | return os.Rename(oldpath, newpath) 69 | } 70 | 71 | func syncDir(name string) error { 72 | f, err := os.Open(name) 73 | if err != nil { 74 | return err 75 | } 76 | defer f.Close() 77 | if err := f.Sync(); err != nil { 78 | return err 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "fmt" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | ) 15 | 16 | var cases = []struct { 17 | oldName []string 18 | name string 19 | ftype FileType 20 | num int64 21 | }{ 22 | {nil, "000100.log", TypeJournal, 100}, 23 | {nil, "000000.log", TypeJournal, 0}, 24 | {[]string{"000000.sst"}, "000000.ldb", TypeTable, 0}, 25 | {nil, "MANIFEST-000002", TypeManifest, 2}, 26 | {nil, "MANIFEST-000007", TypeManifest, 7}, 27 | {nil, "9223372036854775807.log", TypeJournal, 9223372036854775807}, 28 | {nil, "000100.tmp", TypeTemp, 100}, 29 | } 30 | 31 | var invalidCases = []string{ 32 | "", 33 | "foo", 34 | "foo-dx-100.log", 35 | ".log", 36 | "", 37 | "manifest", 38 | "CURREN", 39 | "CURRENTX", 40 | "MANIFES", 41 | "MANIFEST", 42 | "MANIFEST-", 43 | "XMANIFEST-3", 44 | "MANIFEST-3x", 45 | "LOC", 46 | "LOCKx", 47 | "LO", 48 | "LOGx", 49 | "18446744073709551616.log", 50 | "184467440737095516150.log", 51 | "100", 52 | "100.", 53 | "100.lop", 54 | } 55 | 56 | func TestFileStorage_CreateFileName(t *testing.T) { 57 | for _, c := range cases { 58 | if name := fsGenName(FileDesc{c.ftype, c.num}); name != c.name { 59 | t.Errorf("invalid filename got '%s', want '%s'", name, c.name) 60 | } 61 | } 62 | } 63 | 64 | func TestFileStorage_ParseFileName(t *testing.T) { 65 | for _, c := range cases { 66 | for _, name := range append([]string{c.name}, c.oldName...) { 67 | fd, ok := fsParseName(name) 68 | if !ok { 69 | t.Errorf("cannot parse filename '%s'", name) 70 | continue 71 | } 72 | if fd.Type != c.ftype { 73 | t.Errorf("filename '%s' invalid type got '%d', want '%d'", name, fd.Type, c.ftype) 74 | } 75 | if fd.Num != c.num { 76 | t.Errorf("filename '%s' invalid number got '%d', want '%d'", name, fd.Num, c.num) 77 | } 78 | } 79 | } 80 | } 81 | 82 | func TestFileStorage_InvalidFileName(t *testing.T) { 83 | for _, name := range invalidCases { 84 | if fsParseNamePtr(name, nil) { 85 | t.Errorf("filename '%s' should be invalid", name) 86 | } 87 | } 88 | } 89 | 90 | func TestFileStorage_Locking(t *testing.T) { 91 | path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-testrwlock-%d", os.Getuid())) 92 | if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { 93 | t.Fatal("RemoveAll: got error: ", err) 94 | } 95 | defer os.RemoveAll(path) 96 | 97 | p1, err := OpenFile(path, false) 98 | if err != nil { 99 | t.Fatal("OpenFile(1): got error: ", err) 100 | } 101 | 102 | p2, err := OpenFile(path, false) 103 | if err != nil { 104 | t.Logf("OpenFile(2): got error: %s (expected)", err) 105 | } else { 106 | p2.Close() 107 | p1.Close() 108 | t.Fatal("OpenFile(2): expect error") 109 | } 110 | 111 | p1.Close() 112 | 113 | p3, err := OpenFile(path, false) 114 | if err != nil { 115 | t.Fatal("OpenFile(3): got error: ", err) 116 | } 117 | defer p3.Close() 118 | 119 | l, err := p3.Lock() 120 | if err != nil { 121 | t.Fatal("storage lock failed(1): ", err) 122 | } 123 | _, err = p3.Lock() 124 | if err == nil { 125 | t.Fatal("expect error for second storage lock attempt") 126 | } else { 127 | t.Logf("storage lock got error: %s (expected)", err) 128 | } 129 | l.Unlock() 130 | _, err = p3.Lock() 131 | if err != nil { 132 | t.Fatal("storage lock failed(2): ", err) 133 | } 134 | } 135 | 136 | func TestFileStorage_ReadOnlyLocking(t *testing.T) { 137 | path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-testrolock-%d", os.Getuid())) 138 | if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { 139 | t.Fatal("RemoveAll: got error: ", err) 140 | } 141 | defer os.RemoveAll(path) 142 | 143 | p1, err := OpenFile(path, false) 144 | if err != nil { 145 | t.Fatal("OpenFile(1): got error: ", err) 146 | } 147 | 148 | _, err = OpenFile(path, true) 149 | if err != nil { 150 | t.Logf("OpenFile(2): got error: %s (expected)", err) 151 | } else { 152 | t.Fatal("OpenFile(2): expect error") 153 | } 154 | 155 | p1.Close() 156 | 157 | p3, err := OpenFile(path, true) 158 | if err != nil { 159 | t.Fatal("OpenFile(3): got error: ", err) 160 | } 161 | 162 | p4, err := OpenFile(path, true) 163 | if err != nil { 164 | t.Fatal("OpenFile(4): got error: ", err) 165 | } 166 | 167 | _, err = OpenFile(path, false) 168 | if err != nil { 169 | t.Logf("OpenFile(5): got error: %s (expected)", err) 170 | } else { 171 | t.Fatal("OpenFile(2): expect error") 172 | } 173 | 174 | p3.Close() 175 | p4.Close() 176 | } 177 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build darwin dragonfly freebsd linux netbsd openbsd 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | type unixFileLock struct { 17 | f *os.File 18 | } 19 | 20 | func (fl *unixFileLock) release() error { 21 | if err := setFileLock(fl.f, false, false); err != nil { 22 | return err 23 | } 24 | return fl.f.Close() 25 | } 26 | 27 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 28 | var flag int 29 | if readOnly { 30 | flag = os.O_RDONLY 31 | } else { 32 | flag = os.O_RDWR 33 | } 34 | f, err := os.OpenFile(path, flag, 0) 35 | if os.IsNotExist(err) { 36 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 37 | } 38 | if err != nil { 39 | return 40 | } 41 | err = setFileLock(f, readOnly, true) 42 | if err != nil { 43 | f.Close() 44 | return 45 | } 46 | fl = &unixFileLock{f: f} 47 | return 48 | } 49 | 50 | func setFileLock(f *os.File, readOnly, lock bool) error { 51 | how := syscall.LOCK_UN 52 | if lock { 53 | if readOnly { 54 | how = syscall.LOCK_SH 55 | } else { 56 | how = syscall.LOCK_EX 57 | } 58 | } 59 | return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB) 60 | } 61 | 62 | func rename(oldpath, newpath string) error { 63 | return os.Rename(oldpath, newpath) 64 | } 65 | 66 | func isErrInvalid(err error) bool { 67 | if err == os.ErrInvalid { 68 | return true 69 | } 70 | if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL { 71 | return true 72 | } 73 | return false 74 | } 75 | 76 | func syncDir(name string) error { 77 | f, err := os.Open(name) 78 | if err != nil { 79 | return err 80 | } 81 | defer f.Close() 82 | if err := f.Sync(); err != nil && !isErrInvalid(err) { 83 | return err 84 | } 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_wasm.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | ) 7 | 8 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 9 | return nil, syscall.ENOTSUP 10 | } 11 | 12 | func setFileLock(f *os.File, readOnly, lock bool) error { 13 | return syscall.ENOTSUP 14 | } 15 | 16 | func rename(oldpath, newpath string) error { 17 | return syscall.ENOTSUP 18 | } 19 | 20 | func isErrInvalid(err error) bool { 21 | return false 22 | } 23 | 24 | func syncDir(name string) error { 25 | return syscall.ENOTSUP 26 | } 27 | -------------------------------------------------------------------------------- /leveldb/storage/file_storage_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "syscall" 11 | "unsafe" 12 | ) 13 | 14 | var ( 15 | modkernel32 = syscall.NewLazyDLL("kernel32.dll") 16 | 17 | procMoveFileExW = modkernel32.NewProc("MoveFileExW") 18 | ) 19 | 20 | const ( 21 | _MOVEFILE_REPLACE_EXISTING = 1 22 | ) 23 | 24 | type windowsFileLock struct { 25 | fd syscall.Handle 26 | } 27 | 28 | func (fl *windowsFileLock) release() error { 29 | return syscall.Close(fl.fd) 30 | } 31 | 32 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 33 | pathp, err := syscall.UTF16PtrFromString(path) 34 | if err != nil { 35 | return 36 | } 37 | var access, shareMode uint32 38 | if readOnly { 39 | access = syscall.GENERIC_READ 40 | shareMode = syscall.FILE_SHARE_READ 41 | } else { 42 | access = syscall.GENERIC_READ | syscall.GENERIC_WRITE 43 | } 44 | fd, err := syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) 45 | if err == syscall.ERROR_FILE_NOT_FOUND { 46 | fd, err = syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) 47 | } 48 | if err != nil { 49 | return 50 | } 51 | fl = &windowsFileLock{fd: fd} 52 | return 53 | } 54 | 55 | func moveFileEx(from *uint16, to *uint16, flags uint32) error { 56 | r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) 57 | if r1 == 0 { 58 | if e1 != 0 { 59 | return error(e1) 60 | } 61 | return syscall.EINVAL 62 | } 63 | return nil 64 | } 65 | 66 | func rename(oldpath, newpath string) error { 67 | from, err := syscall.UTF16PtrFromString(oldpath) 68 | if err != nil { 69 | return err 70 | } 71 | to, err := syscall.UTF16PtrFromString(newpath) 72 | if err != nil { 73 | return err 74 | } 75 | return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING) 76 | } 77 | 78 | func syncDir(name string) error { return nil } 79 | -------------------------------------------------------------------------------- /leveldb/storage/mem_storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "bytes" 11 | "os" 12 | "sync" 13 | ) 14 | 15 | const typeShift = 3 16 | 17 | type memStorageLock struct { 18 | ms *memStorage 19 | } 20 | 21 | func (lock *memStorageLock) Unlock() { 22 | ms := lock.ms 23 | ms.mu.Lock() 24 | defer ms.mu.Unlock() 25 | if ms.slock == lock { 26 | ms.slock = nil 27 | } 28 | return 29 | } 30 | 31 | // memStorage is a memory-backed storage. 32 | type memStorage struct { 33 | mu sync.Mutex 34 | slock *memStorageLock 35 | files map[uint64]*memFile 36 | meta FileDesc 37 | } 38 | 39 | // NewMemStorage returns a new memory-backed storage implementation. 40 | func NewMemStorage() Storage { 41 | return &memStorage{ 42 | files: make(map[uint64]*memFile), 43 | } 44 | } 45 | 46 | func (ms *memStorage) Lock() (Locker, error) { 47 | ms.mu.Lock() 48 | defer ms.mu.Unlock() 49 | if ms.slock != nil { 50 | return nil, ErrLocked 51 | } 52 | ms.slock = &memStorageLock{ms: ms} 53 | return ms.slock, nil 54 | } 55 | 56 | func (*memStorage) Log(str string) {} 57 | 58 | func (ms *memStorage) SetMeta(fd FileDesc) error { 59 | if !FileDescOk(fd) { 60 | return ErrInvalidFile 61 | } 62 | 63 | ms.mu.Lock() 64 | ms.meta = fd 65 | ms.mu.Unlock() 66 | return nil 67 | } 68 | 69 | func (ms *memStorage) GetMeta() (FileDesc, error) { 70 | ms.mu.Lock() 71 | defer ms.mu.Unlock() 72 | if ms.meta.Zero() { 73 | return FileDesc{}, os.ErrNotExist 74 | } 75 | return ms.meta, nil 76 | } 77 | 78 | func (ms *memStorage) List(ft FileType) ([]FileDesc, error) { 79 | ms.mu.Lock() 80 | var fds []FileDesc 81 | for x := range ms.files { 82 | fd := unpackFile(x) 83 | if fd.Type&ft != 0 { 84 | fds = append(fds, fd) 85 | } 86 | } 87 | ms.mu.Unlock() 88 | return fds, nil 89 | } 90 | 91 | func (ms *memStorage) Open(fd FileDesc) (Reader, error) { 92 | if !FileDescOk(fd) { 93 | return nil, ErrInvalidFile 94 | } 95 | 96 | ms.mu.Lock() 97 | defer ms.mu.Unlock() 98 | if m, exist := ms.files[packFile(fd)]; exist { 99 | if m.open { 100 | return nil, errFileOpen 101 | } 102 | m.open = true 103 | return &memReader{Reader: bytes.NewReader(m.Bytes()), ms: ms, m: m}, nil 104 | } 105 | return nil, os.ErrNotExist 106 | } 107 | 108 | func (ms *memStorage) Create(fd FileDesc) (Writer, error) { 109 | if !FileDescOk(fd) { 110 | return nil, ErrInvalidFile 111 | } 112 | 113 | x := packFile(fd) 114 | ms.mu.Lock() 115 | defer ms.mu.Unlock() 116 | m, exist := ms.files[x] 117 | if exist { 118 | if m.open { 119 | return nil, errFileOpen 120 | } 121 | m.Reset() 122 | } else { 123 | m = &memFile{} 124 | ms.files[x] = m 125 | } 126 | m.open = true 127 | return &memWriter{memFile: m, ms: ms}, nil 128 | } 129 | 130 | func (ms *memStorage) Remove(fd FileDesc) error { 131 | if !FileDescOk(fd) { 132 | return ErrInvalidFile 133 | } 134 | 135 | x := packFile(fd) 136 | ms.mu.Lock() 137 | defer ms.mu.Unlock() 138 | if _, exist := ms.files[x]; exist { 139 | delete(ms.files, x) 140 | return nil 141 | } 142 | return os.ErrNotExist 143 | } 144 | 145 | func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { 146 | if FileDescOk(oldfd) || FileDescOk(newfd) { 147 | return ErrInvalidFile 148 | } 149 | if oldfd == newfd { 150 | return nil 151 | } 152 | 153 | oldx := packFile(oldfd) 154 | newx := packFile(newfd) 155 | ms.mu.Lock() 156 | defer ms.mu.Unlock() 157 | oldm, exist := ms.files[oldx] 158 | if !exist { 159 | return os.ErrNotExist 160 | } 161 | newm, exist := ms.files[newx] 162 | if (exist && newm.open) || oldm.open { 163 | return errFileOpen 164 | } 165 | delete(ms.files, oldx) 166 | ms.files[newx] = oldm 167 | return nil 168 | } 169 | 170 | func (*memStorage) Close() error { return nil } 171 | 172 | type memFile struct { 173 | bytes.Buffer 174 | open bool 175 | } 176 | 177 | type memReader struct { 178 | *bytes.Reader 179 | ms *memStorage 180 | m *memFile 181 | closed bool 182 | } 183 | 184 | func (mr *memReader) Close() error { 185 | mr.ms.mu.Lock() 186 | defer mr.ms.mu.Unlock() 187 | if mr.closed { 188 | return ErrClosed 189 | } 190 | mr.m.open = false 191 | return nil 192 | } 193 | 194 | type memWriter struct { 195 | *memFile 196 | ms *memStorage 197 | closed bool 198 | } 199 | 200 | func (*memWriter) Sync() error { return nil } 201 | 202 | func (mw *memWriter) Close() error { 203 | mw.ms.mu.Lock() 204 | defer mw.ms.mu.Unlock() 205 | if mw.closed { 206 | return ErrClosed 207 | } 208 | mw.memFile.open = false 209 | return nil 210 | } 211 | 212 | func packFile(fd FileDesc) uint64 { 213 | return uint64(fd.Num)<> typeShift)} 218 | } 219 | -------------------------------------------------------------------------------- /leveldb/storage/mem_storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "bytes" 11 | "testing" 12 | ) 13 | 14 | func TestMemStorage(t *testing.T) { 15 | m := NewMemStorage() 16 | 17 | l, err := m.Lock() 18 | if err != nil { 19 | t.Fatal("storage lock failed(1): ", err) 20 | } 21 | _, err = m.Lock() 22 | if err == nil { 23 | t.Fatal("expect error for second storage lock attempt") 24 | } else { 25 | t.Logf("storage lock got error: %s (expected)", err) 26 | } 27 | l.Unlock() 28 | _, err = m.Lock() 29 | if err != nil { 30 | t.Fatal("storage lock failed(2): ", err) 31 | } 32 | 33 | w, err := m.Create(FileDesc{TypeTable, 1}) 34 | if err != nil { 35 | t.Fatal("Storage.Create: ", err) 36 | } 37 | w.Write([]byte("abc")) 38 | w.Close() 39 | if fds, _ := m.List(TypeAll); len(fds) != 1 { 40 | t.Fatal("invalid GetFiles len") 41 | } 42 | buf := new(bytes.Buffer) 43 | r, err := m.Open(FileDesc{TypeTable, 1}) 44 | if err != nil { 45 | t.Fatal("Open: got error: ", err) 46 | } 47 | buf.ReadFrom(r) 48 | r.Close() 49 | if got := buf.String(); got != "abc" { 50 | t.Fatalf("Read: invalid value, want=abc got=%s", got) 51 | } 52 | if _, err := m.Open(FileDesc{TypeTable, 1}); err != nil { 53 | t.Fatal("Open: got error: ", err) 54 | } 55 | if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { 56 | t.Fatal("expecting error") 57 | } 58 | m.Remove(FileDesc{TypeTable, 1}) 59 | if fds, _ := m.List(TypeAll); len(fds) != 0 { 60 | t.Fatal("invalid GetFiles len", len(fds)) 61 | } 62 | if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { 63 | t.Fatal("expecting error") 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /leveldb/storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package storage provides storage abstraction for LevelDB. 8 | package storage 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | "io" 14 | ) 15 | 16 | // FileType represent a file type. 17 | type FileType int 18 | 19 | // File types. 20 | const ( 21 | TypeManifest FileType = 1 << iota 22 | TypeJournal 23 | TypeTable 24 | TypeTemp 25 | 26 | TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp 27 | ) 28 | 29 | func (t FileType) String() string { 30 | switch t { 31 | case TypeManifest: 32 | return "manifest" 33 | case TypeJournal: 34 | return "journal" 35 | case TypeTable: 36 | return "table" 37 | case TypeTemp: 38 | return "temp" 39 | } 40 | return fmt.Sprintf("", t) 41 | } 42 | 43 | // Common error. 44 | var ( 45 | ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument") 46 | ErrLocked = errors.New("leveldb/storage: already locked") 47 | ErrClosed = errors.New("leveldb/storage: closed") 48 | ) 49 | 50 | // ErrCorrupted is the type that wraps errors that indicate corruption of 51 | // a file. Package storage has its own type instead of using 52 | // errors.ErrCorrupted to prevent circular import. 53 | type ErrCorrupted struct { 54 | Fd FileDesc 55 | Err error 56 | } 57 | 58 | func (e *ErrCorrupted) Error() string { 59 | if !e.Fd.Zero() { 60 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 61 | } 62 | return e.Err.Error() 63 | } 64 | 65 | // Syncer is the interface that wraps basic Sync method. 66 | type Syncer interface { 67 | // Sync commits the current contents of the file to stable storage. 68 | Sync() error 69 | } 70 | 71 | // Reader is the interface that groups the basic Read, Seek, ReadAt and Close 72 | // methods. 73 | type Reader interface { 74 | io.ReadSeeker 75 | io.ReaderAt 76 | io.Closer 77 | } 78 | 79 | // Writer is the interface that groups the basic Write, Sync and Close 80 | // methods. 81 | type Writer interface { 82 | io.WriteCloser 83 | Syncer 84 | } 85 | 86 | // Locker is the interface that wraps Unlock method. 87 | type Locker interface { 88 | Unlock() 89 | } 90 | 91 | // FileDesc is a 'file descriptor'. 92 | type FileDesc struct { 93 | Type FileType 94 | Num int64 95 | } 96 | 97 | func (fd FileDesc) String() string { 98 | switch fd.Type { 99 | case TypeManifest: 100 | return fmt.Sprintf("MANIFEST-%06d", fd.Num) 101 | case TypeJournal: 102 | return fmt.Sprintf("%06d.log", fd.Num) 103 | case TypeTable: 104 | return fmt.Sprintf("%06d.ldb", fd.Num) 105 | case TypeTemp: 106 | return fmt.Sprintf("%06d.tmp", fd.Num) 107 | default: 108 | return fmt.Sprintf("%#x-%d", fd.Type, fd.Num) 109 | } 110 | } 111 | 112 | // Zero returns true if fd == (FileDesc{}). 113 | func (fd FileDesc) Zero() bool { 114 | return fd == (FileDesc{}) 115 | } 116 | 117 | // FileDescOk returns true if fd is a valid 'file descriptor'. 118 | func FileDescOk(fd FileDesc) bool { 119 | switch fd.Type { 120 | case TypeManifest: 121 | case TypeJournal: 122 | case TypeTable: 123 | case TypeTemp: 124 | default: 125 | return false 126 | } 127 | return fd.Num >= 0 128 | } 129 | 130 | // Storage is the storage. A storage instance must be safe for concurrent use. 131 | type Storage interface { 132 | // Lock locks the storage. Any subsequent attempt to call Lock will fail 133 | // until the last lock released. 134 | // Caller should call Unlock method after use. 135 | Lock() (Locker, error) 136 | 137 | // Log logs a string. This is used for logging. 138 | // An implementation may write to a file, stdout or simply do nothing. 139 | Log(str string) 140 | 141 | // SetMeta store 'file descriptor' that can later be acquired using GetMeta 142 | // method. The 'file descriptor' should point to a valid file. 143 | // SetMeta should be implemented in such way that changes should happen 144 | // atomically. 145 | SetMeta(fd FileDesc) error 146 | 147 | // GetMeta returns 'file descriptor' stored in meta. The 'file descriptor' 148 | // can be updated using SetMeta method. 149 | // Returns os.ErrNotExist if meta doesn't store any 'file descriptor', or 150 | // 'file descriptor' point to nonexistent file. 151 | GetMeta() (FileDesc, error) 152 | 153 | // List returns file descriptors that match the given file types. 154 | // The file types may be OR'ed together. 155 | List(ft FileType) ([]FileDesc, error) 156 | 157 | // Open opens file with the given 'file descriptor' read-only. 158 | // Returns os.ErrNotExist error if the file does not exist. 159 | // Returns ErrClosed if the underlying storage is closed. 160 | Open(fd FileDesc) (Reader, error) 161 | 162 | // Create creates file with the given 'file descriptor', truncate if already 163 | // exist and opens write-only. 164 | // Returns ErrClosed if the underlying storage is closed. 165 | Create(fd FileDesc) (Writer, error) 166 | 167 | // Remove removes file with the given 'file descriptor'. 168 | // Returns ErrClosed if the underlying storage is closed. 169 | Remove(fd FileDesc) error 170 | 171 | // Rename renames file from oldfd to newfd. 172 | // Returns ErrClosed if the underlying storage is closed. 173 | Rename(oldfd, newfd FileDesc) error 174 | 175 | // Close closes the storage. 176 | // It is valid to call Close multiple times. Other methods should not be 177 | // called after the storage has been closed. 178 | Close() error 179 | } 180 | -------------------------------------------------------------------------------- /leveldb/table/block_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package table 8 | 9 | import ( 10 | "encoding/binary" 11 | "fmt" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | 16 | "github.com/pingcap/goleveldb/leveldb/comparer" 17 | "github.com/pingcap/goleveldb/leveldb/iterator" 18 | "github.com/pingcap/goleveldb/leveldb/testutil" 19 | "github.com/pingcap/goleveldb/leveldb/util" 20 | ) 21 | 22 | type blockTesting struct { 23 | tr *Reader 24 | b *block 25 | } 26 | 27 | func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator { 28 | return t.tr.newBlockIter(t.b, nil, slice, false) 29 | } 30 | 31 | var _ = testutil.Defer(func() { 32 | Describe("Block", func() { 33 | Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting { 34 | // Building the block. 35 | bw := &blockWriter{ 36 | restartInterval: restartInterval, 37 | scratch: make([]byte, 30), 38 | } 39 | kv.Iterate(func(i int, key, value []byte) { 40 | bw.append(key, value) 41 | }) 42 | bw.finish() 43 | 44 | // Opening the block. 45 | data := bw.buf.Bytes() 46 | restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) 47 | return &blockTesting{ 48 | tr: &Reader{cmp: comparer.DefaultComparer}, 49 | b: &block{ 50 | data: data, 51 | restartsLen: restartsLen, 52 | restartsOffset: len(data) - (restartsLen+1)*4, 53 | }, 54 | } 55 | } 56 | 57 | Describe("read test", func() { 58 | for restartInterval := 1; restartInterval <= 5; restartInterval++ { 59 | Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { 60 | kv := &testutil.KeyValue{} 61 | Text := func() string { 62 | return fmt.Sprintf("and %d keys", kv.Len()) 63 | } 64 | 65 | Test := func() { 66 | // Make block. 67 | br := Build(kv, restartInterval) 68 | // Do testing. 69 | testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil) 70 | } 71 | 72 | Describe(Text(), Test) 73 | 74 | kv.PutString("", "empty") 75 | Describe(Text(), Test) 76 | 77 | kv.PutString("a1", "foo") 78 | Describe(Text(), Test) 79 | 80 | kv.PutString("a2", "v") 81 | Describe(Text(), Test) 82 | 83 | kv.PutString("a3qqwrkks", "hello") 84 | Describe(Text(), Test) 85 | 86 | kv.PutString("a4", "bar") 87 | Describe(Text(), Test) 88 | 89 | kv.PutString("a5111111", "v5") 90 | kv.PutString("a6", "") 91 | kv.PutString("a7", "v7") 92 | kv.PutString("a8", "vvvvvvvvvvvvvvvvvvvvvv8") 93 | kv.PutString("b", "v9") 94 | kv.PutString("c9", "v9") 95 | kv.PutString("c91", "v9") 96 | kv.PutString("d0", "v9") 97 | Describe(Text(), Test) 98 | }) 99 | } 100 | }) 101 | 102 | Describe("out-of-bound slice test", func() { 103 | kv := &testutil.KeyValue{} 104 | kv.PutString("k1", "v1") 105 | kv.PutString("k2", "v2") 106 | kv.PutString("k3abcdefgg", "v3") 107 | kv.PutString("k4", "v4") 108 | kv.PutString("k5", "v5") 109 | for restartInterval := 1; restartInterval <= 5; restartInterval++ { 110 | Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { 111 | // Make block. 112 | bt := Build(kv, restartInterval) 113 | 114 | Test := func(r *util.Range) func(done Done) { 115 | return func(done Done) { 116 | iter := bt.TestNewIterator(r) 117 | Expect(iter.Error()).ShouldNot(HaveOccurred()) 118 | 119 | t := testutil.IteratorTesting{ 120 | KeyValue: kv.Clone(), 121 | Iter: iter, 122 | } 123 | 124 | testutil.DoIteratorTesting(&t) 125 | iter.Release() 126 | done <- true 127 | } 128 | } 129 | 130 | It("Should do iterations and seeks correctly #0", 131 | Test(&util.Range{Start: []byte("k0"), Limit: []byte("k6")}), 2.0) 132 | 133 | It("Should do iterations and seeks correctly #1", 134 | Test(&util.Range{Start: []byte(""), Limit: []byte("zzzzzzz")}), 2.0) 135 | }) 136 | } 137 | }) 138 | }) 139 | }) 140 | -------------------------------------------------------------------------------- /leveldb/table/table.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package table allows read and write sorted key/value. 8 | package table 9 | 10 | import ( 11 | "encoding/binary" 12 | ) 13 | 14 | /* 15 | Table: 16 | 17 | Table is consist of one or more data blocks, an optional filter block 18 | a metaindex block, an index block and a table footer. Metaindex block 19 | is a special block used to keep parameters of the table, such as filter 20 | block name and its block handle. Index block is a special block used to 21 | keep record of data blocks offset and length, index block use one as 22 | restart interval. The key used by index block are the last key of preceding 23 | block, shorter separator of adjacent blocks or shorter successor of the 24 | last key of the last block. Filter block is an optional block contains 25 | sequence of filter data generated by a filter generator. 26 | 27 | Table data structure: 28 | + optional 29 | / 30 | +--------------+--------------+--------------+------+-------+-----------------+-------------+--------+ 31 | | data block 1 | ... | data block n | filter block | metaindex block | index block | footer | 32 | +--------------+--------------+--------------+--------------+-----------------+-------------+--------+ 33 | 34 | Each block followed by a 5-bytes trailer contains compression type and checksum. 35 | 36 | Table block trailer: 37 | 38 | +---------------------------+-------------------+ 39 | | compression type (1-byte) | checksum (4-byte) | 40 | +---------------------------+-------------------+ 41 | 42 | The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression 43 | type also included in the checksum. 44 | 45 | Table footer: 46 | 47 | +------------------- 40-bytes -------------------+ 48 | / \ 49 | +------------------------+--------------------+------+-----------------+ 50 | | metaindex block handle / index block handle / ---- | magic (8-bytes) | 51 | +------------------------+--------------------+------+-----------------+ 52 | 53 | The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/". 54 | 55 | NOTE: All fixed-length integer are little-endian. 56 | */ 57 | 58 | /* 59 | Block: 60 | 61 | Block is consist of one or more key/value entries and a block trailer. 62 | Block entry shares key prefix with its preceding key until a restart 63 | point reached. A block should contains at least one restart point. 64 | First restart point are always zero. 65 | 66 | Block data structure: 67 | 68 | + restart point + restart point (depends on restart interval) 69 | / / 70 | +---------------+---------------+---------------+---------------+---------+ 71 | | block entry 1 | block entry 2 | ... | block entry n | trailer | 72 | +---------------+---------------+---------------+---------------+---------+ 73 | 74 | Key/value entry: 75 | 76 | +---- key len ----+ 77 | / \ 78 | +-------+---------+-----------+---------+--------------------+--------------+----------------+ 79 | | shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) | 80 | +-----------------+---------------------+--------------------+--------------+----------------+ 81 | 82 | Block entry shares key prefix with its preceding key: 83 | Conditions: 84 | restart_interval=2 85 | entry one : key=deck,value=v1 86 | entry two : key=dock,value=v2 87 | entry three: key=duck,value=v3 88 | The entries will be encoded as follow: 89 | 90 | + restart point (offset=0) + restart point (offset=16) 91 | / / 92 | +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ 93 | | 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" | 94 | +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ 95 | \ / \ / \ / 96 | +----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+ 97 | 98 | The block trailer will contains two restart points: 99 | 100 | +------------+-----------+--------+ 101 | | 0 | 16 | 2 | 102 | +------------+-----------+---+----+ 103 | \ / \ 104 | +-- restart points --+ + restart points length 105 | 106 | Block trailer: 107 | 108 | +-- 4-bytes --+ 109 | / \ 110 | +-----------------+-----------------+-----------------+------------------------------+ 111 | | restart point 1 | .... | restart point n | restart points len (4-bytes) | 112 | +-----------------+-----------------+-----------------+------------------------------+ 113 | 114 | 115 | NOTE: All fixed-length integer are little-endian. 116 | */ 117 | 118 | /* 119 | Filter block: 120 | 121 | Filter block consist of one or more filter data and a filter block trailer. 122 | The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg. 123 | 124 | Filter block data structure: 125 | 126 | + offset 1 + offset 2 + offset n + trailer offset 127 | / / / / 128 | +---------------+---------------+---------------+---------+ 129 | | filter data 1 | ... | filter data n | trailer | 130 | +---------------+---------------+---------------+---------+ 131 | 132 | Filter block trailer: 133 | 134 | +- 4-bytes -+ 135 | / \ 136 | +---------------+---------------+---------------+-------------------------------+------------------+ 137 | | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) | 138 | +-------------- +---------------+---------------+-------------------------------+------------------+ 139 | 140 | 141 | NOTE: All fixed-length integer are little-endian. 142 | */ 143 | 144 | const ( 145 | blockTrailerLen = 5 146 | footerLen = 48 147 | 148 | magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb" 149 | 150 | // The block type gives the per-block compression format. 151 | // These constants are part of the file format and should not be changed. 152 | blockTypeNoCompression = 0 153 | blockTypeSnappyCompression = 1 154 | 155 | // Generate new filter every 2KB of data 156 | filterBaseLg = 11 157 | filterBase = 1 << filterBaseLg 158 | ) 159 | 160 | type blockHandle struct { 161 | offset, length uint64 162 | } 163 | 164 | func decodeBlockHandle(src []byte) (blockHandle, int) { 165 | offset, n := binary.Uvarint(src) 166 | length, m := binary.Uvarint(src[n:]) 167 | if n == 0 || m == 0 { 168 | return blockHandle{}, 0 169 | } 170 | return blockHandle{offset, length}, n + m 171 | } 172 | 173 | func encodeBlockHandle(dst []byte, b blockHandle) int { 174 | n := binary.PutUvarint(dst, b.offset) 175 | m := binary.PutUvarint(dst[n:], b.length) 176 | return n + m 177 | } 178 | -------------------------------------------------------------------------------- /leveldb/table/table_suite_test.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pingcap/goleveldb/leveldb/testutil" 7 | ) 8 | 9 | func TestTable(t *testing.T) { 10 | testutil.RunSuite(t, "Table Suite") 11 | } 12 | -------------------------------------------------------------------------------- /leveldb/table/table_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package table 8 | 9 | import ( 10 | "bytes" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/iterator" 16 | "github.com/pingcap/goleveldb/leveldb/opt" 17 | "github.com/pingcap/goleveldb/leveldb/storage" 18 | "github.com/pingcap/goleveldb/leveldb/testutil" 19 | "github.com/pingcap/goleveldb/leveldb/util" 20 | ) 21 | 22 | type tableWrapper struct { 23 | *Reader 24 | } 25 | 26 | func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) { 27 | return t.Reader.Find(key, false, nil) 28 | } 29 | 30 | func (t tableWrapper) TestGet(key []byte) (value []byte, err error) { 31 | return t.Reader.Get(key, nil) 32 | } 33 | 34 | func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator { 35 | return t.Reader.NewIterator(slice, nil) 36 | } 37 | 38 | var _ = testutil.Defer(func() { 39 | Describe("Table", func() { 40 | Describe("approximate offset test", func() { 41 | var ( 42 | buf = &bytes.Buffer{} 43 | o = &opt.Options{ 44 | BlockSize: 1024, 45 | Compression: opt.NoCompression, 46 | } 47 | ) 48 | 49 | // Building the table. 50 | tw := NewWriter(buf, o) 51 | tw.Append([]byte("k01"), []byte("hello")) 52 | tw.Append([]byte("k02"), []byte("hello2")) 53 | tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000)) 54 | tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000)) 55 | tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000)) 56 | tw.Append([]byte("k06"), []byte("hello3")) 57 | tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000)) 58 | err := tw.Close() 59 | 60 | It("Should be able to approximate offset of a key correctly", func() { 61 | Expect(err).ShouldNot(HaveOccurred()) 62 | 63 | tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) 64 | Expect(err).ShouldNot(HaveOccurred()) 65 | CheckOffset := func(key string, expect, threshold int) { 66 | offset, err := tr.OffsetOf([]byte(key)) 67 | Expect(err).ShouldNot(HaveOccurred()) 68 | Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key) 69 | } 70 | 71 | CheckOffset("k0", 0, 0) 72 | CheckOffset("k01a", 0, 0) 73 | CheckOffset("k02", 0, 0) 74 | CheckOffset("k03", 0, 0) 75 | CheckOffset("k04", 10000, 1000) 76 | CheckOffset("k04a", 210000, 1000) 77 | CheckOffset("k05", 210000, 1000) 78 | CheckOffset("k06", 510000, 1000) 79 | CheckOffset("k07", 510000, 1000) 80 | CheckOffset("xyz", 610000, 2000) 81 | }) 82 | }) 83 | 84 | Describe("read test", func() { 85 | Build := func(kv testutil.KeyValue) testutil.DB { 86 | o := &opt.Options{ 87 | BlockSize: 512, 88 | BlockRestartInterval: 3, 89 | } 90 | buf := &bytes.Buffer{} 91 | 92 | // Building the table. 93 | tw := NewWriter(buf, o) 94 | kv.Iterate(func(i int, key, value []byte) { 95 | tw.Append(key, value) 96 | }) 97 | tw.Close() 98 | 99 | // Opening the table. 100 | tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) 101 | return tableWrapper{tr} 102 | } 103 | Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() { 104 | return func() { 105 | db := Build(*kv) 106 | if body != nil { 107 | body(db.(tableWrapper).Reader) 108 | } 109 | testutil.KeyValueTesting(nil, *kv, db, nil, nil) 110 | } 111 | } 112 | 113 | testutil.AllKeyValueTesting(nil, Build, nil, nil) 114 | Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 1, 10, 512, 512), func(r *Reader) { 115 | It("should have correct blocks number", func() { 116 | indexBlock, err := r.readBlock(r.indexBH, true) 117 | Expect(err).To(BeNil()) 118 | Expect(indexBlock.restartsLen).Should(Equal(9)) 119 | }) 120 | })) 121 | }) 122 | }) 123 | }) 124 | -------------------------------------------------------------------------------- /leveldb/testutil/db.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/errors" 16 | "github.com/pingcap/goleveldb/leveldb/iterator" 17 | "github.com/pingcap/goleveldb/leveldb/util" 18 | ) 19 | 20 | type DB interface{} 21 | 22 | type Put interface { 23 | TestPut(key []byte, value []byte) error 24 | } 25 | 26 | type Delete interface { 27 | TestDelete(key []byte) error 28 | } 29 | 30 | type Find interface { 31 | TestFind(key []byte) (rkey, rvalue []byte, err error) 32 | } 33 | 34 | type Get interface { 35 | TestGet(key []byte) (value []byte, err error) 36 | } 37 | 38 | type Has interface { 39 | TestHas(key []byte) (ret bool, err error) 40 | } 41 | 42 | type NewIterator interface { 43 | TestNewIterator(slice *util.Range) iterator.Iterator 44 | } 45 | 46 | type DBAct int 47 | 48 | func (a DBAct) String() string { 49 | switch a { 50 | case DBNone: 51 | return "none" 52 | case DBPut: 53 | return "put" 54 | case DBOverwrite: 55 | return "overwrite" 56 | case DBDelete: 57 | return "delete" 58 | case DBDeleteNA: 59 | return "delete_na" 60 | } 61 | return "unknown" 62 | } 63 | 64 | const ( 65 | DBNone DBAct = iota 66 | DBPut 67 | DBOverwrite 68 | DBDelete 69 | DBDeleteNA 70 | ) 71 | 72 | type DBTesting struct { 73 | Rand *rand.Rand 74 | DB interface { 75 | Get 76 | Put 77 | Delete 78 | } 79 | PostFn func(t *DBTesting) 80 | Deleted, Present KeyValue 81 | Act, LastAct DBAct 82 | ActKey, LastActKey []byte 83 | } 84 | 85 | func (t *DBTesting) post() { 86 | if t.PostFn != nil { 87 | t.PostFn(t) 88 | } 89 | } 90 | 91 | func (t *DBTesting) setAct(act DBAct, key []byte) { 92 | t.LastAct, t.Act = t.Act, act 93 | t.LastActKey, t.ActKey = t.ActKey, key 94 | } 95 | 96 | func (t *DBTesting) text() string { 97 | return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey) 98 | } 99 | 100 | func (t *DBTesting) Text() string { 101 | return "DBTesting " + t.text() 102 | } 103 | 104 | func (t *DBTesting) TestPresentKV(key, value []byte) { 105 | rvalue, err := t.DB.TestGet(key) 106 | Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text()) 107 | Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text()) 108 | } 109 | 110 | func (t *DBTesting) TestAllPresent() { 111 | t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) { 112 | t.TestPresentKV(key, value) 113 | }) 114 | } 115 | 116 | func (t *DBTesting) TestDeletedKey(key []byte) { 117 | _, err := t.DB.TestGet(key) 118 | Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text()) 119 | } 120 | 121 | func (t *DBTesting) TestAllDeleted() { 122 | t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) { 123 | t.TestDeletedKey(key) 124 | }) 125 | } 126 | 127 | func (t *DBTesting) TestAll() { 128 | dn := t.Deleted.Len() 129 | pn := t.Present.Len() 130 | ShuffledIndex(t.Rand, dn+pn, 1, func(i int) { 131 | if i >= dn { 132 | key, value := t.Present.Index(i - dn) 133 | t.TestPresentKV(key, value) 134 | } else { 135 | t.TestDeletedKey(t.Deleted.KeyAt(i)) 136 | } 137 | }) 138 | } 139 | 140 | func (t *DBTesting) Put(key, value []byte) { 141 | if new := t.Present.PutU(key, value); new { 142 | t.setAct(DBPut, key) 143 | } else { 144 | t.setAct(DBOverwrite, key) 145 | } 146 | t.Deleted.Delete(key) 147 | err := t.DB.TestPut(key, value) 148 | Expect(err).ShouldNot(HaveOccurred(), t.Text()) 149 | t.TestPresentKV(key, value) 150 | t.post() 151 | } 152 | 153 | func (t *DBTesting) PutRandom() bool { 154 | if t.Deleted.Len() > 0 { 155 | i := t.Rand.Intn(t.Deleted.Len()) 156 | key, value := t.Deleted.Index(i) 157 | t.Put(key, value) 158 | return true 159 | } 160 | return false 161 | } 162 | 163 | func (t *DBTesting) Delete(key []byte) { 164 | if exist, value := t.Present.Delete(key); exist { 165 | t.setAct(DBDelete, key) 166 | t.Deleted.PutU(key, value) 167 | } else { 168 | t.setAct(DBDeleteNA, key) 169 | } 170 | err := t.DB.TestDelete(key) 171 | Expect(err).ShouldNot(HaveOccurred(), t.Text()) 172 | t.TestDeletedKey(key) 173 | t.post() 174 | } 175 | 176 | func (t *DBTesting) DeleteRandom() bool { 177 | if t.Present.Len() > 0 { 178 | i := t.Rand.Intn(t.Present.Len()) 179 | t.Delete(t.Present.KeyAt(i)) 180 | return true 181 | } 182 | return false 183 | } 184 | 185 | func (t *DBTesting) RandomAct(round int) { 186 | for i := 0; i < round; i++ { 187 | if t.Rand.Int()%2 == 0 { 188 | t.PutRandom() 189 | } else { 190 | t.DeleteRandom() 191 | } 192 | } 193 | } 194 | 195 | func DoDBTesting(t *DBTesting) { 196 | if t.Rand == nil { 197 | t.Rand = NewRand() 198 | } 199 | 200 | t.DeleteRandom() 201 | t.PutRandom() 202 | t.DeleteRandom() 203 | t.DeleteRandom() 204 | for i := t.Deleted.Len() / 2; i >= 0; i-- { 205 | t.PutRandom() 206 | } 207 | t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10) 208 | 209 | // Additional iterator testing 210 | if db, ok := t.DB.(NewIterator); ok { 211 | iter := db.TestNewIterator(nil) 212 | Expect(iter.Error()).NotTo(HaveOccurred()) 213 | 214 | it := IteratorTesting{ 215 | KeyValue: t.Present, 216 | Iter: iter, 217 | } 218 | 219 | DoIteratorTesting(&it) 220 | iter.Release() 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /leveldb/testutil/ginkgo.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | ) 7 | 8 | func RunSuite(t GinkgoTestingT, name string) { 9 | RunDefer() 10 | 11 | SynchronizedBeforeSuite(func() []byte { 12 | RunDefer("setup") 13 | return nil 14 | }, func(data []byte) {}) 15 | SynchronizedAfterSuite(func() { 16 | RunDefer("teardown") 17 | }, func() {}) 18 | 19 | RegisterFailHandler(Fail) 20 | RunSpecs(t, name) 21 | } 22 | -------------------------------------------------------------------------------- /leveldb/testutil/iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/gomega" 14 | 15 | "github.com/pingcap/goleveldb/leveldb/iterator" 16 | ) 17 | 18 | type IterAct int 19 | 20 | func (a IterAct) String() string { 21 | switch a { 22 | case IterNone: 23 | return "none" 24 | case IterFirst: 25 | return "first" 26 | case IterLast: 27 | return "last" 28 | case IterPrev: 29 | return "prev" 30 | case IterNext: 31 | return "next" 32 | case IterSeek: 33 | return "seek" 34 | case IterSOI: 35 | return "soi" 36 | case IterEOI: 37 | return "eoi" 38 | } 39 | return "unknown" 40 | } 41 | 42 | const ( 43 | IterNone IterAct = iota 44 | IterFirst 45 | IterLast 46 | IterPrev 47 | IterNext 48 | IterSeek 49 | IterSOI 50 | IterEOI 51 | ) 52 | 53 | type IteratorTesting struct { 54 | KeyValue 55 | Iter iterator.Iterator 56 | Rand *rand.Rand 57 | PostFn func(t *IteratorTesting) 58 | Pos int 59 | Act, LastAct IterAct 60 | 61 | once bool 62 | } 63 | 64 | func (t *IteratorTesting) init() { 65 | if !t.once { 66 | t.Pos = -1 67 | t.once = true 68 | } 69 | } 70 | 71 | func (t *IteratorTesting) post() { 72 | if t.PostFn != nil { 73 | t.PostFn(t) 74 | } 75 | } 76 | 77 | func (t *IteratorTesting) setAct(act IterAct) { 78 | t.LastAct, t.Act = t.Act, act 79 | } 80 | 81 | func (t *IteratorTesting) text() string { 82 | return fmt.Sprintf("at pos %d and last action was <%v> -> <%v>", t.Pos, t.LastAct, t.Act) 83 | } 84 | 85 | func (t *IteratorTesting) Text() string { 86 | return "IteratorTesting is " + t.text() 87 | } 88 | 89 | func (t *IteratorTesting) IsFirst() bool { 90 | t.init() 91 | return t.Len() > 0 && t.Pos == 0 92 | } 93 | 94 | func (t *IteratorTesting) IsLast() bool { 95 | t.init() 96 | return t.Len() > 0 && t.Pos == t.Len()-1 97 | } 98 | 99 | func (t *IteratorTesting) TestKV() { 100 | t.init() 101 | key, value := t.Index(t.Pos) 102 | Expect(t.Iter.Key()).NotTo(BeNil()) 103 | Expect(t.Iter.Key()).Should(Equal(key), "Key is invalid, %s", t.text()) 104 | Expect(t.Iter.Value()).Should(Equal(value), "Value for key %q, %s", key, t.text()) 105 | } 106 | 107 | func (t *IteratorTesting) First() { 108 | t.init() 109 | t.setAct(IterFirst) 110 | 111 | ok := t.Iter.First() 112 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 113 | if t.Len() > 0 { 114 | t.Pos = 0 115 | Expect(ok).Should(BeTrue(), t.Text()) 116 | t.TestKV() 117 | } else { 118 | t.Pos = -1 119 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 120 | } 121 | t.post() 122 | } 123 | 124 | func (t *IteratorTesting) Last() { 125 | t.init() 126 | t.setAct(IterLast) 127 | 128 | ok := t.Iter.Last() 129 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 130 | if t.Len() > 0 { 131 | t.Pos = t.Len() - 1 132 | Expect(ok).Should(BeTrue(), t.Text()) 133 | t.TestKV() 134 | } else { 135 | t.Pos = 0 136 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 137 | } 138 | t.post() 139 | } 140 | 141 | func (t *IteratorTesting) Next() { 142 | t.init() 143 | t.setAct(IterNext) 144 | 145 | ok := t.Iter.Next() 146 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 147 | if t.Pos < t.Len()-1 { 148 | t.Pos++ 149 | Expect(ok).Should(BeTrue(), t.Text()) 150 | t.TestKV() 151 | } else { 152 | t.Pos = t.Len() 153 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 154 | } 155 | t.post() 156 | } 157 | 158 | func (t *IteratorTesting) Prev() { 159 | t.init() 160 | t.setAct(IterPrev) 161 | 162 | ok := t.Iter.Prev() 163 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 164 | if t.Pos > 0 { 165 | t.Pos-- 166 | Expect(ok).Should(BeTrue(), t.Text()) 167 | t.TestKV() 168 | } else { 169 | t.Pos = -1 170 | Expect(ok).ShouldNot(BeTrue(), t.Text()) 171 | } 172 | t.post() 173 | } 174 | 175 | func (t *IteratorTesting) Seek(i int) { 176 | t.init() 177 | t.setAct(IterSeek) 178 | 179 | key, _ := t.Index(i) 180 | oldKey, _ := t.IndexOrNil(t.Pos) 181 | 182 | ok := t.Iter.Seek(key) 183 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 184 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q, to pos %d, %s", oldKey, key, i, t.text())) 185 | 186 | t.Pos = i 187 | t.TestKV() 188 | t.post() 189 | } 190 | 191 | func (t *IteratorTesting) SeekInexact(i int) { 192 | t.init() 193 | t.setAct(IterSeek) 194 | var key0 []byte 195 | key1, _ := t.Index(i) 196 | if i > 0 { 197 | key0, _ = t.Index(i - 1) 198 | } 199 | key := BytesSeparator(key0, key1) 200 | oldKey, _ := t.IndexOrNil(t.Pos) 201 | 202 | ok := t.Iter.Seek(key) 203 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 204 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key1, i, t.text())) 205 | 206 | t.Pos = i 207 | t.TestKV() 208 | t.post() 209 | } 210 | 211 | func (t *IteratorTesting) SeekKey(key []byte) { 212 | t.init() 213 | t.setAct(IterSeek) 214 | oldKey, _ := t.IndexOrNil(t.Pos) 215 | i := t.Search(key) 216 | 217 | ok := t.Iter.Seek(key) 218 | Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) 219 | if i < t.Len() { 220 | key_, _ := t.Index(i) 221 | Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key_, i, t.text())) 222 | t.Pos = i 223 | t.TestKV() 224 | } else { 225 | Expect(ok).ShouldNot(BeTrue(), fmt.Sprintf("Seek from key %q to %q, %s", oldKey, key, t.text())) 226 | } 227 | 228 | t.Pos = i 229 | t.post() 230 | } 231 | 232 | func (t *IteratorTesting) SOI() { 233 | t.init() 234 | t.setAct(IterSOI) 235 | Expect(t.Pos).Should(BeNumerically("<=", 0), t.Text()) 236 | for i := 0; i < 3; i++ { 237 | t.Prev() 238 | } 239 | t.post() 240 | } 241 | 242 | func (t *IteratorTesting) EOI() { 243 | t.init() 244 | t.setAct(IterEOI) 245 | Expect(t.Pos).Should(BeNumerically(">=", t.Len()-1), t.Text()) 246 | for i := 0; i < 3; i++ { 247 | t.Next() 248 | } 249 | t.post() 250 | } 251 | 252 | func (t *IteratorTesting) WalkPrev(fn func(t *IteratorTesting)) { 253 | t.init() 254 | for old := t.Pos; t.Pos > 0; old = t.Pos { 255 | fn(t) 256 | Expect(t.Pos).Should(BeNumerically("<", old), t.Text()) 257 | } 258 | } 259 | 260 | func (t *IteratorTesting) WalkNext(fn func(t *IteratorTesting)) { 261 | t.init() 262 | for old := t.Pos; t.Pos < t.Len()-1; old = t.Pos { 263 | fn(t) 264 | Expect(t.Pos).Should(BeNumerically(">", old), t.Text()) 265 | } 266 | } 267 | 268 | func (t *IteratorTesting) PrevAll() { 269 | t.WalkPrev(func(t *IteratorTesting) { 270 | t.Prev() 271 | }) 272 | } 273 | 274 | func (t *IteratorTesting) NextAll() { 275 | t.WalkNext(func(t *IteratorTesting) { 276 | t.Next() 277 | }) 278 | } 279 | 280 | func DoIteratorTesting(t *IteratorTesting) { 281 | if t.Rand == nil { 282 | t.Rand = NewRand() 283 | } 284 | t.SOI() 285 | t.NextAll() 286 | t.First() 287 | t.SOI() 288 | t.NextAll() 289 | t.EOI() 290 | t.PrevAll() 291 | t.Last() 292 | t.EOI() 293 | t.PrevAll() 294 | t.SOI() 295 | 296 | t.NextAll() 297 | t.PrevAll() 298 | t.NextAll() 299 | t.Last() 300 | t.PrevAll() 301 | t.First() 302 | t.NextAll() 303 | t.EOI() 304 | 305 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 306 | t.Seek(i) 307 | }) 308 | 309 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 310 | t.SeekInexact(i) 311 | }) 312 | 313 | ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { 314 | t.Seek(i) 315 | if i%2 != 0 { 316 | t.PrevAll() 317 | t.SOI() 318 | } else { 319 | t.NextAll() 320 | t.EOI() 321 | } 322 | }) 323 | 324 | for _, key := range []string{"", "foo", "bar", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"} { 325 | t.SeekKey([]byte(key)) 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /leveldb/testutil/kvtest.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "fmt" 11 | "math/rand" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | 16 | "github.com/pingcap/goleveldb/leveldb/errors" 17 | "github.com/pingcap/goleveldb/leveldb/util" 18 | ) 19 | 20 | func TestFind(db Find, kv KeyValue) { 21 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 22 | key_, key, value := kv.IndexInexact(i) 23 | 24 | // Using exact key. 25 | rkey, rvalue, err := db.TestFind(key) 26 | Expect(err).ShouldNot(HaveOccurred(), "Error for exact key %q", key) 27 | Expect(rkey).Should(Equal(key), "Key") 28 | Expect(rvalue).Should(Equal(value), "Value for exact key %q", key) 29 | 30 | // Using inexact key. 31 | rkey, rvalue, err = db.TestFind(key_) 32 | Expect(err).ShouldNot(HaveOccurred(), "Error for inexact key %q (%q)", key_, key) 33 | Expect(rkey).Should(Equal(key), "Key for inexact key %q (%q)", key_, key) 34 | Expect(rvalue).Should(Equal(value), "Value for inexact key %q (%q)", key_, key) 35 | }) 36 | } 37 | 38 | func TestFindAfterLast(db Find, kv KeyValue) { 39 | var key []byte 40 | if kv.Len() > 0 { 41 | key_, _ := kv.Index(kv.Len() - 1) 42 | key = BytesAfter(key_) 43 | } 44 | rkey, _, err := db.TestFind(key) 45 | Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey) 46 | Expect(err).Should(Equal(errors.ErrNotFound)) 47 | } 48 | 49 | func TestGet(db Get, kv KeyValue) { 50 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 51 | key_, key, value := kv.IndexInexact(i) 52 | 53 | // Using exact key. 54 | rvalue, err := db.TestGet(key) 55 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) 56 | Expect(rvalue).Should(Equal(value), "Value for key %q", key) 57 | 58 | // Using inexact key. 59 | if len(key_) > 0 { 60 | _, err = db.TestGet(key_) 61 | Expect(err).Should(HaveOccurred(), "Error for key %q", key_) 62 | Expect(err).Should(Equal(errors.ErrNotFound)) 63 | } 64 | }) 65 | } 66 | 67 | func TestHas(db Has, kv KeyValue) { 68 | ShuffledIndex(nil, kv.Len(), 1, func(i int) { 69 | key_, key, _ := kv.IndexInexact(i) 70 | 71 | // Using exact key. 72 | ret, err := db.TestHas(key) 73 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) 74 | Expect(ret).Should(BeTrue(), "False for key %q", key) 75 | 76 | // Using inexact key. 77 | if len(key_) > 0 { 78 | ret, err = db.TestHas(key_) 79 | Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_) 80 | Expect(ret).ShouldNot(BeTrue(), "True for key %q", key) 81 | } 82 | }) 83 | } 84 | 85 | func TestIter(db NewIterator, r *util.Range, kv KeyValue) { 86 | iter := db.TestNewIterator(r) 87 | Expect(iter.Error()).ShouldNot(HaveOccurred()) 88 | 89 | t := IteratorTesting{ 90 | KeyValue: kv, 91 | Iter: iter, 92 | } 93 | 94 | DoIteratorTesting(&t) 95 | iter.Release() 96 | } 97 | 98 | func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) { 99 | if rnd == nil { 100 | rnd = NewRand() 101 | } 102 | 103 | if p == nil { 104 | BeforeEach(func() { 105 | p = setup(kv) 106 | }) 107 | if teardown != nil { 108 | AfterEach(func() { 109 | teardown(p) 110 | }) 111 | } 112 | } 113 | 114 | It("Should find all keys with Find", func() { 115 | if db, ok := p.(Find); ok { 116 | TestFind(db, kv) 117 | } 118 | }) 119 | 120 | It("Should return error if Find on key after the last", func() { 121 | if db, ok := p.(Find); ok { 122 | TestFindAfterLast(db, kv) 123 | } 124 | }) 125 | 126 | It("Should only find exact key with Get", func() { 127 | if db, ok := p.(Get); ok { 128 | TestGet(db, kv) 129 | } 130 | }) 131 | 132 | It("Should only find present key with Has", func() { 133 | if db, ok := p.(Has); ok { 134 | TestHas(db, kv) 135 | } 136 | }) 137 | 138 | It("Should iterates and seeks correctly", func(done Done) { 139 | if db, ok := p.(NewIterator); ok { 140 | TestIter(db, nil, kv.Clone()) 141 | } 142 | done <- true 143 | }, 30.0) 144 | 145 | It("Should iterates and seeks slice correctly", func(done Done) { 146 | if db, ok := p.(NewIterator); ok { 147 | RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) { 148 | type slice struct { 149 | r *util.Range 150 | start, limit int 151 | } 152 | 153 | key_, _, _ := kv.IndexInexact(i) 154 | for _, x := range []slice{ 155 | {&util.Range{Start: key_, Limit: nil}, i, kv.Len()}, 156 | {&util.Range{Start: nil, Limit: key_}, 0, i}, 157 | } { 158 | By(fmt.Sprintf("Random index of %d .. %d", x.start, x.limit), func() { 159 | TestIter(db, x.r, kv.Slice(x.start, x.limit)) 160 | }) 161 | } 162 | }) 163 | } 164 | done <- true 165 | }, 200.0) 166 | 167 | It("Should iterates and seeks slice correctly", func(done Done) { 168 | if db, ok := p.(NewIterator); ok { 169 | RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) { 170 | By(fmt.Sprintf("Random range of %d .. %d", start, limit), func() { 171 | r := kv.Range(start, limit) 172 | TestIter(db, &r, kv.Slice(start, limit)) 173 | }) 174 | }) 175 | } 176 | done <- true 177 | }, 200.0) 178 | } 179 | 180 | func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) { 181 | Test := func(kv *KeyValue) func() { 182 | return func() { 183 | var p DB 184 | if setup != nil { 185 | Defer("setup", func() { 186 | p = setup(*kv) 187 | }) 188 | } 189 | if teardown != nil { 190 | Defer("teardown", func() { 191 | teardown(p) 192 | }) 193 | } 194 | if body != nil { 195 | p = body(*kv) 196 | } 197 | KeyValueTesting(rnd, *kv, p, func(KeyValue) DB { 198 | return p 199 | }, nil) 200 | } 201 | } 202 | 203 | Describe("with no key/value (empty)", Test(&KeyValue{})) 204 | Describe("with empty key", Test(KeyValue_EmptyKey())) 205 | Describe("with empty value", Test(KeyValue_EmptyValue())) 206 | Describe("with one key/value", Test(KeyValue_OneKeyValue())) 207 | Describe("with big value", Test(KeyValue_BigValue())) 208 | Describe("with special key", Test(KeyValue_SpecialKey())) 209 | Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue())) 210 | Describe("with generated key/value 2-incr", Test(KeyValue_Generate(nil, 120, 2, 1, 50, 10, 120))) 211 | Describe("with generated key/value 3-incr", Test(KeyValue_Generate(nil, 120, 3, 1, 50, 10, 120))) 212 | } 213 | -------------------------------------------------------------------------------- /leveldb/testutil/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package testutil 8 | 9 | import ( 10 | "bytes" 11 | "flag" 12 | "math/rand" 13 | "reflect" 14 | "sync" 15 | 16 | "github.com/onsi/ginkgo/config" 17 | 18 | "github.com/pingcap/goleveldb/leveldb/comparer" 19 | ) 20 | 21 | var ( 22 | runfn = make(map[string][]func()) 23 | runmu sync.Mutex 24 | ) 25 | 26 | func Defer(args ...interface{}) bool { 27 | var ( 28 | group string 29 | fn func() 30 | ) 31 | for _, arg := range args { 32 | v := reflect.ValueOf(arg) 33 | switch v.Kind() { 34 | case reflect.String: 35 | group = v.String() 36 | case reflect.Func: 37 | r := reflect.ValueOf(&fn).Elem() 38 | r.Set(v) 39 | } 40 | } 41 | if fn != nil { 42 | runmu.Lock() 43 | runfn[group] = append(runfn[group], fn) 44 | runmu.Unlock() 45 | } 46 | return true 47 | } 48 | 49 | func RunDefer(groups ...string) bool { 50 | if len(groups) == 0 { 51 | groups = append(groups, "") 52 | } 53 | runmu.Lock() 54 | var runfn_ []func() 55 | for _, group := range groups { 56 | runfn_ = append(runfn_, runfn[group]...) 57 | delete(runfn, group) 58 | } 59 | runmu.Unlock() 60 | for _, fn := range runfn_ { 61 | fn() 62 | } 63 | return runfn_ != nil 64 | } 65 | 66 | func RandomSeed() int64 { 67 | if !flag.Parsed() { 68 | panic("random seed not initialized") 69 | } 70 | return config.GinkgoConfig.RandomSeed 71 | } 72 | 73 | func NewRand() *rand.Rand { 74 | return rand.New(rand.NewSource(RandomSeed())) 75 | } 76 | 77 | var cmp = comparer.DefaultComparer 78 | 79 | func BytesSeparator(a, b []byte) []byte { 80 | if bytes.Equal(a, b) { 81 | return b 82 | } 83 | i, n := 0, len(a) 84 | if n > len(b) { 85 | n = len(b) 86 | } 87 | for ; i < n && (a[i] == b[i]); i++ { 88 | } 89 | x := append([]byte{}, a[:i]...) 90 | if i < n { 91 | if c := a[i] + 1; c < b[i] { 92 | return append(x, c) 93 | } 94 | x = append(x, a[i]) 95 | i++ 96 | } 97 | for ; i < len(a); i++ { 98 | if c := a[i]; c < 0xff { 99 | return append(x, c+1) 100 | } else { 101 | x = append(x, c) 102 | } 103 | } 104 | if len(b) > i && b[i] > 0 { 105 | return append(x, b[i]-1) 106 | } 107 | return append(x, 'x') 108 | } 109 | 110 | func BytesAfter(b []byte) []byte { 111 | var x []byte 112 | for _, c := range b { 113 | if c < 0xff { 114 | return append(x, c+1) 115 | } else { 116 | x = append(x, c) 117 | } 118 | } 119 | return append(x, 'x') 120 | } 121 | 122 | func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) { 123 | if rnd == nil { 124 | rnd = NewRand() 125 | } 126 | for x := 0; x < round; x++ { 127 | fn(rnd.Intn(n)) 128 | } 129 | return 130 | } 131 | 132 | func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) { 133 | if rnd == nil { 134 | rnd = NewRand() 135 | } 136 | for x := 0; x < round; x++ { 137 | for _, i := range rnd.Perm(n) { 138 | fn(i) 139 | } 140 | } 141 | return 142 | } 143 | 144 | func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) { 145 | if rnd == nil { 146 | rnd = NewRand() 147 | } 148 | for x := 0; x < round; x++ { 149 | start := rnd.Intn(n) 150 | length := 0 151 | if j := n - start; j > 0 { 152 | length = rnd.Intn(j) 153 | } 154 | fn(start, start+length) 155 | } 156 | return 157 | } 158 | 159 | func Max(x, y int) int { 160 | if x > y { 161 | return x 162 | } 163 | return y 164 | } 165 | 166 | func Min(x, y int) int { 167 | if x < y { 168 | return x 169 | } 170 | return y 171 | } 172 | -------------------------------------------------------------------------------- /leveldb/testutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | . "github.com/onsi/gomega" 11 | 12 | "github.com/pingcap/goleveldb/leveldb/iterator" 13 | "github.com/pingcap/goleveldb/leveldb/opt" 14 | "github.com/pingcap/goleveldb/leveldb/testutil" 15 | "github.com/pingcap/goleveldb/leveldb/util" 16 | ) 17 | 18 | type testingDB struct { 19 | *DB 20 | ro *opt.ReadOptions 21 | wo *opt.WriteOptions 22 | stor *testutil.Storage 23 | } 24 | 25 | func (t *testingDB) TestPut(key []byte, value []byte) error { 26 | return t.Put(key, value, t.wo) 27 | } 28 | 29 | func (t *testingDB) TestDelete(key []byte) error { 30 | return t.Delete(key, t.wo) 31 | } 32 | 33 | func (t *testingDB) TestGet(key []byte) (value []byte, err error) { 34 | return t.Get(key, t.ro) 35 | } 36 | 37 | func (t *testingDB) TestHas(key []byte) (ret bool, err error) { 38 | return t.Has(key, t.ro) 39 | } 40 | 41 | func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator { 42 | return t.NewIterator(slice, t.ro) 43 | } 44 | 45 | func (t *testingDB) TestClose() { 46 | err := t.Close() 47 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 48 | err = t.stor.Close() 49 | ExpectWithOffset(1, err).NotTo(HaveOccurred()) 50 | } 51 | 52 | func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB { 53 | stor := testutil.NewStorage() 54 | db, err := Open(stor, o) 55 | // FIXME: This may be called from outside It, which may cause panic. 56 | Expect(err).NotTo(HaveOccurred()) 57 | return &testingDB{ 58 | DB: db, 59 | ro: ro, 60 | wo: wo, 61 | stor: stor, 62 | } 63 | } 64 | 65 | type testingTransaction struct { 66 | *Transaction 67 | ro *opt.ReadOptions 68 | wo *opt.WriteOptions 69 | } 70 | 71 | func (t *testingTransaction) TestPut(key []byte, value []byte) error { 72 | return t.Put(key, value, t.wo) 73 | } 74 | 75 | func (t *testingTransaction) TestDelete(key []byte) error { 76 | return t.Delete(key, t.wo) 77 | } 78 | 79 | func (t *testingTransaction) TestGet(key []byte) (value []byte, err error) { 80 | return t.Get(key, t.ro) 81 | } 82 | 83 | func (t *testingTransaction) TestHas(key []byte) (ret bool, err error) { 84 | return t.Has(key, t.ro) 85 | } 86 | 87 | func (t *testingTransaction) TestNewIterator(slice *util.Range) iterator.Iterator { 88 | return t.NewIterator(slice, t.ro) 89 | } 90 | 91 | func (t *testingTransaction) TestClose() {} 92 | -------------------------------------------------------------------------------- /leveldb/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "sort" 12 | 13 | "github.com/pingcap/goleveldb/leveldb/storage" 14 | ) 15 | 16 | func shorten(str string) string { 17 | if len(str) <= 8 { 18 | return str 19 | } 20 | return str[:3] + ".." + str[len(str)-3:] 21 | } 22 | 23 | var bunits = [...]string{"", "Ki", "Mi", "Gi"} 24 | 25 | func shortenb(bytes int) string { 26 | i := 0 27 | for ; bytes > 1024 && i < 4; i++ { 28 | bytes /= 1024 29 | } 30 | return fmt.Sprintf("%d%sB", bytes, bunits[i]) 31 | } 32 | 33 | func sshortenb(bytes int) string { 34 | if bytes == 0 { 35 | return "~" 36 | } 37 | sign := "+" 38 | if bytes < 0 { 39 | sign = "-" 40 | bytes *= -1 41 | } 42 | i := 0 43 | for ; bytes > 1024 && i < 4; i++ { 44 | bytes /= 1024 45 | } 46 | return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i]) 47 | } 48 | 49 | func sint(x int) string { 50 | if x == 0 { 51 | return "~" 52 | } 53 | sign := "+" 54 | if x < 0 { 55 | sign = "-" 56 | x *= -1 57 | } 58 | return fmt.Sprintf("%s%d", sign, x) 59 | } 60 | 61 | func minInt(a, b int) int { 62 | if a < b { 63 | return a 64 | } 65 | return b 66 | } 67 | 68 | func maxInt(a, b int) int { 69 | if a > b { 70 | return a 71 | } 72 | return b 73 | } 74 | 75 | type fdSorter []storage.FileDesc 76 | 77 | func (p fdSorter) Len() int { 78 | return len(p) 79 | } 80 | 81 | func (p fdSorter) Less(i, j int) bool { 82 | return p[i].Num < p[j].Num 83 | } 84 | 85 | func (p fdSorter) Swap(i, j int) { 86 | p[i], p[j] = p[j], p[i] 87 | } 88 | 89 | func sortFds(fds []storage.FileDesc) { 90 | sort.Sort(fdSorter(fds)) 91 | } 92 | 93 | func ensureBuffer(b []byte, n int) []byte { 94 | if cap(b) < n { 95 | return make([]byte, n) 96 | } 97 | return b[:n] 98 | } 99 | -------------------------------------------------------------------------------- /leveldb/util/buffer_pool.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "fmt" 11 | "sync" 12 | "sync/atomic" 13 | "time" 14 | ) 15 | 16 | type buffer struct { 17 | b []byte 18 | miss int 19 | } 20 | 21 | // BufferPool is a 'buffer pool'. 22 | type BufferPool struct { 23 | pool [6]chan []byte 24 | size [5]uint32 25 | sizeMiss [5]uint32 26 | sizeHalf [5]uint32 27 | baseline [4]int 28 | baseline0 int 29 | 30 | mu sync.RWMutex 31 | closed bool 32 | closeC chan struct{} 33 | 34 | get uint32 35 | put uint32 36 | half uint32 37 | less uint32 38 | equal uint32 39 | greater uint32 40 | miss uint32 41 | } 42 | 43 | func (p *BufferPool) poolNum(n int) int { 44 | if n <= p.baseline0 && n > p.baseline0/2 { 45 | return 0 46 | } 47 | for i, x := range p.baseline { 48 | if n <= x { 49 | return i + 1 50 | } 51 | } 52 | return len(p.baseline) + 1 53 | } 54 | 55 | // Get returns buffer with length of n. 56 | func (p *BufferPool) Get(n int) []byte { 57 | if p == nil { 58 | return make([]byte, n) 59 | } 60 | 61 | p.mu.RLock() 62 | defer p.mu.RUnlock() 63 | 64 | if p.closed { 65 | return make([]byte, n) 66 | } 67 | 68 | atomic.AddUint32(&p.get, 1) 69 | 70 | poolNum := p.poolNum(n) 71 | pool := p.pool[poolNum] 72 | if poolNum == 0 { 73 | // Fast path. 74 | select { 75 | case b := <-pool: 76 | switch { 77 | case cap(b) > n: 78 | if cap(b)-n >= n { 79 | atomic.AddUint32(&p.half, 1) 80 | select { 81 | case pool <- b: 82 | default: 83 | } 84 | return make([]byte, n) 85 | } else { 86 | atomic.AddUint32(&p.less, 1) 87 | return b[:n] 88 | } 89 | case cap(b) == n: 90 | atomic.AddUint32(&p.equal, 1) 91 | return b[:n] 92 | default: 93 | atomic.AddUint32(&p.greater, 1) 94 | } 95 | default: 96 | atomic.AddUint32(&p.miss, 1) 97 | } 98 | 99 | return make([]byte, n, p.baseline0) 100 | } else { 101 | sizePtr := &p.size[poolNum-1] 102 | 103 | select { 104 | case b := <-pool: 105 | switch { 106 | case cap(b) > n: 107 | if cap(b)-n >= n { 108 | atomic.AddUint32(&p.half, 1) 109 | sizeHalfPtr := &p.sizeHalf[poolNum-1] 110 | if atomic.AddUint32(sizeHalfPtr, 1) == 20 { 111 | atomic.StoreUint32(sizePtr, uint32(cap(b)/2)) 112 | atomic.StoreUint32(sizeHalfPtr, 0) 113 | } else { 114 | select { 115 | case pool <- b: 116 | default: 117 | } 118 | } 119 | return make([]byte, n) 120 | } else { 121 | atomic.AddUint32(&p.less, 1) 122 | return b[:n] 123 | } 124 | case cap(b) == n: 125 | atomic.AddUint32(&p.equal, 1) 126 | return b[:n] 127 | default: 128 | atomic.AddUint32(&p.greater, 1) 129 | if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) { 130 | select { 131 | case pool <- b: 132 | default: 133 | } 134 | } 135 | } 136 | default: 137 | atomic.AddUint32(&p.miss, 1) 138 | } 139 | 140 | if size := atomic.LoadUint32(sizePtr); uint32(n) > size { 141 | if size == 0 { 142 | atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n)) 143 | } else { 144 | sizeMissPtr := &p.sizeMiss[poolNum-1] 145 | if atomic.AddUint32(sizeMissPtr, 1) == 20 { 146 | atomic.StoreUint32(sizePtr, uint32(n)) 147 | atomic.StoreUint32(sizeMissPtr, 0) 148 | } 149 | } 150 | return make([]byte, n) 151 | } else { 152 | return make([]byte, n, size) 153 | } 154 | } 155 | } 156 | 157 | // Put adds given buffer to the pool. 158 | func (p *BufferPool) Put(b []byte) { 159 | if p == nil { 160 | return 161 | } 162 | 163 | p.mu.RLock() 164 | defer p.mu.RUnlock() 165 | 166 | if p.closed { 167 | return 168 | } 169 | 170 | atomic.AddUint32(&p.put, 1) 171 | 172 | pool := p.pool[p.poolNum(cap(b))] 173 | select { 174 | case pool <- b: 175 | default: 176 | } 177 | 178 | } 179 | 180 | func (p *BufferPool) Close() { 181 | if p == nil { 182 | return 183 | } 184 | 185 | p.mu.Lock() 186 | if !p.closed { 187 | p.closed = true 188 | p.closeC <- struct{}{} 189 | } 190 | p.mu.Unlock() 191 | } 192 | 193 | func (p *BufferPool) String() string { 194 | if p == nil { 195 | return "" 196 | } 197 | 198 | return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}", 199 | p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss) 200 | } 201 | 202 | func (p *BufferPool) drain() { 203 | ticker := time.NewTicker(2 * time.Second) 204 | defer ticker.Stop() 205 | for { 206 | select { 207 | case <-ticker.C: 208 | for _, ch := range p.pool { 209 | select { 210 | case <-ch: 211 | default: 212 | } 213 | } 214 | case <-p.closeC: 215 | close(p.closeC) 216 | for _, ch := range p.pool { 217 | close(ch) 218 | } 219 | return 220 | } 221 | } 222 | } 223 | 224 | // NewBufferPool creates a new initialized 'buffer pool'. 225 | func NewBufferPool(baseline int) *BufferPool { 226 | if baseline <= 0 { 227 | panic("baseline can't be <= 0") 228 | } 229 | p := &BufferPool{ 230 | baseline0: baseline, 231 | baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4}, 232 | closeC: make(chan struct{}, 1), 233 | } 234 | for i, cap := range []int{2, 2, 4, 4, 2, 1} { 235 | p.pool[i] = make(chan []byte, cap) 236 | } 237 | go p.drain() 238 | return p 239 | } 240 | -------------------------------------------------------------------------------- /leveldb/util/crc32.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The LevelDB-Go Authors. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style license that can be 4 | // found in the LICENSE file. 5 | 6 | package util 7 | 8 | import ( 9 | "hash/crc32" 10 | ) 11 | 12 | var table = crc32.MakeTable(crc32.Castagnoli) 13 | 14 | // CRC is a CRC-32 checksum computed using Castagnoli's polynomial. 15 | type CRC uint32 16 | 17 | // NewCRC creates a new crc based on the given bytes. 18 | func NewCRC(b []byte) CRC { 19 | return CRC(0).Update(b) 20 | } 21 | 22 | // Update updates the crc with the given bytes. 23 | func (c CRC) Update(b []byte) CRC { 24 | return CRC(crc32.Update(uint32(c), table, b)) 25 | } 26 | 27 | // Value returns a masked crc. 28 | func (c CRC) Value() uint32 { 29 | return uint32(c>>15|c<<17) + 0xa282ead8 30 | } 31 | -------------------------------------------------------------------------------- /leveldb/util/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "encoding/binary" 11 | ) 12 | 13 | // Hash return hash of the given data. 14 | func Hash(data []byte, seed uint32) uint32 { 15 | // Similar to murmur hash 16 | const ( 17 | m = uint32(0xc6a4a793) 18 | r = uint32(24) 19 | ) 20 | var ( 21 | h = seed ^ (uint32(len(data)) * m) 22 | i int 23 | ) 24 | 25 | for n := len(data) - len(data)%4; i < n; i += 4 { 26 | h += binary.LittleEndian.Uint32(data[i:]) 27 | h *= m 28 | h ^= (h >> 16) 29 | } 30 | 31 | switch len(data) - i { 32 | default: 33 | panic("not reached") 34 | case 3: 35 | h += uint32(data[i+2]) << 16 36 | fallthrough 37 | case 2: 38 | h += uint32(data[i+1]) << 8 39 | fallthrough 40 | case 1: 41 | h += uint32(data[i]) 42 | h *= m 43 | h ^= (h >> r) 44 | case 0: 45 | } 46 | 47 | return h 48 | } 49 | -------------------------------------------------------------------------------- /leveldb/util/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "testing" 11 | ) 12 | 13 | var hashTests = []struct { 14 | data []byte 15 | seed uint32 16 | hash uint32 17 | }{ 18 | {nil, 0xbc9f1d34, 0xbc9f1d34}, 19 | {[]byte{0x62}, 0xbc9f1d34, 0xef1345c4}, 20 | {[]byte{0xc3, 0x97}, 0xbc9f1d34, 0x5b663814}, 21 | {[]byte{0xe2, 0x99, 0xa5}, 0xbc9f1d34, 0x323c078f}, 22 | {[]byte{0xe1, 0x80, 0xb9, 0x32}, 0xbc9f1d34, 0xed21633a}, 23 | {[]byte{ 24 | 0x01, 0xc0, 0x00, 0x00, 25 | 0x00, 0x00, 0x00, 0x00, 26 | 0x00, 0x00, 0x00, 0x00, 27 | 0x00, 0x00, 0x00, 0x00, 28 | 0x14, 0x00, 0x00, 0x00, 29 | 0x00, 0x00, 0x04, 0x00, 30 | 0x00, 0x00, 0x00, 0x14, 31 | 0x00, 0x00, 0x00, 0x18, 32 | 0x28, 0x00, 0x00, 0x00, 33 | 0x00, 0x00, 0x00, 0x00, 34 | 0x02, 0x00, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | }, 0x12345678, 0xf333dabb}, 37 | } 38 | 39 | func TestHash(t *testing.T) { 40 | for i, x := range hashTests { 41 | h := Hash(x.data, x.seed) 42 | if h != x.hash { 43 | t.Fatalf("test-%d: invalid hash, %#x vs %#x", i, h, x.hash) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /leveldb/util/range.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | // Range is a key range. 10 | type Range struct { 11 | // Start of the key range, include in the range. 12 | Start []byte 13 | 14 | // Limit of the key range, not include in the range. 15 | Limit []byte 16 | } 17 | 18 | // BytesPrefix returns key range that satisfy the given prefix. 19 | // This only applicable for the standard 'bytes comparer'. 20 | func BytesPrefix(prefix []byte) *Range { 21 | var limit []byte 22 | for i := len(prefix) - 1; i >= 0; i-- { 23 | c := prefix[i] 24 | if c < 0xff { 25 | limit = make([]byte, i+1) 26 | copy(limit, prefix) 27 | limit[i] = c + 1 28 | break 29 | } 30 | } 31 | return &Range{prefix, limit} 32 | } 33 | -------------------------------------------------------------------------------- /leveldb/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package util provides utilities used throughout leveldb. 8 | package util 9 | 10 | import ( 11 | "errors" 12 | ) 13 | 14 | var ( 15 | ErrReleased = errors.New("leveldb: resource already relesed") 16 | ErrHasReleaser = errors.New("leveldb: releaser already defined") 17 | ) 18 | 19 | // Releaser is the interface that wraps the basic Release method. 20 | type Releaser interface { 21 | // Release releases associated resources. Release should always success 22 | // and can be called multipe times without causing error. 23 | Release() 24 | } 25 | 26 | // ReleaseSetter is the interface that wraps the basic SetReleaser method. 27 | type ReleaseSetter interface { 28 | // SetReleaser associates the given releaser to the resources. The 29 | // releaser will be called once coresponding resources released. 30 | // Calling SetReleaser with nil will clear the releaser. 31 | // 32 | // This will panic if a releaser already present or coresponding 33 | // resource is already released. Releaser should be cleared first 34 | // before assigned a new one. 35 | SetReleaser(releaser Releaser) 36 | } 37 | 38 | // BasicReleaser provides basic implementation of Releaser and ReleaseSetter. 39 | type BasicReleaser struct { 40 | releaser Releaser 41 | released bool 42 | } 43 | 44 | // Released returns whether Release method already called. 45 | func (r *BasicReleaser) Released() bool { 46 | return r.released 47 | } 48 | 49 | // Release implements Releaser.Release. 50 | func (r *BasicReleaser) Release() { 51 | if !r.released { 52 | if r.releaser != nil { 53 | r.releaser.Release() 54 | r.releaser = nil 55 | } 56 | r.released = true 57 | } 58 | } 59 | 60 | // SetReleaser implements ReleaseSetter.SetReleaser. 61 | func (r *BasicReleaser) SetReleaser(releaser Releaser) { 62 | if r.released { 63 | panic(ErrReleased) 64 | } 65 | if r.releaser != nil && releaser != nil { 66 | panic(ErrHasReleaser) 67 | } 68 | r.releaser = releaser 69 | } 70 | 71 | type NoopReleaser struct{} 72 | 73 | func (NoopReleaser) Release() {} 74 | -------------------------------------------------------------------------------- /leveldb/version_test.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "encoding/binary" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/onsi/gomega" 9 | 10 | "github.com/pingcap/goleveldb/leveldb/testutil" 11 | ) 12 | 13 | type testFileRec struct { 14 | level int 15 | num int64 16 | } 17 | 18 | func TestVersionStaging(t *testing.T) { 19 | gomega.RegisterTestingT(t) 20 | stor := testutil.NewStorage() 21 | defer stor.Close() 22 | s, err := newSession(stor, nil) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | v := newVersion(s) 28 | v.newStaging() 29 | 30 | tmp := make([]byte, 4) 31 | mik := func(i uint64) []byte { 32 | binary.BigEndian.PutUint32(tmp, uint32(i)) 33 | return []byte(makeInternalKey(nil, tmp, 0, keyTypeVal)) 34 | } 35 | 36 | for i, x := range []struct { 37 | add, del []testFileRec 38 | levels [][]int64 39 | }{ 40 | { 41 | add: []testFileRec{ 42 | {1, 1}, 43 | }, 44 | levels: [][]int64{ 45 | {}, 46 | {1}, 47 | }, 48 | }, 49 | { 50 | add: []testFileRec{ 51 | {1, 1}, 52 | }, 53 | levels: [][]int64{ 54 | {}, 55 | {1}, 56 | }, 57 | }, 58 | { 59 | del: []testFileRec{ 60 | {1, 1}, 61 | }, 62 | levels: [][]int64{}, 63 | }, 64 | { 65 | add: []testFileRec{ 66 | {0, 1}, 67 | {0, 3}, 68 | {0, 2}, 69 | {2, 5}, 70 | {1, 4}, 71 | }, 72 | levels: [][]int64{ 73 | {3, 2, 1}, 74 | {4}, 75 | {5}, 76 | }, 77 | }, 78 | { 79 | add: []testFileRec{ 80 | {1, 6}, 81 | {2, 5}, 82 | }, 83 | del: []testFileRec{ 84 | {0, 1}, 85 | {0, 4}, 86 | }, 87 | levels: [][]int64{ 88 | {3, 2}, 89 | {4, 6}, 90 | {5}, 91 | }, 92 | }, 93 | { 94 | del: []testFileRec{ 95 | {0, 3}, 96 | {0, 2}, 97 | {1, 4}, 98 | {1, 6}, 99 | {2, 5}, 100 | }, 101 | levels: [][]int64{}, 102 | }, 103 | { 104 | add: []testFileRec{ 105 | {0, 1}, 106 | }, 107 | levels: [][]int64{ 108 | {1}, 109 | }, 110 | }, 111 | { 112 | add: []testFileRec{ 113 | {1, 2}, 114 | }, 115 | levels: [][]int64{ 116 | {1}, 117 | {2}, 118 | }, 119 | }, 120 | { 121 | add: []testFileRec{ 122 | {0, 3}, 123 | }, 124 | levels: [][]int64{ 125 | {3, 1}, 126 | {2}, 127 | }, 128 | }, 129 | { 130 | add: []testFileRec{ 131 | {6, 9}, 132 | }, 133 | levels: [][]int64{ 134 | {3, 1}, 135 | {2}, 136 | {}, 137 | {}, 138 | {}, 139 | {}, 140 | {9}, 141 | }, 142 | }, 143 | { 144 | del: []testFileRec{ 145 | {6, 9}, 146 | }, 147 | levels: [][]int64{ 148 | {3, 1}, 149 | {2}, 150 | }, 151 | }, 152 | } { 153 | rec := &sessionRecord{} 154 | for _, f := range x.add { 155 | ik := mik(uint64(f.num)) 156 | rec.addTable(f.level, f.num, 1, ik, ik) 157 | } 158 | for _, f := range x.del { 159 | rec.delTable(f.level, f.num) 160 | } 161 | vs := v.newStaging() 162 | vs.commit(rec) 163 | v = vs.finish() 164 | if len(v.levels) != len(x.levels) { 165 | t.Fatalf("#%d: invalid level count: want=%d got=%d", i, len(x.levels), len(v.levels)) 166 | } 167 | for j, want := range x.levels { 168 | tables := v.levels[j] 169 | if len(want) != len(tables) { 170 | t.Fatalf("#%d.%d: invalid tables count: want=%d got=%d", i, j, len(want), len(tables)) 171 | } 172 | got := make([]int64, len(tables)) 173 | for k, t := range tables { 174 | got[k] = t.fd.Num 175 | } 176 | if !reflect.DeepEqual(want, got) { 177 | t.Fatalf("#%d.%d: invalid tables: want=%v got=%v", i, j, want, got) 178 | } 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /manualtest/dbstress/key.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | 7 | "github.com/pingcap/goleveldb/leveldb/errors" 8 | "github.com/pingcap/goleveldb/leveldb/storage" 9 | ) 10 | 11 | type ErrIkeyCorrupted struct { 12 | Ikey []byte 13 | Reason string 14 | } 15 | 16 | func (e *ErrIkeyCorrupted) Error() string { 17 | return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason) 18 | } 19 | 20 | func newErrIkeyCorrupted(ikey []byte, reason string) error { 21 | return errors.NewErrCorrupted(storage.FileDesc{}, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason}) 22 | } 23 | 24 | type kType int 25 | 26 | func (kt kType) String() string { 27 | switch kt { 28 | case ktDel: 29 | return "d" 30 | case ktVal: 31 | return "v" 32 | } 33 | return "x" 34 | } 35 | 36 | // Value types encoded as the last component of internal keys. 37 | // Don't modify; this value are saved to disk. 38 | const ( 39 | ktDel kType = iota 40 | ktVal 41 | ) 42 | 43 | // ktSeek defines the kType that should be passed when constructing an 44 | // internal key for seeking to a particular sequence number (since we 45 | // sort sequence numbers in decreasing order and the value type is 46 | // embedded as the low 8 bits in the sequence number in internal keys, 47 | // we need to use the highest-numbered ValueType, not the lowest). 48 | const ktSeek = ktVal 49 | 50 | const ( 51 | // Maximum value possible for sequence number; the 8-bits are 52 | // used by value type, so its can packed together in single 53 | // 64-bit integer. 54 | kMaxSeq uint64 = (uint64(1) << 56) - 1 55 | // Maximum value possible for packed sequence number and type. 56 | kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek) 57 | ) 58 | 59 | // Maximum number encoded in bytes. 60 | var kMaxNumBytes = make([]byte, 8) 61 | 62 | func init() { 63 | binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum) 64 | } 65 | 66 | type iKey []byte 67 | 68 | func newIkey(ukey []byte, seq uint64, kt kType) iKey { 69 | if seq > kMaxSeq { 70 | panic("leveldb: invalid sequence number") 71 | } else if kt > ktVal { 72 | panic("leveldb: invalid type") 73 | } 74 | 75 | ik := make(iKey, len(ukey)+8) 76 | copy(ik, ukey) 77 | binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt)) 78 | return ik 79 | } 80 | 81 | func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) { 82 | if len(ik) < 8 { 83 | return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length") 84 | } 85 | num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 86 | seq, kt = uint64(num>>8), kType(num&0xff) 87 | if kt > ktVal { 88 | return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type") 89 | } 90 | ukey = ik[:len(ik)-8] 91 | return 92 | } 93 | 94 | func validIkey(ik []byte) bool { 95 | _, _, _, err := parseIkey(ik) 96 | return err == nil 97 | } 98 | 99 | func (ik iKey) assert() { 100 | if ik == nil { 101 | panic("leveldb: nil iKey") 102 | } 103 | if len(ik) < 8 { 104 | panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", ik, len(ik))) 105 | } 106 | } 107 | 108 | func (ik iKey) ukey() []byte { 109 | ik.assert() 110 | return ik[:len(ik)-8] 111 | } 112 | 113 | func (ik iKey) num() uint64 { 114 | ik.assert() 115 | return binary.LittleEndian.Uint64(ik[len(ik)-8:]) 116 | } 117 | 118 | func (ik iKey) parseNum() (seq uint64, kt kType) { 119 | num := ik.num() 120 | seq, kt = uint64(num>>8), kType(num&0xff) 121 | if kt > ktVal { 122 | panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", ik, len(ik), kt)) 123 | } 124 | return 125 | } 126 | 127 | func (ik iKey) String() string { 128 | if ik == nil { 129 | return "" 130 | } 131 | 132 | if ukey, seq, kt, err := parseIkey(ik); err == nil { 133 | return fmt.Sprintf("%x,%s%d", ukey, kt, seq) 134 | } else { 135 | return "" 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /manualtest/filelock/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | 12 | "github.com/pingcap/goleveldb/leveldb/storage" 13 | ) 14 | 15 | var ( 16 | filename string 17 | child bool 18 | ) 19 | 20 | func init() { 21 | flag.StringVar(&filename, "filename", filepath.Join(os.TempDir(), "goleveldb_filelock_test"), "Filename used for testing") 22 | flag.BoolVar(&child, "child", false, "This is the child") 23 | } 24 | 25 | func runChild() error { 26 | var args []string 27 | args = append(args, os.Args[1:]...) 28 | args = append(args, "-child") 29 | cmd := exec.Command(os.Args[0], args...) 30 | var out bytes.Buffer 31 | cmd.Stdout = &out 32 | err := cmd.Run() 33 | r := bufio.NewReader(&out) 34 | for { 35 | line, _, e1 := r.ReadLine() 36 | if e1 != nil { 37 | break 38 | } 39 | fmt.Println("[Child]", string(line)) 40 | } 41 | return err 42 | } 43 | 44 | func main() { 45 | flag.Parse() 46 | 47 | fmt.Printf("Using path: %s\n", filename) 48 | if child { 49 | fmt.Println("Child flag set.") 50 | } 51 | 52 | stor, err := storage.OpenFile(filename, false) 53 | if err != nil { 54 | fmt.Printf("Could not open storage: %s", err) 55 | os.Exit(10) 56 | } 57 | 58 | if !child { 59 | fmt.Println("Executing child -- first test (expecting error)") 60 | err := runChild() 61 | if err == nil { 62 | fmt.Println("Expecting error from child") 63 | } else if err.Error() != "exit status 10" { 64 | fmt.Println("Got unexpected error from child:", err) 65 | } else { 66 | fmt.Printf("Got error from child: %s (expected)\n", err) 67 | } 68 | } 69 | 70 | err = stor.Close() 71 | if err != nil { 72 | fmt.Printf("Error when closing storage: %s", err) 73 | os.Exit(11) 74 | } 75 | 76 | if !child { 77 | fmt.Println("Executing child -- second test") 78 | err := runChild() 79 | if err != nil { 80 | fmt.Println("Got unexpected error from child:", err) 81 | } 82 | } 83 | 84 | os.RemoveAll(filename) 85 | } 86 | --------------------------------------------------------------------------------