├── .gitignore ├── LICENSE ├── README.md ├── btree.go ├── btree_test.go ├── debug.go ├── delete.go ├── insert.go ├── merge.go ├── metadata.pb.go ├── metadata.proto ├── search.go ├── serialize.go ├── split.go ├── update.go └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | treedump* 2 | btree.test 3 | *.out 4 | test.sh 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013, datastream 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the FreeBSD Project. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Btree library 2 | 3 | This is pure golang btree library. it's copy on write btree. 4 | 5 | ``` 6 | go get github.com/datastream/btree 7 | ``` 8 | 9 | ## API 10 | 11 | ### NewRecord(key, value []byte) 12 | 13 | create a record 14 | 15 | ### NewBtree() 16 | 17 | create a btree 18 | 19 | LEAFSIZE = 1 << 5 20 | NODESIZE = 1 << 6 21 | 22 | ### NewBtreeSize(leafsize, nodesize) 23 | 24 | create new btree with custom leafsize/nodesize 25 | 26 | ### btree.Insert(key, value) 27 | 28 | Insert a record, if insert success, it return nil 29 | 30 | ### btree.Update(key, value) 31 | 32 | Update a record, if update success, it return nil 33 | 34 | ### btree.Delete(key) 35 | 36 | Delete a record, if delete success, it return nil 37 | 38 | ### btree.Search(key) 39 | 40 | Search a key, if find success, it return value, nil 41 | 42 | ### btree.Marshal(filename) 43 | 44 | Write btree data into disk. 45 | 46 | tree.Dump("treedump") 47 | 48 | ### btree.Unmarshal(filename) 49 | 50 | Read btree from disk 51 | 52 | ## TODO 53 | 54 | 1. more test 55 | 2. code tunning for performance or mem 56 | -------------------------------------------------------------------------------- /btree.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "github.com/golang/protobuf/proto" 5 | "time" 6 | ) 7 | 8 | // Btree metadata 9 | type Btree struct { 10 | BtreeMetadata 11 | gcIndex int64 12 | dupnodelist map[int64]int 13 | opChan chan *treeOperation 14 | exitChan chan int 15 | } 16 | 17 | const ( 18 | // TreeSize is tree size 19 | TreeSize = 1 << 10 20 | // LeafSize is leaf size 21 | LeafSize = 1 << 5 22 | // NodeSize is node size 23 | NodeSize = 1 << 6 24 | ) 25 | 26 | // isNode, isLeaf is treenode tag 27 | const ( 28 | isNode = iota 29 | isLeaf 30 | ) 31 | 32 | // NewBtree create a btree 33 | func NewBtree() *Btree { 34 | return NewBtreeSize(LeafSize, NodeSize) 35 | } 36 | 37 | // NewBtreeSize create new btree with custom leafsize/nodesize 38 | func NewBtreeSize(leafsize int64, nodesize int64) *Btree { 39 | tree := &Btree{ 40 | dupnodelist: make(map[int64]int), 41 | opChan: make(chan *treeOperation), 42 | BtreeMetadata: BtreeMetadata{ 43 | Root: proto.Int64(0), 44 | Size: proto.Int64(TreeSize), 45 | LeafMax: proto.Int64(leafsize), 46 | NodeMax: proto.Int64(nodesize), 47 | IndexCursor: proto.Int64(0), 48 | Nodes: make([][]byte, TreeSize), 49 | }, 50 | } 51 | go tree.run() 52 | return tree 53 | } 54 | 55 | func (t *Btree) run() { 56 | tick := time.Tick(time.Second * 2) 57 | for { 58 | select { 59 | case <-t.exitChan: 60 | break 61 | case op := <-t.opChan: 62 | switch op.GetAction() { 63 | case "insert": 64 | op.errChan <- t.insert(op.TreeLog) 65 | case "delete": 66 | op.errChan <- t.dodelete(op.Key) 67 | case "update": 68 | op.errChan <- t.update(op.TreeLog) 69 | case "search": 70 | rst, err := t.search(op.Key) 71 | op.valueChan <- rst 72 | op.errChan <- err 73 | } 74 | t.Index = proto.Int64(t.GetIndexCursor()) 75 | case <-tick: 76 | t.gc() 77 | } 78 | } 79 | t.Marshal("treedump.tmp") 80 | } 81 | 82 | func (t *Btree) Sync(file string) { 83 | //t.Marshal(file) 84 | } 85 | 86 | // Insert can insert record into a btree 87 | func (t *Btree) Insert(key, value []byte) error { 88 | q := &treeOperation{ 89 | valueChan: make(chan []byte), 90 | errChan: make(chan error), 91 | } 92 | q.Action = proto.String("insert") 93 | q.Key = key 94 | q.Value = value 95 | t.opChan <- q 96 | return <-q.errChan 97 | } 98 | 99 | // Delete can delete record 100 | func (t *Btree) Delete(key []byte) error { 101 | q := &treeOperation{ 102 | valueChan: make(chan []byte), 103 | errChan: make(chan error), 104 | } 105 | q.Action = proto.String("delete") 106 | q.Key = key 107 | t.opChan <- q 108 | return <-q.errChan 109 | } 110 | 111 | // Search return value 112 | func (t *Btree) Search(key []byte) ([]byte, error) { 113 | q := &treeOperation{ 114 | valueChan: make(chan []byte), 115 | errChan: make(chan error), 116 | } 117 | q.Action = proto.String("search") 118 | q.Key = key 119 | t.opChan <- q 120 | return <-q.valueChan, <-q.errChan 121 | } 122 | 123 | // Update is used to update key/value 124 | func (t *Btree) Update(key, value []byte) error { 125 | q := &treeOperation{ 126 | valueChan: make(chan []byte), 127 | errChan: make(chan error), 128 | } 129 | q.Action = proto.String("update") 130 | q.Key = key 131 | q.Value = value 132 | t.opChan <- q 133 | return <-q.errChan 134 | } 135 | -------------------------------------------------------------------------------- /btree_test.go: -------------------------------------------------------------------------------- 1 | package btree_test 2 | 3 | import ( 4 | "../btree" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestInsert(t *testing.T) { 10 | tree := btree.NewBtreeSize(2, 2) 11 | size := 100 12 | for i := 0; i < size; i++ { 13 | if err := tree.Insert([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { 14 | t.Fatal("Insert Failed", i, err) 15 | } 16 | } 17 | } 18 | 19 | func TestSearch(t *testing.T) { 20 | tree := btree.NewBtreeSize(2, 3) 21 | size := 100 22 | for i := 0; i < size; i++ { 23 | tree.Insert([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) 24 | } 25 | for i := 0; i < size; i++ { 26 | rst, err := tree.Search([]byte(strconv.Itoa(i))) 27 | if string(rst) != strconv.Itoa(i) { 28 | t.Fatal("Find Failed", i, err) 29 | } 30 | } 31 | } 32 | func TestUpdate(t *testing.T) { 33 | tree := btree.NewBtreeSize(3, 2) 34 | size := 100 35 | for i := 0; i < size; i++ { 36 | tree.Insert([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) 37 | } 38 | for i := 0; i < size; i++ { 39 | if err := tree.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i+1))); err != nil { 40 | t.Fatal("Update Failed", i, err) 41 | } 42 | } 43 | for i := 0; i < size; i++ { 44 | rst, _ := tree.Search([]byte(strconv.Itoa(i))) 45 | if string(rst) != strconv.Itoa(i+1) { 46 | t.Fatal("Find Failed", i) 47 | } 48 | } 49 | } 50 | func TestDelete(t *testing.T) { 51 | tree := btree.NewBtreeSize(3, 3) 52 | size := 8 53 | for i := 0; i < size; i++ { 54 | tree.Insert([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) 55 | } 56 | for i := 0; i < size; i++ { 57 | if err := tree.Delete([]byte(strconv.Itoa(i))); err != nil { 58 | t.Fatal("delete Failed", i) 59 | } 60 | if _, err := tree.Search([]byte(strconv.Itoa(i))); err == nil { 61 | t.Fatal("Find Failed", i) 62 | } 63 | } 64 | } 65 | 66 | func BenchmarkInsert(t *testing.B) { 67 | size := 100000 68 | tree := btree.NewBtreeSize(16, 16) 69 | for i := 0; i < size; i++ { 70 | tree.Insert([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) 71 | } 72 | tree.Marshal("treedump") 73 | } 74 | 75 | func BenchmarkSearch(t *testing.B) { 76 | tree, err := btree.Unmarshal("treedump") 77 | if err != nil { 78 | t.Fatal("Unmarshal Failed") 79 | } 80 | size := 100000 81 | for i := 0; i < size; i++ { 82 | rst, err := tree.Search([]byte(strconv.Itoa(i))) 83 | if string(rst) != strconv.Itoa(i) { 84 | t.Fatal("Find Failed", i, err) 85 | } 86 | } 87 | } 88 | 89 | func BenchmarkUpdate(t *testing.B) { 90 | tree, err := btree.Unmarshal("treedump") 91 | if err != nil { 92 | t.Fatal("Unmarshal Failed") 93 | } 94 | size := 100000 95 | for i := 0; i < size; i++ { 96 | if err := tree.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i+1))); err != nil { 97 | t.Fatal("Update Failed", i, err) 98 | } 99 | } 100 | for i := 0; i < size; i++ { 101 | rst, _ := tree.Search([]byte(strconv.Itoa(i))) 102 | if string(rst) != strconv.Itoa(i+1) { 103 | t.Fatal("Find Failed", i) 104 | } 105 | } 106 | } 107 | func BenchmarkDelete(t *testing.B) { 108 | tree, err := btree.Unmarshal("treedump") 109 | if err != nil { 110 | t.Fatal("Unmarshal Failed") 111 | } 112 | size := 100000 113 | for i := 0; i < size; i++ { 114 | if err := tree.Delete([]byte(strconv.Itoa(i))); err != nil { 115 | t.Fatal("delete Failed", i) 116 | } 117 | if _, err := tree.Search([]byte(strconv.Itoa(i))); err == nil { 118 | t.Fatal("Find Failed", i) 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | //debug func 8 | 9 | func (n *TreeNode) printNode() { 10 | if n.GetNodeType() == isLeaf { 11 | fmt.Println("---LeafID---", n.GetId()) 12 | fmt.Println("Key", toArray(n.GetKeys())) 13 | fmt.Println("values", toArray(n.GetValues())) 14 | } else { 15 | fmt.Println("---NodeID---", n.GetId()) 16 | fmt.Println("Key", toArray(n.GetKeys())) 17 | fmt.Println("Childrens", n.GetChildrens()) 18 | } 19 | } 20 | 21 | func toArray(data [][]byte) []string { 22 | var rst []string 23 | for _, v := range data { 24 | rst = append(rst, string(v)) 25 | } 26 | return rst 27 | } 28 | 29 | // PrintInfo print some basic btree info 30 | func (t *Btree) PrintInfo() { 31 | fmt.Println("Root", t.GetRoot()) 32 | fmt.Println("IndexCursor", t.GetIndexCursor()) 33 | } 34 | 35 | // PrintTree print all btree's leafs/nodes 36 | func (t *Btree) PrintTree() { 37 | fmt.Println("-----------Tree-------------") 38 | for i := 0; i < int(t.GetIndexCursor()); i++ { 39 | if node, err := t.getTreeNode(int64(i)); err == nil { 40 | if node.GetIsDirt() == 0 { 41 | node.printNode() 42 | fmt.Println("--------") 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /delete.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/golang/protobuf/proto" 7 | "sync/atomic" 8 | ) 9 | 10 | func (t *Btree) dodelete(key []byte) error { 11 | tnode, err := t.getTreeNode(t.GetRoot()) 12 | if err != nil { 13 | return err 14 | } 15 | clonedNode, _, err := tnode.deleteRecord(key, t) 16 | if err == nil { 17 | if len(clonedNode.GetKeys()) == 0 { 18 | if clonedNode.GetNodeType() == isNode { 19 | if len(clonedNode.GetChildrens()) > 0 { 20 | newroot, err := t.getTreeNode(clonedNode.Childrens[0]) 21 | if err == nil { 22 | atomic.StoreInt32(clonedNode.IsDirt, 1) 23 | t.Nodes[clonedNode.GetId()], err = proto.Marshal(clonedNode) 24 | t.Root = proto.Int64(newroot.GetId()) 25 | } 26 | return err 27 | } 28 | } 29 | } 30 | t.Root = proto.Int64(clonedNode.GetId()) 31 | } 32 | return err 33 | } 34 | 35 | func (n *TreeNode) deleteRecord(key []byte, tree *Btree) (*TreeNode, []byte, error) { 36 | index := n.locate(key) 37 | var err error 38 | var nnode *TreeNode 39 | var newKey []byte 40 | var clonedNode *TreeNode 41 | if n.GetNodeType() == isNode { 42 | tnode, err := tree.getTreeNode(n.Childrens[index]) 43 | clonedNode, newKey, err = tnode.deleteRecord(key, tree) 44 | if err == nil { 45 | nnode = n.clone(tree) 46 | nnode.Childrens[index] = clonedNode.GetId() 47 | tmpKey := newKey 48 | if len(newKey) > 0 { 49 | if nnode.replace(key, newKey) { 50 | newKey = []byte{} 51 | } 52 | } 53 | if index == 0 { 54 | index = 1 55 | } 56 | if len(nnode.Keys) > 0 { 57 | left := nnode.merge(tree, index-1) 58 | if index == 1 && len(tmpKey) == 0 { 59 | tt, _ := tree.getTreeNode(nnode.Childrens[0]) 60 | if tt.GetNodeType() == isLeaf { 61 | if len(tt.Keys) > 0 { 62 | newKey = tt.Keys[0] 63 | } 64 | } 65 | } 66 | if left > 0 { 67 | nnode.Childrens[index-1] = left 68 | } 69 | } 70 | } 71 | } 72 | if n.GetNodeType() == isLeaf { 73 | index -= 1 74 | if index >= 0 && bytes.Compare(n.Keys[index], key) == 0 { 75 | nnode = n.clone(tree) 76 | nnode.Keys = append(nnode.Keys[:index], nnode.Keys[index+1:]...) 77 | nnode.Values = append(nnode.Values[:index], nnode.Values[index+1:]...) 78 | if index == 0 && len(nnode.Keys) > 0 { 79 | newKey = nnode.Keys[0] 80 | } 81 | } else { 82 | return nil, newKey, fmt.Errorf("delete failed") 83 | } 84 | } 85 | tree.Nodes[nnode.GetId()], err = proto.Marshal(nnode) 86 | return nnode, newKey, err 87 | } 88 | 89 | // replace delete key 90 | func (n *TreeNode) replace(oldKey, newKey []byte) bool { 91 | index := n.locate(oldKey) - 1 92 | if index >= 0 { 93 | if bytes.Compare(n.Keys[index], oldKey) == 0 { 94 | n.Keys[index] = newKey 95 | return true 96 | } 97 | } 98 | return false 99 | } 100 | -------------------------------------------------------------------------------- /insert.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/golang/protobuf/proto" 7 | ) 8 | 9 | // Insert can insert record into a btree 10 | func (t *Btree) insert(record TreeLog) error { 11 | tnode, err := t.getTreeNode(t.GetRoot()) 12 | if err != nil { 13 | if err.Error() != "no data" { 14 | return err 15 | } 16 | nnode := t.newTreeNode() 17 | nnode.NodeType = proto.Int32(isLeaf) 18 | _, err = nnode.insertRecord(record, t) 19 | if err == nil { 20 | t.Nodes[nnode.GetId()], err = proto.Marshal(nnode) 21 | } 22 | t.Root = proto.Int64(nnode.GetId()) 23 | return err 24 | } 25 | clonednode, err := tnode.insertRecord(record, t) 26 | if err == nil && len(clonednode.GetKeys()) > int(t.GetNodeMax()) { 27 | nnode := t.newTreeNode() 28 | nnode.NodeType = proto.Int32(isNode) 29 | key, left, right := clonednode.split(t) 30 | nnode.insertOnce(key, left, right, t) 31 | t.Nodes[nnode.GetId()], err = proto.Marshal(nnode) 32 | t.Root = proto.Int64(nnode.GetId()) 33 | } else { 34 | t.Root = proto.Int64(clonednode.GetId()) 35 | } 36 | return err 37 | } 38 | 39 | // insert node 40 | func (n *TreeNode) insertRecord(record TreeLog, tree *Btree) (*TreeNode, error) { 41 | var err error 42 | index := n.locate(record.Key) 43 | if n.GetNodeType() == isNode { 44 | tnode, err := tree.getTreeNode(n.Childrens[index]) 45 | clonedTreeNode, err := tnode.insertRecord(record, tree) 46 | if err == nil { 47 | clonedNode := n.clone(tree) 48 | clonedNode.Childrens[index] = clonedTreeNode.GetId() 49 | if len(clonedTreeNode.GetKeys()) > int(tree.GetNodeMax()) { 50 | key, left, right := clonedTreeNode.split(tree) 51 | err = clonedNode.insertOnce(key, left, right, tree) 52 | if err != nil { 53 | return nil, err 54 | } 55 | } 56 | tree.Nodes[clonedNode.GetId()], err = proto.Marshal(clonedNode) 57 | return clonedNode, err 58 | } 59 | return nil, err 60 | } 61 | if n.GetNodeType() == isLeaf { 62 | if index > 0 { 63 | if bytes.Compare(n.Keys[index-1], record.Key) == 0 { 64 | return nil, fmt.Errorf("key already inserted") 65 | } 66 | } 67 | var nnode *TreeNode 68 | if len(n.GetKeys()) == 0 { 69 | nnode = n 70 | } else { 71 | nnode = n.clone(tree) 72 | } 73 | nnode.Keys = append(nnode.Keys[:index], 74 | append([][]byte{record.Key}, nnode.Keys[index:]...)...) 75 | nnode.Values = append(nnode.Values[:index], 76 | append([][]byte{record.Value}, nnode.Values[index:]...)...) 77 | tree.Nodes[nnode.GetId()], err = proto.Marshal(nnode) 78 | return nnode, err 79 | } 80 | return nil, fmt.Errorf("insert record failed") 81 | } 82 | 83 | // Insert key into tree node 84 | func (n *TreeNode) insertOnce(key []byte, leftID int64, rightID int64, tree *Btree) error { 85 | var err error 86 | index := n.locate(key) 87 | if len(n.Keys) == 0 { 88 | n.Childrens = append([]int64{leftID}, rightID) 89 | } else { 90 | n.Childrens = append(n.Childrens[:index+1], 91 | append([]int64{rightID}, n.Childrens[index+1:]...)...) 92 | } 93 | n.Keys = append(n.Keys[:index], append([][]byte{key}, n.Keys[index:]...)...) 94 | tree.Nodes[n.GetId()], err = proto.Marshal(n) 95 | return err 96 | } 97 | -------------------------------------------------------------------------------- /merge.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "github.com/golang/protobuf/proto" 5 | "sync/atomic" 6 | ) 7 | 8 | func (n *TreeNode) merge(tree *Btree, index int) int64 { 9 | left, err := tree.getTreeNode(n.Childrens[index]) 10 | if err != nil { 11 | return -1 12 | } 13 | right, err := tree.getTreeNode(n.Childrens[index+1]) 14 | if err != nil { 15 | return -1 16 | } 17 | if len(left.Keys)+len(right.Keys) > int(tree.GetNodeMax()) { 18 | return -1 19 | } 20 | if (len(left.Values) + len(right.Values)) > int(tree.GetLeafMax()) { 21 | return -1 22 | } 23 | leftClone := left.clone(tree) 24 | n.Childrens[index] = leftClone.GetId() 25 | if leftClone.GetNodeType() == isLeaf { 26 | if index == len(n.Keys) { 27 | n.Childrens = n.Childrens[:index] 28 | n.Keys = n.Keys[:index-1] 29 | } else { 30 | n.Childrens = append(n.Childrens[:index+1], n.Childrens[index+2:]...) 31 | n.Keys = append(n.Keys[:index], n.Keys[index+1:]...) 32 | } 33 | // add right to left 34 | leftClone.Values = append(leftClone.Values, right.Values...) 35 | leftClone.Keys = append(leftClone.Keys, right.Keys...) 36 | } else { 37 | leftClone.Keys = append(leftClone.Keys, append([][]byte{n.Keys[index]}, right.Keys...)...) 38 | // merge childrens 39 | leftClone.Childrens = append(leftClone.Childrens, right.Childrens...) 40 | // remove old key 41 | n.Keys = append(n.Keys[:index], n.Keys[index+1:]...) 42 | // remove old right node 43 | n.Childrens = append(n.Childrens[:index+1], n.Childrens[index+2:]...) 44 | // check size, spilt if over size 45 | if len(leftClone.Keys) > int(tree.GetNodeMax()) { 46 | key, left, right := leftClone.split(tree) 47 | n.insertOnce(key, left, right, tree) 48 | } 49 | } 50 | atomic.StoreInt32(right.IsDirt, 1) 51 | atomic.StoreInt32(left.IsDirt, 1) 52 | tree.Nodes[right.GetId()], err = proto.Marshal(right) 53 | tree.Nodes[left.GetId()], err = proto.Marshal(left) 54 | tree.Nodes[leftClone.GetId()], err = proto.Marshal(leftClone) 55 | return leftClone.GetId() 56 | } 57 | -------------------------------------------------------------------------------- /metadata.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: metadata.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package btree is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | metadata.proto 10 | 11 | It has these top-level messages: 12 | BtreeMetadata 13 | TreeNode 14 | TreeLog 15 | */ 16 | package btree 17 | 18 | import proto "github.com/golang/protobuf/proto" 19 | import math "math" 20 | 21 | // Reference imports to suppress errors if they are not otherwise used. 22 | var _ = proto.Marshal 23 | var _ = math.Inf 24 | 25 | type BtreeMetadata struct { 26 | Root *int64 `protobuf:"varint,1,opt,name=root" json:"root,omitempty"` 27 | Size *int64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` 28 | LeafMax *int64 `protobuf:"varint,4,opt,name=leaf_max" json:"leaf_max,omitempty"` 29 | NodeMax *int64 `protobuf:"varint,5,opt,name=node_max" json:"node_max,omitempty"` 30 | IndexCursor *int64 `protobuf:"varint,7,opt,name=index_cursor" json:"index_cursor,omitempty"` 31 | Index *int64 `protobuf:"varint,8,opt,name=index" json:"index,omitempty"` 32 | Nodes [][]byte `protobuf:"bytes,9,rep,name=nodes" json:"nodes,omitempty"` 33 | XXX_unrecognized []byte `json:"-"` 34 | } 35 | 36 | func (m *BtreeMetadata) Reset() { *m = BtreeMetadata{} } 37 | func (m *BtreeMetadata) String() string { return proto.CompactTextString(m) } 38 | func (*BtreeMetadata) ProtoMessage() {} 39 | 40 | func (m *BtreeMetadata) GetRoot() int64 { 41 | if m != nil && m.Root != nil { 42 | return *m.Root 43 | } 44 | return 0 45 | } 46 | 47 | func (m *BtreeMetadata) GetSize() int64 { 48 | if m != nil && m.Size != nil { 49 | return *m.Size 50 | } 51 | return 0 52 | } 53 | 54 | func (m *BtreeMetadata) GetLeafMax() int64 { 55 | if m != nil && m.LeafMax != nil { 56 | return *m.LeafMax 57 | } 58 | return 0 59 | } 60 | 61 | func (m *BtreeMetadata) GetNodeMax() int64 { 62 | if m != nil && m.NodeMax != nil { 63 | return *m.NodeMax 64 | } 65 | return 0 66 | } 67 | 68 | func (m *BtreeMetadata) GetIndexCursor() int64 { 69 | if m != nil && m.IndexCursor != nil { 70 | return *m.IndexCursor 71 | } 72 | return 0 73 | } 74 | 75 | func (m *BtreeMetadata) GetIndex() int64 { 76 | if m != nil && m.Index != nil { 77 | return *m.Index 78 | } 79 | return 0 80 | } 81 | 82 | func (m *BtreeMetadata) GetNodes() [][]byte { 83 | if m != nil { 84 | return m.Nodes 85 | } 86 | return nil 87 | } 88 | 89 | type TreeNode struct { 90 | Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` 91 | Keys [][]byte `protobuf:"bytes,2,rep,name=keys" json:"keys,omitempty"` 92 | Childrens []int64 `protobuf:"varint,3,rep,name=childrens" json:"childrens,omitempty"` 93 | Values [][]byte `protobuf:"bytes,4,rep,name=values" json:"values,omitempty"` 94 | NodeType *int32 `protobuf:"varint,5,opt,name=node_type" json:"node_type,omitempty"` 95 | IsDirt *int32 `protobuf:"varint,6,opt,name=is_dirt" json:"is_dirt,omitempty"` 96 | XXX_unrecognized []byte `json:"-"` 97 | } 98 | 99 | func (m *TreeNode) Reset() { *m = TreeNode{} } 100 | func (m *TreeNode) String() string { return proto.CompactTextString(m) } 101 | func (*TreeNode) ProtoMessage() {} 102 | 103 | func (m *TreeNode) GetId() int64 { 104 | if m != nil && m.Id != nil { 105 | return *m.Id 106 | } 107 | return 0 108 | } 109 | 110 | func (m *TreeNode) GetKeys() [][]byte { 111 | if m != nil { 112 | return m.Keys 113 | } 114 | return nil 115 | } 116 | 117 | func (m *TreeNode) GetChildrens() []int64 { 118 | if m != nil { 119 | return m.Childrens 120 | } 121 | return nil 122 | } 123 | 124 | func (m *TreeNode) GetValues() [][]byte { 125 | if m != nil { 126 | return m.Values 127 | } 128 | return nil 129 | } 130 | 131 | func (m *TreeNode) GetNodeType() int32 { 132 | if m != nil && m.NodeType != nil { 133 | return *m.NodeType 134 | } 135 | return 0 136 | } 137 | 138 | func (m *TreeNode) GetIsDirt() int32 { 139 | if m != nil && m.IsDirt != nil { 140 | return *m.IsDirt 141 | } 142 | return 0 143 | } 144 | 145 | type TreeLog struct { 146 | Action *string `protobuf:"bytes,1,opt,name=action" json:"action,omitempty"` 147 | Key []byte `protobuf:"bytes,2,opt,name=key" json:"key,omitempty"` 148 | Value []byte `protobuf:"bytes,3,opt,name=value" json:"value,omitempty"` 149 | XXX_unrecognized []byte `json:"-"` 150 | } 151 | 152 | func (m *TreeLog) Reset() { *m = TreeLog{} } 153 | func (m *TreeLog) String() string { return proto.CompactTextString(m) } 154 | func (*TreeLog) ProtoMessage() {} 155 | 156 | func (m *TreeLog) GetAction() string { 157 | if m != nil && m.Action != nil { 158 | return *m.Action 159 | } 160 | return "" 161 | } 162 | 163 | func (m *TreeLog) GetKey() []byte { 164 | if m != nil { 165 | return m.Key 166 | } 167 | return nil 168 | } 169 | 170 | func (m *TreeLog) GetValue() []byte { 171 | if m != nil { 172 | return m.Value 173 | } 174 | return nil 175 | } 176 | 177 | func init() { 178 | } 179 | -------------------------------------------------------------------------------- /metadata.proto: -------------------------------------------------------------------------------- 1 | package btree; 2 | 3 | message btree_metadata 4 | { 5 | optional int64 root = 1; 6 | optional int64 size = 3; 7 | optional int64 leaf_max = 4; 8 | optional int64 node_max = 5; 9 | optional int64 index_cursor = 7; 10 | optional int64 index = 8; 11 | repeated bytes nodes = 9; 12 | } 13 | 14 | message tree_node 15 | { 16 | optional int64 id = 1; 17 | repeated bytes keys = 2; 18 | repeated int64 childrens = 3; 19 | repeated bytes values = 4; 20 | optional int32 node_type = 5; 21 | optional int32 is_dirt = 6; 22 | } 23 | 24 | message tree_log 25 | { 26 | optional string action = 1; 27 | optional bytes key = 2; 28 | optional bytes value = 3; 29 | } 30 | -------------------------------------------------------------------------------- /search.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | ) 7 | 8 | // Search return value 9 | func (t *Btree) search(key []byte) ([]byte, error) { 10 | var value []byte 11 | tnode, err := t.getTreeNode(t.GetRoot()) 12 | if err != nil { 13 | return value, err 14 | } 15 | return tnode.searchRecord(key, t) 16 | } 17 | 18 | // node search record 19 | func (n *TreeNode) searchRecord(key []byte, tree *Btree) ([]byte, error) { 20 | var value []byte 21 | index := n.locate(key) 22 | if n.GetNodeType() == isNode { 23 | tnode, err := tree.getTreeNode(n.Childrens[index]) 24 | if err != nil { 25 | return value, err 26 | } 27 | return tnode.searchRecord(key, tree) 28 | } else { 29 | index-- 30 | if index >= 0 { 31 | if bytes.Compare(n.Keys[index], key) == 0 { 32 | return n.Values[index], nil 33 | } 34 | } 35 | } 36 | return value, fmt.Errorf("%s not find", string(key)) 37 | } 38 | -------------------------------------------------------------------------------- /serialize.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bufio" 5 | "github.com/golang/protobuf/proto" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | // Marshal btree to disk 11 | func (t *Btree) Marshal(filename string) error { 12 | fd, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_SYNC, 0644) 13 | if err != nil { 14 | return err 15 | } 16 | defer fd.Close() 17 | fb := bufio.NewWriter(fd) 18 | data, err := proto.Marshal(&t.BtreeMetadata) 19 | if err != nil { 20 | return err 21 | } 22 | _, err = fb.Write(data) 23 | if err != nil { 24 | return err 25 | } 26 | return fb.Flush() 27 | } 28 | 29 | // Unmarshal btree from disk 30 | func Unmarshal(filename string) (*Btree, error) { 31 | tree := &Btree{ 32 | dupnodelist: make(map[int64]int), 33 | opChan: make(chan *treeOperation), 34 | } 35 | fd, err := os.Open(filename) 36 | if err != nil { 37 | return tree, err 38 | } 39 | defer fd.Close() 40 | dataRecord, err := ioutil.ReadAll(fd) 41 | if err != nil { 42 | return tree, err 43 | } 44 | tree.BtreeMetadata = BtreeMetadata{} 45 | proto.Unmarshal(dataRecord, &tree.BtreeMetadata) 46 | go tree.run() 47 | return tree, err 48 | } 49 | -------------------------------------------------------------------------------- /split.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "github.com/golang/protobuf/proto" 5 | ) 6 | 7 | func (n *TreeNode) split(tree *Btree) (key []byte, left, right int64) { 8 | nnode := tree.newTreeNode() 9 | nnode.NodeType = proto.Int32(n.GetNodeType()) 10 | if n.GetNodeType() == isLeaf { 11 | mid := tree.GetLeafMax() / 2 12 | nnode.Values = n.GetValues()[mid:] 13 | nnode.Keys = n.GetKeys()[mid:] 14 | key = nnode.Keys[0] 15 | n.Keys = n.Keys[:mid] 16 | n.Values = n.Values[:mid] 17 | } else { 18 | mid := tree.GetNodeMax() / 2 19 | key = n.Keys[mid] 20 | nnode.Keys = n.GetKeys()[mid+1:] 21 | nnode.Childrens = n.GetChildrens()[mid+1:] 22 | n.Keys = n.Keys[:mid] 23 | n.Childrens = n.Childrens[:mid+1] 24 | } 25 | left = n.GetId() 26 | right = nnode.GetId() 27 | tree.Nodes[nnode.GetId()], _ = proto.Marshal(nnode) 28 | tree.Nodes[n.GetId()], _ = proto.Marshal(n) 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /update.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bytes" 5 | "github.com/golang/protobuf/proto" 6 | ) 7 | 8 | // Update is used to update key/value 9 | func (t *Btree) update(record TreeLog) error { 10 | tnode, err := t.getTreeNode(t.GetRoot()) 11 | if err != nil { 12 | return err 13 | } 14 | clonedNode, err := tnode.updateRecord(record, t) 15 | if err == nil { 16 | t.Root = proto.Int64(clonedNode.GetId()) 17 | } 18 | return err 19 | } 20 | 21 | // update node 22 | func (n *TreeNode) updateRecord(record TreeLog, tree *Btree) (*TreeNode, error) { 23 | index := n.locate(record.Key) 24 | var nnode *TreeNode 25 | var clonedNode *TreeNode 26 | var err error 27 | if n.GetNodeType() == isNode { 28 | tnode, err := tree.getTreeNode(n.Childrens[index]) 29 | if err != nil { 30 | return tnode, err 31 | } 32 | clonedNode, err = tnode.updateRecord(record, tree) 33 | if err == nil { 34 | nnode = n.clone(tree) 35 | nnode.Childrens[index] = clonedNode.GetId() 36 | } 37 | } else { 38 | index-- 39 | if index >= 0 { 40 | if bytes.Compare(n.Keys[index], record.Key) == 0 { 41 | nnode = n.clone(tree) 42 | nnode.Values[index] = record.Value 43 | } 44 | } 45 | } 46 | tree.Nodes[nnode.GetId()], err = proto.Marshal(nnode) 47 | return nnode, err 48 | } 49 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/golang/protobuf/proto" 7 | "sync/atomic" 8 | ) 9 | 10 | type treeOperation struct { 11 | TreeLog 12 | valueChan chan []byte 13 | errChan chan error 14 | } 15 | 16 | // genrate node/leaf id 17 | func (t *Btree) genrateID() int64 { 18 | var id int64 19 | id = -1 20 | for k, _ := range t.dupnodelist { 21 | id = k 22 | delete(t.dupnodelist, k) 23 | break 24 | } 25 | if id == -1 { 26 | if t.GetIndexCursor() >= t.GetSize() { 27 | t.Nodes = append(t.Nodes, make([][]byte, TreeSize)...) 28 | *t.Size += int64(TreeSize) 29 | } 30 | id = t.GetIndexCursor() 31 | *t.IndexCursor++ 32 | } 33 | return id 34 | } 35 | 36 | //alloc new tree node 37 | func (t *Btree) newTreeNode() *TreeNode { 38 | id := t.genrateID() 39 | node := &TreeNode{ 40 | Id: proto.Int64(id), 41 | IsDirt: proto.Int32(0), 42 | } 43 | return node 44 | } 45 | 46 | func (t *Btree) getTreeNode(id int64) (*TreeNode, error) { 47 | var tnode TreeNode 48 | var err error 49 | if len(t.Nodes[id]) > 0 { 50 | err = proto.Unmarshal(t.Nodes[id], &tnode) 51 | } else { 52 | err = fmt.Errorf("no data") 53 | } 54 | return &tnode, err 55 | } 56 | 57 | //locate key's index in a node 58 | func (n *TreeNode) locate(key []byte) int { 59 | i := 0 60 | size := len(n.Keys) 61 | for { 62 | mid := (i + size) / 2 63 | if i == size { 64 | break 65 | } 66 | if bytes.Compare(n.Keys[mid], key) <= 0 { 67 | i = mid + 1 68 | } else { 69 | size = mid 70 | } 71 | } 72 | return i 73 | } 74 | 75 | //clone node 76 | func (n *TreeNode) clone(tree *Btree) *TreeNode { 77 | nnode := tree.newTreeNode() 78 | nnode.Keys = n.GetKeys() 79 | nnode.Childrens = n.GetChildrens() 80 | nnode.Values = n.GetValues() 81 | nnode.NodeType = proto.Int32(n.GetNodeType()) 82 | atomic.StoreInt32(n.IsDirt, 1) 83 | tree.Nodes[n.GetId()], _ = proto.Marshal(n) 84 | return nnode 85 | } 86 | 87 | //gc dupnodelist 88 | func (t *Btree) gc() { 89 | pos := t.gcIndex 90 | for i, n := range t.Nodes[pos:] { 91 | t.gcIndex++ 92 | if t.gcIndex > t.GetIndexCursor() { 93 | t.gcIndex = 0 94 | break 95 | } 96 | if i > 100 { 97 | break 98 | } 99 | var v TreeNode 100 | err := proto.Unmarshal(n, &v) 101 | if err == nil && v.isReleaseAble() { 102 | t.dupnodelist[v.GetId()] = 1 103 | } 104 | } 105 | } 106 | 107 | func (n *TreeNode) isReleaseAble() bool { 108 | if n.IsDirt == nil || atomic.LoadInt32(n.IsDirt) > 0 { 109 | return true 110 | } 111 | return false 112 | } 113 | --------------------------------------------------------------------------------