├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── app ├── app.go ├── app_test.go └── state.go ├── benchmarks ├── README.md ├── bench_test.go ├── results │ ├── Ethans-MBP-2717167.txt │ ├── aws-c4-large-f6f41ca.txt │ └── digital-ocean-2gb-2717167.txt └── setup │ ├── INSTALL_ROOT.sh │ ├── INSTALL_USER.sh │ └── RUN_BENCHMARKS.sh ├── circle.yml ├── client └── client.go ├── cmd ├── app.go ├── dump.go ├── loadtest.go ├── merkleeyes │ └── main.go ├── root.go └── version.go ├── glide.lock ├── glide.yaml ├── iavl ├── PERFORMANCE.md ├── README.md ├── circle.yml ├── iavl_node.go ├── iavl_proof.go ├── iavl_test.go ├── iavl_tree.go ├── iavl_tree_dump.go ├── util.go └── version.go ├── scripts ├── dist.sh ├── dist_build.sh ├── looper.go ├── merkleeyes-builder │ └── Dockerfile └── publish.sh ├── tests └── client_test.go ├── testutil └── testutil.go ├── types └── types.go └── version └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | .glide 3 | 4 | # created in test code 5 | test.db 6 | 7 | # profiling data 8 | *\.test 9 | cpu*.out 10 | mem*.out 11 | cpu*.pdf 12 | mem*.pdf 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.2.2 (June 5, 2017) 4 | 5 | BUG FIXES: 6 | - Actually start the Merkleeyes server 7 | 8 | ## 0.2.1 (June 2, 2017) 9 | 10 | IMPROVEMENTS: 11 | - Add version number to the source code 12 | - Update dependencies 13 | 14 | ## 0.2.0 (May 18, 2017) 15 | 16 | Merge in the IAVL tree from `go-merkle` and update import paths for new `tmlibs` 17 | 18 | ## 0.1.0 (March 6, 2017) 19 | 20 | Initial release 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Tendermint MerkleEyes 2 | Copyright (C) 2015 Tendermint 3 | 4 | 5 | 6 | Apache License 7 | Version 2.0, January 2004 8 | https://www.apache.org/licenses/ 9 | 10 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 11 | 12 | 1. Definitions. 13 | 14 | "License" shall mean the terms and conditions for use, reproduction, 15 | and distribution as defined by Sections 1 through 9 of this document. 16 | 17 | "Licensor" shall mean the copyright owner or entity authorized by 18 | the copyright owner that is granting the License. 19 | 20 | "Legal Entity" shall mean the union of the acting entity and all 21 | other entities that control, are controlled by, or are under common 22 | control with that entity. For the purposes of this definition, 23 | "control" means (i) the power, direct or indirect, to cause the 24 | direction or management of such entity, whether by contract or 25 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 26 | outstanding shares, or (iii) beneficial ownership of such entity. 27 | 28 | "You" (or "Your") shall mean an individual or Legal Entity 29 | exercising permissions granted by this License. 30 | 31 | "Source" form shall mean the preferred form for making modifications, 32 | including but not limited to software source code, documentation 33 | source, and configuration files. 34 | 35 | "Object" form shall mean any form resulting from mechanical 36 | transformation or translation of a Source form, including but 37 | not limited to compiled object code, generated documentation, 38 | and conversions to other media types. 39 | 40 | "Work" shall mean the work of authorship, whether in Source or 41 | Object form, made available under the License, as indicated by a 42 | copyright notice that is included in or attached to the work 43 | (an example is provided in the Appendix below). 44 | 45 | "Derivative Works" shall mean any work, whether in Source or Object 46 | form, that is based on (or derived from) the Work and for which the 47 | editorial revisions, annotations, elaborations, or other modifications 48 | represent, as a whole, an original work of authorship. For the purposes 49 | of this License, Derivative Works shall not include works that remain 50 | separable from, or merely link (or bind by name) to the interfaces of, 51 | the Work and Derivative Works thereof. 52 | 53 | "Contribution" shall mean any work of authorship, including 54 | the original version of the Work and any modifications or additions 55 | to that Work or Derivative Works thereof, that is intentionally 56 | submitted to Licensor for inclusion in the Work by the copyright owner 57 | or by an individual or Legal Entity authorized to submit on behalf of 58 | the copyright owner. For the purposes of this definition, "submitted" 59 | means any form of electronic, verbal, or written communication sent 60 | to the Licensor or its representatives, including but not limited to 61 | communication on electronic mailing lists, source code control systems, 62 | and issue tracking systems that are managed by, or on behalf of, the 63 | Licensor for the purpose of discussing and improving the Work, but 64 | excluding communication that is conspicuously marked or otherwise 65 | designated in writing by the copyright owner as "Not a Contribution." 66 | 67 | "Contributor" shall mean Licensor and any individual or Legal Entity 68 | on behalf of whom a Contribution has been received by Licensor and 69 | subsequently incorporated within the Work. 70 | 71 | 2. Grant of Copyright License. Subject to the terms and conditions of 72 | this License, each Contributor hereby grants to You a perpetual, 73 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 74 | copyright license to reproduce, prepare Derivative Works of, 75 | publicly display, publicly perform, sublicense, and distribute the 76 | Work and such Derivative Works in Source or Object form. 77 | 78 | 3. Grant of Patent License. Subject to the terms and conditions of 79 | this License, each Contributor hereby grants to You a perpetual, 80 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 81 | (except as stated in this section) patent license to make, have made, 82 | use, offer to sell, sell, import, and otherwise transfer the Work, 83 | where such license applies only to those patent claims licensable 84 | by such Contributor that are necessarily infringed by their 85 | Contribution(s) alone or by combination of their Contribution(s) 86 | with the Work to which such Contribution(s) was submitted. If You 87 | institute patent litigation against any entity (including a 88 | cross-claim or counterclaim in a lawsuit) alleging that the Work 89 | or a Contribution incorporated within the Work constitutes direct 90 | or contributory patent infringement, then any patent licenses 91 | granted to You under this License for that Work shall terminate 92 | as of the date such litigation is filed. 93 | 94 | 4. Redistribution. You may reproduce and distribute copies of the 95 | Work or Derivative Works thereof in any medium, with or without 96 | modifications, and in Source or Object form, provided that You 97 | meet the following conditions: 98 | 99 | (a) You must give any other recipients of the Work or 100 | Derivative Works a copy of this License; and 101 | 102 | (b) You must cause any modified files to carry prominent notices 103 | stating that You changed the files; and 104 | 105 | (c) You must retain, in the Source form of any Derivative Works 106 | that You distribute, all copyright, patent, trademark, and 107 | attribution notices from the Source form of the Work, 108 | excluding those notices that do not pertain to any part of 109 | the Derivative Works; and 110 | 111 | (d) If the Work includes a "NOTICE" text file as part of its 112 | distribution, then any Derivative Works that You distribute must 113 | include a readable copy of the attribution notices contained 114 | within such NOTICE file, excluding those notices that do not 115 | pertain to any part of the Derivative Works, in at least one 116 | of the following places: within a NOTICE text file distributed 117 | as part of the Derivative Works; within the Source form or 118 | documentation, if provided along with the Derivative Works; or, 119 | within a display generated by the Derivative Works, if and 120 | wherever such third-party notices normally appear. The contents 121 | of the NOTICE file are for informational purposes only and 122 | do not modify the License. You may add Your own attribution 123 | notices within Derivative Works that You distribute, alongside 124 | or as an addendum to the NOTICE text from the Work, provided 125 | that such additional attribution notices cannot be construed 126 | as modifying the License. 127 | 128 | You may add Your own copyright statement to Your modifications and 129 | may provide additional or different license terms and conditions 130 | for use, reproduction, or distribution of Your modifications, or 131 | for any such Derivative Works as a whole, provided Your use, 132 | reproduction, and distribution of the Work otherwise complies with 133 | the conditions stated in this License. 134 | 135 | 5. Submission of Contributions. Unless You explicitly state otherwise, 136 | any Contribution intentionally submitted for inclusion in the Work 137 | by You to the Licensor shall be under the terms and conditions of 138 | this License, without any additional terms or conditions. 139 | Notwithstanding the above, nothing herein shall supersede or modify 140 | the terms of any separate license agreement you may have executed 141 | with Licensor regarding such Contributions. 142 | 143 | 6. Trademarks. This License does not grant permission to use the trade 144 | names, trademarks, service marks, or product names of the Licensor, 145 | except as required for reasonable and customary use in describing the 146 | origin of the Work and reproducing the content of the NOTICE file. 147 | 148 | 7. Disclaimer of Warranty. Unless required by applicable law or 149 | agreed to in writing, Licensor provides the Work (and each 150 | Contributor provides its Contributions) on an "AS IS" BASIS, 151 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 152 | implied, including, without limitation, any warranties or conditions 153 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 154 | PARTICULAR PURPOSE. You are solely responsible for determining the 155 | appropriateness of using or redistributing the Work and assume any 156 | risks associated with Your exercise of permissions under this License. 157 | 158 | 8. Limitation of Liability. In no event and under no legal theory, 159 | whether in tort (including negligence), contract, or otherwise, 160 | unless required by applicable law (such as deliberate and grossly 161 | negligent acts) or agreed to in writing, shall any Contributor be 162 | liable to You for damages, including any direct, indirect, special, 163 | incidental, or consequential damages of any character arising as a 164 | result of this License or out of the use or inability to use the 165 | Work (including but not limited to damages for loss of goodwill, 166 | work stoppage, computer failure or malfunction, or any and all 167 | other commercial damages or losses), even if such Contributor 168 | has been advised of the possibility of such damages. 169 | 170 | 9. Accepting Warranty or Additional Liability. While redistributing 171 | the Work or Derivative Works thereof, You may choose to offer, 172 | and charge a fee for, acceptance of support, warranty, indemnity, 173 | or other liability obligations and/or rights consistent with this 174 | License. However, in accepting such obligations, You may act only 175 | on Your own behalf and on Your sole responsibility, not on behalf 176 | of any other Contributor, and only if You agree to indemnify, 177 | defend, and hold each Contributor harmless for any liability 178 | incurred by, or claims asserted against, such Contributor by reason 179 | of your accepting any such warranty or additional liability. 180 | 181 | END OF TERMS AND CONDITIONS 182 | 183 | Licensed under the Apache License, Version 2.0 (the "License"); 184 | you may not use this file except in compliance with the License. 185 | You may obtain a copy of the License at 186 | 187 | https://www.apache.org/licenses/LICENSE-2.0 188 | 189 | Unless required by applicable law or agreed to in writing, software 190 | distributed under the License is distributed on an "AS IS" BASIS, 191 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 192 | See the License for the specific language governing permissions and 193 | limitations under the License. 194 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOTOOLS = \ 2 | github.com/mitchellh/gox \ 3 | github.com/Masterminds/glide 4 | 5 | PDFFLAGS=-pdf --nodefraction=0.1 6 | 7 | all: test install 8 | 9 | install: 10 | go install github.com/tendermint/merkleeyes/cmd/... 11 | 12 | dist: 13 | @ bash scripts/dist.sh 14 | @ bash scripts/publish.sh 15 | 16 | test: 17 | go test -v --race `glide novendor` 18 | 19 | get_deps: 20 | go get -d github.com/tendermint/merkleeyes/... 21 | 22 | tools: 23 | go get -u -v $(GOTOOLS) 24 | 25 | get_vendor_deps: 26 | go get github.com/Masterminds/glide 27 | glide install 28 | 29 | # bench is the basic tests that shouldn't crash an aws instance 30 | bench: 31 | cd benchmarks && \ 32 | go test -bench=RandomBytes . && \ 33 | go test -bench=Small . && \ 34 | go test -bench=Medium . && \ 35 | go test -bench=BenchmarkMemKeySizes . 36 | 37 | # fullbench is extra tests needing lots of memory and to run locally 38 | fullbench: 39 | cd benchmarks && \ 40 | go test -bench=RandomBytes . && \ 41 | go test -bench=Small . && \ 42 | go test -bench=Medium . && \ 43 | go test -timeout=30m -bench=Large . && \ 44 | go test -bench=Mem . && \ 45 | go test -timeout=60m -bench=LevelDB . 46 | 47 | 48 | # note that this just profiles the in-memory version, not persistence 49 | profile: 50 | cd benchmarks && \ 51 | go test -bench=Mem -cpuprofile=cpu.out -memprofile=mem.out . && \ 52 | go tool pprof ${PDFFLAGS} benchmarks.test cpu.out > cpu.pdf && \ 53 | go tool pprof --alloc_space ${PDFFLAGS} benchmarks.test mem.out > mem_space.pdf && \ 54 | go tool pprof --alloc_objects ${PDFFLAGS} benchmarks.test mem.out > mem_obj.pdf 55 | 56 | explorecpu: 57 | cd benchmarks && \ 58 | go tool pprof benchmarks.test cpu.out 59 | 60 | exploremem: 61 | cd benchmarks && \ 62 | go tool pprof --alloc_objects benchmarks.test mem.out 63 | 64 | delve: 65 | dlv test ./benchmarks -- -test.bench=. 66 | 67 | .PHONY: all test get_deps install tools dist 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # merkleeyes 2 | 3 | [![CircleCI](https://circleci.com/gh/tendermint/merkleeyes.svg?style=svg)](https://circleci.com/gh/tendermint/merkleeyes) 4 | 5 | A simple [ABCI application](http://github.com/tendermint/abci) serving a [merkle-tree key-value store](http://github.com/tendermint/merkleeyes/iavl) 6 | 7 | # Use 8 | 9 | Merkleeyes allows inserts and removes by key, and queries by key or index. 10 | Inserts and removes happen through the `DeliverTx` message, while queries happen through the `Query` message. 11 | `CheckTx` simply mirrors `DeliverTx`. 12 | 13 | # Formatting 14 | 15 | ## Byte arrays 16 | 17 | Byte-array `B` is serialized to `Encode(B)` as follows: 18 | 19 | ``` 20 | Len(B) := Big-Endian encoded length of B 21 | Encode(B) = Len(Len(B)) | Len(B) | B 22 | ``` 23 | 24 | So if `B = "eric"`, then `Encode(B) = 0x010465726963` 25 | 26 | ## Transactions 27 | 28 | There are four types of transaction, each associated with a type-byte and a list of arguments: 29 | 30 | ``` 31 | Set 0x01 Key, Value 32 | Remove 0x02 Key 33 | Get 0x03 Key 34 | Compare and Set 0x04 Key, Compare Value, Set Value 35 | Validator Set Change 0x05 PubKey, Power (uint64) 36 | Validator Set Read 0x06 37 | Validator Set CAS 0x07 Version (uint64), PubKey, Power (uint64) 38 | ``` 39 | 40 | A transaction consists of a 12-byte random nonce, the type-byte, and the encoded arguments. 41 | 42 | For instance, to insert a key-value pair, you would submit a transaction that looked like `NONCE | 01 | Encode(key) | Encode(value)`, 43 | where `|` denotes concatenation. 44 | Thus, a transaction inserting the key-value pair `(eric, clapton)` would look like: 45 | 46 | ``` 47 | 0xF4FCDC5BF26E227B66A1BA90010104657269630107636c6170746f6e 48 | ``` 49 | 50 | The first 12-bytes, `F4FCDC5BF26E227B66A1BA90`, are the nonce. The next byte, `01`, is the transaction type. 51 | Following that are the encodings of `eric` and `clapton`. 52 | 53 | 54 | Here's a session from the [abci-cli](https://tendermint.com/intro/getting-started/first-abci): 55 | 56 | ``` 57 | # SET ("eric", "clapton") 58 | > deliver_tx 0xF4FCDC5BF26E227B66A1BA90010104657269630107636c6170746f6e 59 | 60 | # GET ("eric") 61 | > deliver_tx 0xB980403FF73E79A3A2D90A1E03010465726963 62 | -> data: clapton 63 | -> data.hex: 636C6170746F6E 64 | 65 | # CAS ("eric", "clapton", "ericson") 66 | > deliver_tx 0x18D892B6D62773E6AA8804CF040104657269630107636C6170746F6E010765726963736f6e 67 | 68 | # GET ("eric") 69 | > deliver_tx 0x4FB9DAB513493E602FF085C603010465726963 70 | -> data: ericson 71 | -> data.hex: 65726963736F6E 72 | 73 | # COMMIT 74 | > commit 75 | -> data: ���Ώ�R�Ng�=HK}��7� 76 | -> data.hex: BAEDE5CE8F9A52B64E67873D484B7DABF69537DA 77 | 78 | # QUERY ("eric") 79 | > query 0x65726963 80 | -> height: 2 81 | -> key: eric 82 | -> key.hex: 65726963 83 | -> value: ericson 84 | -> value.hex: 65726963736F6E 85 | ``` 86 | 87 | 88 | # Poem 89 | 90 | To the tune of Eric Clapton's "My Father's Eyes" 91 | 92 | ``` 93 | writing down, my checksum 94 | waiting for the, data to come 95 | no need to pray for integrity 96 | thats cuz I use, a merkle tree 97 | 98 | grab the root, with a quick hash run 99 | if the hash works out, 100 | it must have been done 101 | 102 | theres no need, for trust to arise 103 | thanks to the crypto 104 | now that I can merkleyes 105 | 106 | take that data, merklize 107 | ye, I merklize ... 108 | 109 | then the truth, begins to shine 110 | the inverse of a hash, you will never find 111 | and as I watch, the dataset grow 112 | producing a proof, is never slow 113 | 114 | Where do I find, the will to hash 115 | How do I teach it? 116 | It doesn't pay in cash 117 | Bitcoin, here, I've realized 118 | Thats what I need now, 119 | cuz real currencies merklize 120 | -EB 121 | ``` 122 | -------------------------------------------------------------------------------- /app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "path" 8 | 9 | abci "github.com/tendermint/abci/types" 10 | "github.com/tendermint/go-crypto" 11 | "github.com/tendermint/go-wire" 12 | "github.com/tendermint/go-wire/data" 13 | "github.com/tendermint/merkleeyes/iavl" 14 | cmn "github.com/tendermint/tmlibs/common" 15 | dbm "github.com/tendermint/tmlibs/db" 16 | "github.com/tendermint/tmlibs/merkle" 17 | ) 18 | 19 | // MerkleEyesApp is a Merkle KV-store served as an ABCI app 20 | type MerkleEyesApp struct { 21 | abci.BaseApplication 22 | 23 | state State 24 | db dbm.DB 25 | height uint64 26 | 27 | validators *ValidatorSetState 28 | 29 | changes []*abci.Validator // NOTE: go-wire encoded because thats what Tendermint needs 30 | } 31 | 32 | // just make sure we really are an application, if the interface 33 | // ever changes in the future 34 | func (app *MerkleEyesApp) assertApplication() abci.Application { 35 | return app 36 | } 37 | 38 | var eyesStateKey = []byte("merkleeyes:state") // Database key for merkle tree save value db values 39 | 40 | // MerkleEyesState contains the latest Merkle root hash and the number of times `Commit` has been called 41 | type MerkleEyesState struct { 42 | Hash []byte 43 | Height uint64 44 | Validators *ValidatorSetState 45 | } 46 | 47 | type ValidatorSetState struct { 48 | Version uint64 `json:"version"` 49 | Validators []*Validator `json:"validators"` // NOTE: non-go-wire encoded for client convenience 50 | } 51 | 52 | type Validator struct { 53 | PubKey data.Bytes `json:"pub_key"` 54 | Power uint64 `json:"power"` 55 | } 56 | 57 | func (vss *ValidatorSetState) Has(v *Validator) bool { 58 | for _, v_ := range vss.Validators { 59 | if bytes.Equal(v_.PubKey, v.PubKey) { 60 | return true 61 | } 62 | } 63 | return false 64 | } 65 | 66 | func (vss *ValidatorSetState) Remove(v *Validator) { 67 | vals := make([]*Validator, 0, len(vss.Validators)-1) 68 | for _, v_ := range vss.Validators { 69 | if !bytes.Equal(v_.PubKey, v.PubKey) { 70 | vals = append(vals, v_) 71 | } 72 | } 73 | vss.Validators = vals 74 | } 75 | 76 | func (vss *ValidatorSetState) Set(v *Validator) { 77 | for i, v_ := range vss.Validators { 78 | if bytes.Equal(v_.PubKey, v.PubKey) { 79 | vss.Validators[i] = v 80 | return 81 | } 82 | } 83 | vss.Validators = append(vss.Validators, v) 84 | } 85 | 86 | // Transaction type bytes 87 | const ( 88 | TxTypeSet byte = 0x01 89 | TxTypeRm byte = 0x02 90 | TxTypeGet byte = 0x03 91 | TxTypeCompareAndSet byte = 0x04 92 | TxTypeValSetChange byte = 0x05 93 | TxTypeValSetRead byte = 0x06 94 | TxTypeValSetCAS byte = 0x07 95 | 96 | NonceLength = 12 97 | ) 98 | 99 | // NewMerkleEyesApp initializes the database, loads any existing state, and returns a new MerkleEyesApp 100 | func NewMerkleEyesApp(dbName string, cacheSize int) *MerkleEyesApp { 101 | // start at 1 so the height returned by query is for the 102 | // next block, ie. the one that includes the AppHash for our current state 103 | initialHeight := uint64(1) 104 | 105 | // Non-persistent case 106 | if dbName == "" { 107 | tree := iavl.NewIAVLTree( 108 | 0, 109 | nil, 110 | ) 111 | return &MerkleEyesApp{ 112 | state: NewState(tree, false), 113 | db: nil, 114 | height: initialHeight, 115 | } 116 | } 117 | 118 | // Setup the persistent merkle tree 119 | empty, _ := cmn.IsDirEmpty(path.Join(dbName, dbName+".db")) 120 | 121 | // Open the db, if the db doesn't exist it will be created 122 | db := dbm.NewDB(dbName, dbm.LevelDBBackendStr, dbName) 123 | 124 | // Load Tree 125 | tree := iavl.NewIAVLTree(cacheSize, db) 126 | 127 | if empty { 128 | fmt.Println("no existing db, creating new db") 129 | db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{ 130 | Hash: tree.Save(), 131 | Height: initialHeight, 132 | })) 133 | } else { 134 | fmt.Println("loading existing db") 135 | } 136 | 137 | // Load merkle state 138 | eyesStateBytes := db.Get(eyesStateKey) 139 | var eyesState MerkleEyesState 140 | err := wire.ReadBinaryBytes(eyesStateBytes, &eyesState) 141 | if err != nil { 142 | fmt.Println("error reading MerkleEyesState") 143 | panic(err.Error()) 144 | } 145 | tree.Load(eyesState.Hash) 146 | 147 | return &MerkleEyesApp{ 148 | state: NewState(tree, true), 149 | db: db, 150 | height: eyesState.Height, 151 | validators: new(ValidatorSetState), 152 | } 153 | } 154 | 155 | // CloseDB closes the database 156 | func (app *MerkleEyesApp) CloseDB() { 157 | if app.db != nil { 158 | app.db.Close() 159 | } 160 | } 161 | 162 | // Info implements abci.Application 163 | func (app *MerkleEyesApp) Info() abci.ResponseInfo { 164 | return abci.ResponseInfo{Data: "merkleeyes"} 165 | } 166 | 167 | // SetOption implements abci.Application 168 | func (app *MerkleEyesApp) SetOption(key string, value string) (log string) { 169 | return "No options are supported yet" 170 | } 171 | 172 | // DeliverTx implements abci.Application 173 | func (app *MerkleEyesApp) DeliverTx(tx []byte) abci.Result { 174 | tree := app.state.Deliver() 175 | r := app.doTx(tree, tx) 176 | if r.IsErr() { 177 | fmt.Println("DeliverTx Err", r) 178 | } 179 | return r 180 | } 181 | 182 | // CheckTx implements abci.Application 183 | func (app *MerkleEyesApp) CheckTx(tx []byte) abci.Result { 184 | return abci.OK 185 | // tree := app.state.Check() 186 | // return app.doTx(tree, tx) 187 | } 188 | 189 | func nonceKey(nonce []byte) []byte { 190 | return append([]byte("/nonce/"), nonce...) 191 | } 192 | 193 | func storeKey(key []byte) []byte { 194 | return append([]byte("/key/"), key...) 195 | } 196 | 197 | func (app *MerkleEyesApp) doTx(tree merkle.Tree, tx []byte) abci.Result { 198 | // minimum length is 12 (nonce) + 1 (type byte) = 13 199 | minTxLen := NonceLength + 1 200 | if len(tx) < minTxLen { 201 | return abci.ErrEncodingError.SetLog(fmt.Sprintf("Tx length must be at least %d", minTxLen)) 202 | } 203 | 204 | nonce := tx[:12] 205 | tx = tx[12:] 206 | 207 | // check nonce 208 | _, _, exists := tree.Get(nonceKey(nonce)) 209 | if exists { 210 | return abci.ErrBadNonce.AppendLog(fmt.Sprintf("Nonce %X already exists", nonce)) 211 | } 212 | 213 | // set nonce 214 | tree.Set(nonceKey(nonce), []byte("found")) 215 | 216 | typeByte := tx[0] 217 | tx = tx[1:] 218 | switch typeByte { 219 | case TxTypeSet: // Set 220 | key, n, err := wire.GetByteSlice(tx) 221 | if err != nil { 222 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error())) 223 | } 224 | tx = tx[n:] 225 | value, n, err := wire.GetByteSlice(tx) 226 | if err != nil { 227 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading value: %v", err.Error())) 228 | } 229 | tx = tx[n:] 230 | if len(tx) != 0 { 231 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over")) 232 | } 233 | 234 | tree.Set(storeKey(key), value) 235 | 236 | fmt.Println("SET", cmn.Fmt("%X", key), cmn.Fmt("%X", value)) 237 | case TxTypeRm: // Remove 238 | key, n, err := wire.GetByteSlice(tx) 239 | if err != nil { 240 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error())) 241 | } 242 | tx = tx[n:] 243 | if len(tx) != 0 { 244 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over")) 245 | } 246 | tree.Remove(storeKey(key)) 247 | fmt.Println("RM", cmn.Fmt("%X", key)) 248 | case TxTypeGet: // Get 249 | key, n, err := wire.GetByteSlice(tx) 250 | if err != nil { 251 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error())) 252 | } 253 | tx = tx[n:] 254 | if len(tx) != 0 { 255 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over")) 256 | } 257 | 258 | _, value, exists := tree.Get(storeKey(key)) 259 | if exists { 260 | fmt.Println("GET", cmn.Fmt("%X", key), cmn.Fmt("%X", value)) 261 | return abci.OK.SetData(value) 262 | } else { 263 | return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key)) 264 | } 265 | case TxTypeCompareAndSet: // Compare and Set 266 | key, n, err := wire.GetByteSlice(tx) 267 | if err != nil { 268 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading key: %v", err.Error())) 269 | } 270 | tx = tx[n:] 271 | 272 | compareValue, n, err := wire.GetByteSlice(tx) 273 | if err != nil { 274 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading compare value: %v", err.Error())) 275 | } 276 | tx = tx[n:] 277 | 278 | setValue, n, err := wire.GetByteSlice(tx) 279 | if err != nil { 280 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading set value: %v", err.Error())) 281 | } 282 | tx = tx[n:] 283 | 284 | if len(tx) != 0 { 285 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Got bytes left over")) 286 | } 287 | 288 | _, value, exists := tree.Get(storeKey(key)) 289 | if !exists { 290 | return abci.ErrBaseUnknownAddress.AppendLog(fmt.Sprintf("Cannot find key: %X", key)) 291 | } 292 | if !bytes.Equal(value, compareValue) { 293 | return abci.ErrUnauthorized.AppendLog(fmt.Sprintf("Value was %X, not %X", value, compareValue)) 294 | } 295 | tree.Set(storeKey(key), setValue) 296 | 297 | fmt.Println("CAS-SET", cmn.Fmt("%X", key), cmn.Fmt("%X", compareValue), cmn.Fmt("%X", setValue)) 298 | case TxTypeValSetChange: 299 | pubKey, n, err := wire.GetByteSlice(tx) 300 | if err != nil { 301 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading pubkey: %v", err.Error())) 302 | } 303 | if len(pubKey) != 32 { 304 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Pubkey must be 32 bytes: %X is %d bytes", pubKey, len(pubKey))) 305 | } 306 | tx = tx[n:] 307 | if len(tx) != 8 { 308 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Power must be 8 bytes: %X is %d bytes", tx, len(tx))) 309 | } 310 | power := wire.GetUint64(tx) 311 | 312 | return app.updateValidator(pubKey, power) 313 | 314 | case TxTypeValSetRead: 315 | b, err := json.Marshal(app.validators) 316 | if err != nil { 317 | return abci.ErrInternalError.SetLog(cmn.Fmt("Error marshalling validator info: %v", err)) 318 | } 319 | return abci.OK.SetData(b).SetLog(string(b)) 320 | 321 | case TxTypeValSetCAS: 322 | if len(tx) < 8 { 323 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Version number must be 8 bytes: remaining tx (%X) is %d bytes", tx, len(tx))) 324 | } 325 | version := wire.GetUint64(tx) 326 | if app.validators.Version != version { 327 | return abci.ErrUnauthorized.AppendLog(fmt.Sprintf("Version was %d, not %d", app.validators.Version, version)) 328 | } 329 | tx = tx[8:] 330 | 331 | pubKey, n, err := wire.GetByteSlice(tx) 332 | if err != nil { 333 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Error reading pubkey: %v", err.Error())) 334 | } 335 | if len(pubKey) != 32 { 336 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Pubkey must be 32 bytes: %X is %d bytes", pubKey, len(pubKey))) 337 | } 338 | tx = tx[n:] 339 | if len(tx) != 8 { 340 | return abci.ErrEncodingError.SetLog(cmn.Fmt("Power must be 8 bytes: %X is %d bytes", tx, len(tx))) 341 | } 342 | power := wire.GetUint64(tx) 343 | 344 | return app.updateValidator(pubKey, power) 345 | 346 | default: 347 | return abci.ErrUnknownRequest.SetLog(cmn.Fmt("Unexpected Tx type byte %X", typeByte)) 348 | } 349 | return abci.OK 350 | } 351 | 352 | func (app *MerkleEyesApp) updateValidator(pubKey []byte, power uint64) abci.Result { 353 | v := &Validator{pubKey, power} 354 | if v.Power == 0 { 355 | // remove validator 356 | if !app.validators.Has(v) { 357 | return abci.ErrUnauthorized.SetLog(cmn.Fmt("Cannot remove non-existent validator %v", v)) 358 | } 359 | app.validators.Remove(v) 360 | } else { 361 | // add or update validator 362 | app.validators.Set(v) 363 | } 364 | 365 | // copy to PubKeyEd25519 so we can go-wire encode properly for the changes array 366 | var pubKeyEd crypto.PubKeyEd25519 367 | copy(pubKeyEd[:], pubKey) 368 | app.changes = append(app.changes, &abci.Validator{pubKeyEd.Bytes(), power}) 369 | 370 | return abci.OK 371 | } 372 | 373 | func (app *MerkleEyesApp) InitChain(validators []*abci.Validator) { 374 | for _, v := range validators { 375 | // want non-go-wire encoded for the state 376 | p, _ := crypto.PubKeyFromBytes(v.PubKey) 377 | pubKey := p.Unwrap().(crypto.PubKeyEd25519) 378 | app.validators.Set(&Validator{pubKey[:], v.Power}) 379 | } 380 | } 381 | 382 | func (app *MerkleEyesApp) BeginBlock(hash []byte, header *abci.Header) { 383 | // reset valset changes 384 | app.changes = make([]*abci.Validator, 0) 385 | } 386 | 387 | func (app *MerkleEyesApp) EndBlock(height uint64) (resEndBlock abci.ResponseEndBlock) { 388 | if len(app.changes) > 0 { 389 | app.validators.Version++ 390 | } 391 | return abci.ResponseEndBlock{Diffs: app.changes} 392 | } 393 | 394 | // Commit implements abci.Application 395 | func (app *MerkleEyesApp) Commit() abci.Result { 396 | 397 | hash := app.state.Commit() 398 | 399 | app.height++ 400 | if app.db != nil { 401 | app.db.Set(eyesStateKey, wire.BinaryBytes(MerkleEyesState{ 402 | Hash: hash, 403 | Height: app.height, 404 | Validators: app.validators, 405 | })) 406 | } 407 | 408 | if app.state.Committed().Size() == 0 { 409 | return abci.NewResultOK(nil, "Empty hash for empty tree") 410 | } 411 | return abci.NewResultOK(hash, "") 412 | } 413 | 414 | // Query implements abci.Application 415 | func (app *MerkleEyesApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) { 416 | if len(reqQuery.Data) == 0 { 417 | return 418 | } 419 | tree := app.state.Committed() 420 | 421 | if reqQuery.Height != 0 { 422 | // TODO: support older commits 423 | resQuery.Code = abci.CodeType_InternalError 424 | resQuery.Log = "merkleeyes only supports queries on latest commit" 425 | return 426 | } 427 | 428 | // set the query response height 429 | resQuery.Height = app.height 430 | 431 | switch reqQuery.Path { 432 | case "/store", "/key": // Get by key 433 | key := reqQuery.Data // Data holds the key bytes 434 | resQuery.Key = key 435 | if reqQuery.Prove { 436 | value, proof, exists := tree.Proof(storeKey(key)) 437 | if !exists { 438 | resQuery.Log = "Key not found" 439 | } 440 | resQuery.Value = value 441 | resQuery.Proof = proof 442 | // TODO: return index too? 443 | } else { 444 | index, value, _ := tree.Get(storeKey(key)) 445 | resQuery.Value = value 446 | resQuery.Index = int64(index) 447 | } 448 | 449 | case "/index": // Get by Index 450 | index := wire.GetInt64(reqQuery.Data) 451 | key, value := tree.GetByIndex(int(index)) 452 | resQuery.Key = key 453 | resQuery.Index = int64(index) 454 | resQuery.Value = value 455 | 456 | case "/size": // Get size 457 | size := tree.Size() 458 | sizeBytes := wire.BinaryBytes(size) 459 | resQuery.Value = sizeBytes 460 | 461 | default: 462 | resQuery.Code = abci.CodeType_UnknownRequest 463 | resQuery.Log = cmn.Fmt("Unexpected Query path: %v", reqQuery.Path) 464 | } 465 | return 466 | } 467 | -------------------------------------------------------------------------------- /app/app_test.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | abci "github.com/tendermint/abci/types" 8 | wire "github.com/tendermint/go-wire" 9 | "github.com/tendermint/merkleeyes/iavl" 10 | cmn "github.com/tendermint/tmlibs/common" 11 | ) 12 | 13 | func makeTx(typ byte, args ...[]byte) []byte { 14 | n := 12 + 1 15 | for _, arg := range args { 16 | n += wire.ByteSliceSize(arg) 17 | } 18 | tx := make([]byte, n) 19 | buf := tx 20 | 21 | // nonce 22 | copy(buf[:12], cmn.RandBytes(12)) 23 | buf = buf[12:] 24 | 25 | // type byte 26 | buf[0] = typ 27 | n = 1 28 | var err error 29 | for _, arg := range args { 30 | buf = buf[n:] 31 | n, err = wire.PutByteSlice(buf, arg) 32 | if err != nil { 33 | panic(err) 34 | } 35 | } 36 | return tx 37 | } 38 | 39 | func makeSet(key, value []byte) []byte { 40 | return makeTx(TxTypeSet, key, value) 41 | } 42 | 43 | func makeRemove(key []byte) []byte { 44 | return makeTx(TxTypeRm, key) 45 | } 46 | 47 | func makeGet(key []byte) []byte { 48 | return makeTx(TxTypeGet, key) 49 | } 50 | 51 | func makeQuery(key []byte, prove bool, height uint64) (reqQuery abci.RequestQuery) { 52 | reqQuery.Path = "/key" 53 | reqQuery.Data = key 54 | reqQuery.Prove = prove 55 | reqQuery.Height = height 56 | return 57 | } 58 | 59 | func TestAppQueries(t *testing.T) { 60 | assert := assert.New(t) 61 | 62 | app := NewMerkleEyesApp("", 0) 63 | com := app.Commit() 64 | assert.EqualValues([]byte(nil), com.Data) 65 | 66 | // prepare some actions 67 | key, value := []byte("foobar"), []byte("works!") 68 | addTx := makeSet(key, value) 69 | removeTx := makeRemove(key) 70 | 71 | // need to commit append before it shows in queries 72 | txResult := app.DeliverTx(addTx) 73 | assert.True(txResult.IsOK(), txResult.Log) 74 | resQuery := app.Query(makeQuery(key, false, 0)) 75 | assert.True(resQuery.Code.IsOK(), resQuery.Log) 76 | assert.Equal([]byte(nil), resQuery.Value) 77 | // but get works before commit 78 | txResult = app.DeliverTx(makeGet(key)) 79 | assert.True(txResult.IsOK(), txResult.Log) 80 | assert.EqualValues(txResult.Data, value) 81 | 82 | com = app.Commit() 83 | hash := com.Data 84 | assert.NotEqual(t, nil, hash) 85 | resQuery = app.Query(makeQuery(key, false, 0)) 86 | assert.True(resQuery.Code.IsOK(), resQuery.Log) 87 | assert.Equal(value, resQuery.Value) 88 | txResult = app.DeliverTx(makeGet(key)) 89 | assert.EqualValues(txResult.Data, value, txResult.Error()) 90 | 91 | com = app.Commit() 92 | hash = com.Data 93 | // modifying check has no effect 94 | check := app.CheckTx(removeTx) 95 | assert.True(check.IsOK(), check.Log) 96 | com = app.Commit() 97 | assert.True(com.IsOK(), com.Log) 98 | hash2 := com.Data 99 | assert.Equal(hash, hash2) 100 | 101 | // proofs come from the last commited state, not working state 102 | txResult = app.DeliverTx(removeTx) 103 | assert.True(txResult.IsOK(), txResult.Log) 104 | // currently don't support specifying block height 105 | resQuery = app.Query(makeQuery(key, true, 1)) 106 | assert.False(resQuery.Code.IsOK(), resQuery.Log) 107 | resQuery = app.Query(makeQuery(key, true, 0)) 108 | if assert.NotEmpty(resQuery.Value) { 109 | proof, err := iavl.ReadProof(resQuery.Proof) 110 | if assert.Nil(err) { 111 | assert.True(proof.Verify(storeKey(key), resQuery.Value, proof.RootHash)) 112 | } 113 | } 114 | 115 | // commit remove actually removes it now 116 | com = app.Commit() 117 | assert.True(com.IsOK(), com.Log) 118 | hash3 := com.Data 119 | assert.NotEqual(hash, hash3) 120 | 121 | // nothing here... 122 | resQuery = app.Query(makeQuery(key, false, 0)) 123 | assert.True(resQuery.Code.IsOK(), resQuery.Log) 124 | assert.Equal([]byte(nil), resQuery.Value) 125 | // neither with proof... 126 | resQuery = app.Query(makeQuery(key, true, 0)) 127 | assert.True(resQuery.Code.IsOK(), resQuery.Log) 128 | assert.Equal([]byte(nil), resQuery.Value) 129 | assert.Empty(resQuery.Proof) 130 | // nor with get 131 | txResult = app.DeliverTx(makeGet(key)) 132 | assert.True(txResult.IsSameCode(abci.ErrBaseUnknownAddress), txResult.Log) 133 | } 134 | -------------------------------------------------------------------------------- /app/state.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import "github.com/tendermint/tmlibs/merkle" 4 | 5 | // State represents the app states, separating the commited state (for queries) 6 | // from the working state (for CheckTx and AppendTx) 7 | type State struct { 8 | committed merkle.Tree 9 | deliverTx merkle.Tree 10 | checkTx merkle.Tree 11 | persistent bool 12 | } 13 | 14 | func NewState(tree merkle.Tree, persistent bool) State { 15 | return State{ 16 | committed: tree, 17 | deliverTx: tree.Copy(), 18 | checkTx: tree.Copy(), 19 | persistent: persistent, 20 | } 21 | } 22 | 23 | func (s State) Committed() merkle.Tree { 24 | return s.committed 25 | } 26 | 27 | func (s State) Deliver() merkle.Tree { 28 | return s.deliverTx 29 | } 30 | 31 | func (s State) Check() merkle.Tree { 32 | return s.checkTx 33 | } 34 | 35 | // Commit stores the current Deliver() state as committed 36 | // starts new Append/Check state, and 37 | // returns the hash for the commit 38 | func (s *State) Commit() []byte { 39 | var hash []byte 40 | if s.persistent { 41 | hash = s.deliverTx.Save() 42 | } else { 43 | hash = s.deliverTx.Hash() 44 | } 45 | 46 | s.committed = s.deliverTx 47 | s.deliverTx = s.committed.Copy() 48 | s.checkTx = s.committed.Copy() 49 | return hash 50 | } 51 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # Running Benchmarks 2 | 3 | These instructions are mainly for running the benchmarks on an cloud instance that is intended to be thrown away, not on a dev machine. Be careful with the install scripts locally. 4 | 5 | This has only been tested on Ubuntu 16.04. It *should* work on Ubuntu 14.04 as well. It *may* work on Debian, but has never been tested. 6 | 7 | 8 | ## Setting up the machine 9 | 10 | Put the files on the machine and login (all code assumes you are in this directory locally) 11 | 12 | ``` 13 | scp -r setup user@host: 14 | ssh user@host 15 | ``` 16 | 17 | Run the install scripts (once per machine) 18 | 19 | ``` 20 | cd setup 21 | chmod +x * 22 | sudo ./INSTALL_ROOT.sh 23 | ./INSTALL_USER.sh 24 | ``` 25 | 26 | ## Running the tests 27 | 28 | Make sure the hostname is set to a good value for recording: 29 | 30 | ``` 31 | hostname -s 32 | sudo hostname 33 | ``` 34 | 35 | Run the benchmarks in a screen: 36 | 37 | ``` 38 | screen 39 | ~/RUN_BENCHMARKS.sh 40 | ``` 41 | 42 | Copy them back from your local machine: 43 | 44 | ``` 45 | scp user@host:go/src/github.com/tendermint/merkleeyes/iavl/benchmarks/results/* results 46 | git add results 47 | ``` 48 | -------------------------------------------------------------------------------- /benchmarks/bench_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "runtime" 8 | "testing" 9 | 10 | "github.com/tendermint/merkleeyes/iavl" 11 | db "github.com/tendermint/tmlibs/db" 12 | "github.com/tendermint/tmlibs/merkle" 13 | ) 14 | 15 | func randBytes(length int) []byte { 16 | key := make([]byte, length) 17 | // math.rand.Read always returns err=nil 18 | rand.Read(key) 19 | return key 20 | } 21 | 22 | func prepareTree(db db.DB, size, keyLen, dataLen int) (merkle.Tree, [][]byte) { 23 | t := iavl.NewIAVLTree(size, db) 24 | keys := make([][]byte, size) 25 | 26 | for i := 0; i < size; i++ { 27 | key := randBytes(keyLen) 28 | t.Set(key, randBytes(dataLen)) 29 | keys[i] = key 30 | } 31 | t.Hash() 32 | t.Save() 33 | runtime.GC() 34 | return t, keys 35 | } 36 | 37 | func runQueries(b *testing.B, t merkle.Tree, keyLen int) { 38 | for i := 0; i < b.N; i++ { 39 | q := randBytes(keyLen) 40 | t.Get(q) 41 | } 42 | } 43 | 44 | func runKnownQueries(b *testing.B, t merkle.Tree, keys [][]byte) { 45 | l := int32(len(keys)) 46 | for i := 0; i < b.N; i++ { 47 | q := keys[rand.Int31n(l)] 48 | t.Get(q) 49 | } 50 | } 51 | 52 | func runInsert(b *testing.B, t merkle.Tree, keyLen, dataLen, blockSize int) merkle.Tree { 53 | for i := 1; i <= b.N; i++ { 54 | t.Set(randBytes(keyLen), randBytes(dataLen)) 55 | if i%blockSize == 0 { 56 | t.Hash() 57 | t.Save() 58 | } 59 | } 60 | return t 61 | } 62 | 63 | func runUpdate(b *testing.B, t merkle.Tree, dataLen, blockSize int, keys [][]byte) merkle.Tree { 64 | l := int32(len(keys)) 65 | for i := 1; i <= b.N; i++ { 66 | key := keys[rand.Int31n(l)] 67 | t.Set(key, randBytes(dataLen)) 68 | if i%blockSize == 0 { 69 | t.Hash() 70 | t.Save() 71 | } 72 | } 73 | return t 74 | } 75 | 76 | func runDelete(b *testing.B, t merkle.Tree, blockSize int, keys [][]byte) merkle.Tree { 77 | var key []byte 78 | l := int32(len(keys)) 79 | for i := 1; i <= b.N; i++ { 80 | key = keys[rand.Int31n(l)] 81 | // key = randBytes(16) 82 | // TODO: test if removed, use more keys (from insert) 83 | t.Remove(key) 84 | if i%blockSize == 0 { 85 | t.Hash() 86 | t.Save() 87 | } 88 | } 89 | return t 90 | } 91 | 92 | // runBlock measures time for an entire block, not just one tx 93 | func runBlock(b *testing.B, t merkle.Tree, keyLen, dataLen, blockSize int, keys [][]byte) merkle.Tree { 94 | l := int32(len(keys)) 95 | 96 | lastCommit := t 97 | real := t.Copy() 98 | check := t.Copy() 99 | 100 | for i := 0; i < b.N; i++ { 101 | for j := 0; j < blockSize; j++ { 102 | // 50% insert, 50% update 103 | var key []byte 104 | if i%2 == 0 { 105 | key = keys[rand.Int31n(l)] 106 | } else { 107 | key = randBytes(keyLen) 108 | } 109 | data := randBytes(dataLen) 110 | 111 | // perform query and write on check and then real 112 | check.Get(key) 113 | check.Set(key, data) 114 | real.Get(key) 115 | real.Set(key, data) 116 | } 117 | 118 | // at the end of a block, move it all along.... 119 | real.Hash() 120 | real.Save() 121 | lastCommit = real 122 | real = lastCommit.Copy() 123 | check = lastCommit.Copy() 124 | } 125 | 126 | return lastCommit 127 | } 128 | 129 | func BenchmarkRandomBytes(b *testing.B) { 130 | benchmarks := []struct { 131 | length int 132 | }{ 133 | {4}, {16}, {32}, {100}, {1000}, 134 | } 135 | for _, bench := range benchmarks { 136 | name := fmt.Sprintf("random-%d", bench.length) 137 | b.Run(name, func(b *testing.B) { 138 | for i := 0; i < b.N; i++ { 139 | randBytes(bench.length) 140 | } 141 | runtime.GC() 142 | }) 143 | } 144 | } 145 | 146 | type benchmark struct { 147 | dbType string 148 | initSize, blockSize int 149 | keyLen, dataLen int 150 | } 151 | 152 | func BenchmarkMedium(b *testing.B) { 153 | benchmarks := []benchmark{ 154 | {"nodb", 100000, 100, 16, 40}, 155 | {"memdb", 100000, 100, 16, 40}, 156 | {"goleveldb", 100000, 100, 16, 40}, 157 | // FIXME: this crashes on init! Either remove support, or make it work. 158 | // {"cleveldb", 100000, 100, 16, 40}, 159 | {"leveldb", 100000, 100, 16, 40}, 160 | } 161 | runBenchmarks(b, benchmarks) 162 | } 163 | 164 | func BenchmarkSmall(b *testing.B) { 165 | benchmarks := []benchmark{ 166 | {"nodb", 1000, 100, 4, 10}, 167 | {"memdb", 1000, 100, 4, 10}, 168 | {"goleveldb", 1000, 100, 4, 10}, 169 | // FIXME: this crashes on init! Either remove support, or make it work. 170 | // {"cleveldb", 100000, 100, 16, 40}, 171 | {"leveldb", 1000, 100, 4, 10}, 172 | } 173 | runBenchmarks(b, benchmarks) 174 | } 175 | 176 | func BenchmarkLarge(b *testing.B) { 177 | benchmarks := []benchmark{ 178 | {"nodb", 1000000, 100, 16, 40}, 179 | {"memdb", 1000000, 100, 16, 40}, 180 | {"goleveldb", 1000000, 100, 16, 40}, 181 | // FIXME: this crashes on init! Either remove support, or make it work. 182 | // {"cleveldb", 100000, 100, 16, 40}, 183 | {"leveldb", 1000000, 100, 16, 40}, 184 | } 185 | runBenchmarks(b, benchmarks) 186 | } 187 | 188 | func BenchmarkMemInitSizes(b *testing.B) { 189 | benchmarks := []benchmark{ 190 | {"nodb", 10000, 100, 16, 40}, 191 | {"nodb", 70000, 100, 16, 40}, 192 | {"nodb", 500000, 100, 16, 40}, 193 | // This uses something like 1.5-2GB RAM 194 | {"nodb", 3500000, 100, 16, 40}, 195 | // This requires something like 5GB RAM and may crash on some AWS instances 196 | {"nodb", 10000000, 100, 16, 40}, 197 | } 198 | runBenchmarks(b, benchmarks) 199 | } 200 | 201 | func BenchmarkMemKeySizes(b *testing.B) { 202 | benchmarks := []benchmark{ 203 | {"nodb", 100000, 100, 4, 80}, 204 | {"nodb", 100000, 100, 16, 80}, 205 | {"nodb", 100000, 100, 32, 80}, 206 | {"nodb", 100000, 100, 64, 80}, 207 | {"nodb", 100000, 100, 128, 80}, 208 | {"nodb", 100000, 100, 256, 80}, 209 | } 210 | runBenchmarks(b, benchmarks) 211 | } 212 | 213 | func BenchmarkLevelDBBatchSizes(b *testing.B) { 214 | benchmarks := []benchmark{ 215 | {"goleveldb", 100000, 5, 16, 40}, 216 | {"goleveldb", 100000, 25, 16, 40}, 217 | {"goleveldb", 100000, 100, 16, 40}, 218 | {"goleveldb", 100000, 400, 16, 40}, 219 | {"goleveldb", 100000, 2000, 16, 40}, 220 | } 221 | runBenchmarks(b, benchmarks) 222 | } 223 | 224 | // BenchmarkLevelDBLargeData is intended to push disk limits 225 | // in the leveldb, to make sure not everything is cached 226 | func BenchmarkLevelDBLargeData(b *testing.B) { 227 | benchmarks := []benchmark{ 228 | {"goleveldb", 50000, 100, 32, 100}, 229 | {"goleveldb", 50000, 100, 32, 1000}, 230 | {"goleveldb", 50000, 100, 32, 10000}, 231 | {"goleveldb", 50000, 100, 32, 100000}, 232 | } 233 | runBenchmarks(b, benchmarks) 234 | } 235 | 236 | func runBenchmarks(b *testing.B, benchmarks []benchmark) { 237 | for _, bb := range benchmarks { 238 | prefix := fmt.Sprintf("%s-%d-%d-%d-%d", bb.dbType, bb.initSize, 239 | bb.blockSize, bb.keyLen, bb.dataLen) 240 | 241 | // prepare a dir for the db and cleanup afterwards 242 | dirName := fmt.Sprintf("./%s-db", prefix) 243 | defer func() { 244 | err := os.RemoveAll(dirName) 245 | if err != nil { 246 | fmt.Printf("%+v\n", err) 247 | } 248 | }() 249 | 250 | // note that "" leads to nil backing db! 251 | var d db.DB 252 | if bb.dbType != "nodb" { 253 | d = db.NewDB("test", bb.dbType, dirName) 254 | defer d.Close() 255 | } 256 | b.Run(prefix, func(sub *testing.B) { 257 | runSuite(sub, d, bb.initSize, bb.blockSize, bb.keyLen, bb.dataLen) 258 | }) 259 | } 260 | } 261 | 262 | // returns number of MB in use 263 | func memUseMB() float64 { 264 | var mem runtime.MemStats 265 | runtime.ReadMemStats(&mem) 266 | asize := mem.Alloc 267 | mb := float64(asize) / 1000000 268 | return mb 269 | } 270 | 271 | func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) { 272 | // measure mem usage 273 | runtime.GC() 274 | init := memUseMB() 275 | 276 | t, keys := prepareTree(d, initSize, keyLen, dataLen) 277 | used := memUseMB() - init 278 | fmt.Printf("Init Tree took %0.2f MB\n", used) 279 | 280 | b.ResetTimer() 281 | 282 | b.Run("query-miss", func(sub *testing.B) { 283 | runQueries(sub, t, keyLen) 284 | }) 285 | b.Run("query-hits", func(sub *testing.B) { 286 | runKnownQueries(sub, t, keys) 287 | }) 288 | b.Run("update", func(sub *testing.B) { 289 | t = runUpdate(sub, t, dataLen, blockSize, keys) 290 | }) 291 | b.Run("block", func(sub *testing.B) { 292 | t = runBlock(sub, t, keyLen, dataLen, blockSize, keys) 293 | }) 294 | 295 | // both of these edit size of the tree too much 296 | // need to run with their own tree 297 | t = nil // for gc 298 | b.Run("insert", func(sub *testing.B) { 299 | it, _ := prepareTree(d, initSize, keyLen, dataLen) 300 | sub.ResetTimer() 301 | runInsert(sub, it, keyLen, dataLen, blockSize) 302 | }) 303 | b.Run("delete", func(sub *testing.B) { 304 | dt, dkeys := prepareTree(d, initSize+sub.N, keyLen, dataLen) 305 | sub.ResetTimer() 306 | runDelete(sub, dt, blockSize, dkeys) 307 | }) 308 | } 309 | -------------------------------------------------------------------------------- /benchmarks/results/Ethans-MBP-2717167.txt: -------------------------------------------------------------------------------- 1 | cd benchmarks && \ 2 | go test -bench=RandomBytes . && \ 3 | go test -bench=Small . && \ 4 | go test -bench=Medium . && \ 5 | go test -bench=BenchmarkMemKeySizes . 6 | BenchmarkRandomBytes/random-4-4 20000000 63.1 ns/op 7 | BenchmarkRandomBytes/random-16-4 20000000 88.9 ns/op 8 | BenchmarkRandomBytes/random-32-4 10000000 123 ns/op 9 | BenchmarkRandomBytes/random-100-4 5000000 275 ns/op 10 | BenchmarkRandomBytes/random-1000-4 1000000 2131 ns/op 11 | PASS 12 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 8.390s 13 | Init Tree took 0.42 MB 14 | BenchmarkSmall/nodb-1000-100-4-10/query-miss-4 5000000 345 ns/op 15 | BenchmarkSmall/nodb-1000-100-4-10/query-hits-4 5000000 320 ns/op 16 | BenchmarkSmall/nodb-1000-100-4-10/update-4 200000 7959 ns/op 17 | BenchmarkSmall/nodb-1000-100-4-10/tmsp-4 100000 25878 ns/op 18 | BenchmarkSmall/nodb-1000-100-4-10/insert-4 100000 28215 ns/op 19 | BenchmarkSmall/nodb-1000-100-4-10/delete-4 100000 15757 ns/op 20 | Init Tree took 0.84 MB 21 | BenchmarkSmall/memdb-1000-100-4-10/query-miss-4 500000 3296 ns/op 22 | BenchmarkSmall/memdb-1000-100-4-10/query-hits-4 300000 3664 ns/op 23 | BenchmarkSmall/memdb-1000-100-4-10/update-4 50000 25869 ns/op 24 | BenchmarkSmall/memdb-1000-100-4-10/tmsp-4 30000 87781 ns/op 25 | BenchmarkSmall/memdb-1000-100-4-10/insert-4 20000 70119 ns/op 26 | BenchmarkSmall/memdb-1000-100-4-10/delete-4 50000 49289 ns/op 27 | Init Tree took 0.47 MB 28 | BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-4 300000 5219 ns/op 29 | BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-4 300000 6329 ns/op 30 | BenchmarkSmall/goleveldb-1000-100-4-10/update-4 30000 65849 ns/op 31 | BenchmarkSmall/goleveldb-1000-100-4-10/tmsp-4 10000 157153 ns/op 32 | BenchmarkSmall/goleveldb-1000-100-4-10/insert-4 10000 223123 ns/op 33 | BenchmarkSmall/goleveldb-1000-100-4-10/delete-4 20000 105517 ns/op 34 | Init Tree took 0.48 MB 35 | BenchmarkSmall/leveldb-1000-100-4-10/query-miss-4 200000 6629 ns/op 36 | BenchmarkSmall/leveldb-1000-100-4-10/query-hits-4 200000 7247 ns/op 37 | BenchmarkSmall/leveldb-1000-100-4-10/update-4 20000 60328 ns/op 38 | BenchmarkSmall/leveldb-1000-100-4-10/tmsp-4 10000 133760 ns/op 39 | BenchmarkSmall/leveldb-1000-100-4-10/insert-4 10000 128193 ns/op 40 | BenchmarkSmall/leveldb-1000-100-4-10/delete-4 20000 87809 ns/op 41 | PASS 42 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 57.747s 43 | Init Tree took 47.20 MB 44 | BenchmarkMedium/nodb-100000-100-16-40/query-miss-4 1000000 1372 ns/op 45 | BenchmarkMedium/nodb-100000-100-16-40/query-hits-4 1000000 1630 ns/op 46 | BenchmarkMedium/nodb-100000-100-16-40/update-4 50000 27774 ns/op 47 | BenchmarkMedium/nodb-100000-100-16-40/tmsp-4 50000 35904 ns/op 48 | BenchmarkMedium/nodb-100000-100-16-40/insert-4 50000 29889 ns/op 49 | BenchmarkMedium/nodb-100000-100-16-40/delete-4 100000 21200 ns/op 50 | Init Tree took 85.08 MB 51 | BenchmarkMedium/memdb-100000-100-16-40/query-miss-4 200000 8863 ns/op 52 | BenchmarkMedium/memdb-100000-100-16-40/query-hits-4 200000 9878 ns/op 53 | BenchmarkMedium/memdb-100000-100-16-40/update-4 20000 75840 ns/op 54 | BenchmarkMedium/memdb-100000-100-16-40/tmsp-4 10000 112251 ns/op 55 | BenchmarkMedium/memdb-100000-100-16-40/insert-4 20000 103080 ns/op 56 | BenchmarkMedium/memdb-100000-100-16-40/delete-4 20000 66677 ns/op 57 | Init Tree took 45.17 MB 58 | BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-4 50000 21601 ns/op 59 | BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-4 50000 27372 ns/op 60 | BenchmarkMedium/goleveldb-100000-100-16-40/update-4 10000 148820 ns/op 61 | BenchmarkMedium/goleveldb-100000-100-16-40/tmsp-4 5000 319488 ns/op 62 | BenchmarkMedium/goleveldb-100000-100-16-40/insert-4 2000 530568 ns/op 63 | BenchmarkMedium/goleveldb-100000-100-16-40/delete-4 2000 569455 ns/op 64 | Init Tree took 36.12 MB 65 | BenchmarkMedium/leveldb-100000-100-16-40/query-miss-4 50000 23190 ns/op 66 | BenchmarkMedium/leveldb-100000-100-16-40/query-hits-4 50000 27447 ns/op 67 | BenchmarkMedium/leveldb-100000-100-16-40/update-4 10000 147662 ns/op 68 | BenchmarkMedium/leveldb-100000-100-16-40/tmsp-4 5000 310984 ns/op 69 | BenchmarkMedium/leveldb-100000-100-16-40/insert-4 2000 549814 ns/op 70 | BenchmarkMedium/leveldb-100000-100-16-40/delete-4 2000 716094 ns/op 71 | PASS 72 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 202.957s 73 | Init Tree took 49.20 MB 74 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-miss-4 1000000 1079 ns/op 75 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-hits-4 1000000 1122 ns/op 76 | BenchmarkMemKeySizes/nodb-100000-100-4-80/update-4 100000 25405 ns/op 77 | BenchmarkMemKeySizes/nodb-100000-100-4-80/tmsp-4 50000 35486 ns/op 78 | BenchmarkMemKeySizes/nodb-100000-100-4-80/insert-4 50000 29162 ns/op 79 | BenchmarkMemKeySizes/nodb-100000-100-4-80/delete-4 100000 19531 ns/op 80 | Init Tree took 50.40 MB 81 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-miss-4 1000000 1224 ns/op 82 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-hits-4 1000000 1262 ns/op 83 | BenchmarkMemKeySizes/nodb-100000-100-16-80/update-4 50000 25344 ns/op 84 | BenchmarkMemKeySizes/nodb-100000-100-16-80/tmsp-4 50000 33859 ns/op 85 | BenchmarkMemKeySizes/nodb-100000-100-16-80/insert-4 50000 28705 ns/op 86 | BenchmarkMemKeySizes/nodb-100000-100-16-80/delete-4 100000 19679 ns/op 87 | Init Tree took 52.00 MB 88 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-miss-4 1000000 1333 ns/op 89 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-hits-4 1000000 1348 ns/op 90 | BenchmarkMemKeySizes/nodb-100000-100-32-80/update-4 50000 25319 ns/op 91 | BenchmarkMemKeySizes/nodb-100000-100-32-80/tmsp-4 50000 33329 ns/op 92 | BenchmarkMemKeySizes/nodb-100000-100-32-80/insert-4 50000 29624 ns/op 93 | BenchmarkMemKeySizes/nodb-100000-100-32-80/delete-4 100000 20300 ns/op 94 | Init Tree took 55.20 MB 95 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-miss-4 1000000 1503 ns/op 96 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-hits-4 1000000 1412 ns/op 97 | BenchmarkMemKeySizes/nodb-100000-100-64-80/update-4 50000 26542 ns/op 98 | BenchmarkMemKeySizes/nodb-100000-100-64-80/tmsp-4 50000 33529 ns/op 99 | BenchmarkMemKeySizes/nodb-100000-100-64-80/insert-4 50000 32928 ns/op 100 | BenchmarkMemKeySizes/nodb-100000-100-64-80/delete-4 100000 22184 ns/op 101 | Init Tree took 61.60 MB 102 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-miss-4 500000 2234 ns/op 103 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-hits-4 1000000 2012 ns/op 104 | BenchmarkMemKeySizes/nodb-100000-100-128-80/update-4 50000 28088 ns/op 105 | BenchmarkMemKeySizes/nodb-100000-100-128-80/tmsp-4 50000 49556 ns/op 106 | BenchmarkMemKeySizes/nodb-100000-100-128-80/insert-4 50000 31738 ns/op 107 | BenchmarkMemKeySizes/nodb-100000-100-128-80/delete-4 100000 20232 ns/op 108 | Init Tree took 74.40 MB 109 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-miss-4 500000 2222 ns/op 110 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-hits-4 1000000 1592 ns/op 111 | BenchmarkMemKeySizes/nodb-100000-100-256-80/update-4 50000 29363 ns/op 112 | BenchmarkMemKeySizes/nodb-100000-100-256-80/tmsp-4 50000 37905 ns/op 113 | BenchmarkMemKeySizes/nodb-100000-100-256-80/insert-4 50000 33531 ns/op 114 | BenchmarkMemKeySizes/nodb-100000-100-256-80/delete-4 100000 19828 ns/op 115 | PASS 116 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 147.856s 117 | -------------------------------------------------------------------------------- /benchmarks/results/aws-c4-large-f6f41ca.txt: -------------------------------------------------------------------------------- 1 | make[1]: Entering directory '/home/ubuntu/go/src/github.com/tendermint/merkleeyes/iavl' 2 | cd benchmarks && \ 3 | go test -bench=RandomBytes . && \ 4 | go test -bench=Small . && \ 5 | go test -bench=Medium . && \ 6 | go test -bench=BenchmarkMemKeySizes . 7 | BenchmarkRandomBytes/random-4-2 20000000 58.1 ns/op 8 | BenchmarkRandomBytes/random-16-2 20000000 85.5 ns/op 9 | BenchmarkRandomBytes/random-32-2 20000000 118 ns/op 10 | BenchmarkRandomBytes/random-100-2 5000000 267 ns/op 11 | BenchmarkRandomBytes/random-1000-2 1000000 2129 ns/op 12 | PASS 13 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 9.276s 14 | Init Tree took 0.42 MB 15 | BenchmarkSmall/nodb-1000-100-4-10/query-miss-2 5000000 316 ns/op 16 | BenchmarkSmall/nodb-1000-100-4-10/query-hits-2 5000000 297 ns/op 17 | BenchmarkSmall/nodb-1000-100-4-10/update-2 200000 8469 ns/op 18 | BenchmarkSmall/nodb-1000-100-4-10/tmsp-2 100000 31893 ns/op 19 | BenchmarkSmall/nodb-1000-100-4-10/insert-2 100000 30303 ns/op 20 | BenchmarkSmall/nodb-1000-100-4-10/delete-2 200000 20423 ns/op 21 | Init Tree took 0.84 MB 22 | BenchmarkSmall/memdb-1000-100-4-10/query-miss-2 500000 3156 ns/op 23 | BenchmarkSmall/memdb-1000-100-4-10/query-hits-2 500000 3527 ns/op 24 | BenchmarkSmall/memdb-1000-100-4-10/update-2 100000 22616 ns/op 25 | BenchmarkSmall/memdb-1000-100-4-10/tmsp-2 30000 76474 ns/op 26 | BenchmarkSmall/memdb-1000-100-4-10/insert-2 30000 69403 ns/op 27 | BenchmarkSmall/memdb-1000-100-4-10/delete-2 50000 45710 ns/op 28 | Init Tree took 0.47 MB 29 | BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-2 300000 5451 ns/op 30 | BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-2 200000 6857 ns/op 31 | BenchmarkSmall/goleveldb-1000-100-4-10/update-2 20000 83362 ns/op 32 | BenchmarkSmall/goleveldb-1000-100-4-10/tmsp-2 10000 165200 ns/op 33 | BenchmarkSmall/goleveldb-1000-100-4-10/insert-2 10000 158884 ns/op 34 | BenchmarkSmall/goleveldb-1000-100-4-10/delete-2 20000 116426 ns/op 35 | Init Tree took -7.19 MB 36 | BenchmarkSmall/leveldb-1000-100-4-10/query-miss-2 300000 5168 ns/op 37 | BenchmarkSmall/leveldb-1000-100-4-10/query-hits-2 200000 6381 ns/op 38 | BenchmarkSmall/leveldb-1000-100-4-10/update-2 20000 83882 ns/op 39 | BenchmarkSmall/leveldb-1000-100-4-10/tmsp-2 10000 154254 ns/op 40 | BenchmarkSmall/leveldb-1000-100-4-10/insert-2 10000 152926 ns/op 41 | BenchmarkSmall/leveldb-1000-100-4-10/delete-2 20000 115319 ns/op 42 | PASS 43 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 63.175s 44 | Init Tree took 47.20 MB 45 | BenchmarkMedium/nodb-100000-100-16-40/query-miss-2 2000000 971 ns/op 46 | BenchmarkMedium/nodb-100000-100-16-40/query-hits-2 2000000 981 ns/op 47 | BenchmarkMedium/nodb-100000-100-16-40/update-2 50000 30686 ns/op 48 | BenchmarkMedium/nodb-100000-100-16-40/tmsp-2 30000 44455 ns/op 49 | BenchmarkMedium/nodb-100000-100-16-40/insert-2 50000 35721 ns/op 50 | BenchmarkMedium/nodb-100000-100-16-40/delete-2 50000 25793 ns/op 51 | Init Tree took 85.13 MB 52 | BenchmarkMedium/memdb-100000-100-16-40/query-miss-2 200000 7887 ns/op 53 | BenchmarkMedium/memdb-100000-100-16-40/query-hits-2 200000 8736 ns/op 54 | BenchmarkMedium/memdb-100000-100-16-40/update-2 20000 86067 ns/op 55 | BenchmarkMedium/memdb-100000-100-16-40/tmsp-2 10000 122596 ns/op 56 | BenchmarkMedium/memdb-100000-100-16-40/insert-2 20000 92261 ns/op 57 | BenchmarkMedium/memdb-100000-100-16-40/delete-2 20000 66919 ns/op 58 | Init Tree took 45.41 MB 59 | BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-2 100000 17993 ns/op 60 | BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-2 100000 22781 ns/op 61 | BenchmarkMedium/goleveldb-100000-100-16-40/update-2 10000 210399 ns/op 62 | BenchmarkMedium/goleveldb-100000-100-16-40/tmsp-2 3000 371019 ns/op 63 | BenchmarkMedium/goleveldb-100000-100-16-40/insert-2 2000 597672 ns/op 64 | BenchmarkMedium/goleveldb-100000-100-16-40/delete-2 2000 660506 ns/op 65 | Init Tree took 36.39 MB 66 | BenchmarkMedium/leveldb-100000-100-16-40/query-miss-2 100000 17543 ns/op 67 | BenchmarkMedium/leveldb-100000-100-16-40/query-hits-2 100000 22734 ns/op 68 | BenchmarkMedium/leveldb-100000-100-16-40/update-2 10000 208948 ns/op 69 | BenchmarkMedium/leveldb-100000-100-16-40/tmsp-2 3000 392419 ns/op 70 | BenchmarkMedium/leveldb-100000-100-16-40/insert-2 2000 622588 ns/op 71 | BenchmarkMedium/leveldb-100000-100-16-40/delete-2 2000 580446 ns/op 72 | PASS 73 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 185.401s 74 | Init Tree took 49.20 MB 75 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-miss-2 2000000 875 ns/op 76 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-hits-2 2000000 928 ns/op 77 | BenchmarkMemKeySizes/nodb-100000-100-4-80/update-2 50000 30808 ns/op 78 | BenchmarkMemKeySizes/nodb-100000-100-4-80/tmsp-2 30000 41638 ns/op 79 | BenchmarkMemKeySizes/nodb-100000-100-4-80/insert-2 50000 35774 ns/op 80 | BenchmarkMemKeySizes/nodb-100000-100-4-80/delete-2 50000 23646 ns/op 81 | Init Tree took 50.40 MB 82 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-miss-2 1000000 1005 ns/op 83 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-hits-2 1000000 1002 ns/op 84 | BenchmarkMemKeySizes/nodb-100000-100-16-80/update-2 50000 31509 ns/op 85 | BenchmarkMemKeySizes/nodb-100000-100-16-80/tmsp-2 30000 41108 ns/op 86 | BenchmarkMemKeySizes/nodb-100000-100-16-80/insert-2 50000 36185 ns/op 87 | BenchmarkMemKeySizes/nodb-100000-100-16-80/delete-2 50000 23535 ns/op 88 | Init Tree took 52.00 MB 89 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-miss-2 1000000 1023 ns/op 90 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-hits-2 1000000 1010 ns/op 91 | BenchmarkMemKeySizes/nodb-100000-100-32-80/update-2 50000 31243 ns/op 92 | BenchmarkMemKeySizes/nodb-100000-100-32-80/tmsp-2 50000 43901 ns/op 93 | BenchmarkMemKeySizes/nodb-100000-100-32-80/insert-2 50000 35957 ns/op 94 | BenchmarkMemKeySizes/nodb-100000-100-32-80/delete-2 50000 23259 ns/op 95 | Init Tree took 55.20 MB 96 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-miss-2 1000000 1104 ns/op 97 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-hits-2 2000000 937 ns/op 98 | BenchmarkMemKeySizes/nodb-100000-100-64-80/update-2 50000 31014 ns/op 99 | BenchmarkMemKeySizes/nodb-100000-100-64-80/tmsp-2 50000 40717 ns/op 100 | BenchmarkMemKeySizes/nodb-100000-100-64-80/insert-2 50000 35122 ns/op 101 | BenchmarkMemKeySizes/nodb-100000-100-64-80/delete-2 50000 22827 ns/op 102 | Init Tree took 61.60 MB 103 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-miss-2 1000000 1343 ns/op 104 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-hits-2 2000000 978 ns/op 105 | BenchmarkMemKeySizes/nodb-100000-100-128-80/update-2 50000 31088 ns/op 106 | BenchmarkMemKeySizes/nodb-100000-100-128-80/tmsp-2 50000 42017 ns/op 107 | BenchmarkMemKeySizes/nodb-100000-100-128-80/insert-2 50000 35814 ns/op 108 | BenchmarkMemKeySizes/nodb-100000-100-128-80/delete-2 100000 21684 ns/op 109 | Init Tree took 74.40 MB 110 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-miss-2 1000000 1686 ns/op 111 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-hits-2 2000000 937 ns/op 112 | BenchmarkMemKeySizes/nodb-100000-100-256-80/update-2 50000 31305 ns/op 113 | BenchmarkMemKeySizes/nodb-100000-100-256-80/tmsp-2 50000 41948 ns/op 114 | BenchmarkMemKeySizes/nodb-100000-100-256-80/insert-2 50000 35809 ns/op 115 | BenchmarkMemKeySizes/nodb-100000-100-256-80/delete-2 100000 20561 ns/op 116 | PASS 117 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 160.512s 118 | make[1]: Leaving directory '/home/ubuntu/go/src/github.com/tendermint/merkleeyes/iavl' 119 | -------------------------------------------------------------------------------- /benchmarks/results/digital-ocean-2gb-2717167.txt: -------------------------------------------------------------------------------- 1 | make[1]: Entering directory '/root/go/src/github.com/tendermint/merkleeyes/iavl' 2 | cd benchmarks && \ 3 | go test -bench=RandomBytes . && \ 4 | go test -bench=Small . && \ 5 | go test -bench=Medium . && \ 6 | go test -bench=BenchmarkMemKeySizes . 7 | BenchmarkRandomBytes/random-4-2 10000000 109 ns/op 8 | BenchmarkRandomBytes/random-16-2 10000000 138 ns/op 9 | BenchmarkRandomBytes/random-32-2 10000000 293 ns/op 10 | BenchmarkRandomBytes/random-100-2 2000000 929 ns/op 11 | BenchmarkRandomBytes/random-1000-2 300000 7309 ns/op 12 | PASS 13 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 11.873s 14 | Init Tree took 0.42 MB 15 | BenchmarkSmall/nodb-1000-100-4-10/query-miss-2 2000000 699 ns/op 16 | BenchmarkSmall/nodb-1000-100-4-10/query-hits-2 2000000 709 ns/op 17 | BenchmarkSmall/nodb-1000-100-4-10/update-2 100000 25067 ns/op 18 | BenchmarkSmall/nodb-1000-100-4-10/tmsp-2 30000 79326 ns/op 19 | BenchmarkSmall/nodb-1000-100-4-10/insert-2 30000 71205 ns/op 20 | BenchmarkSmall/nodb-1000-100-4-10/delete-2 50000 37782 ns/op 21 | Init Tree took 0.84 MB 22 | BenchmarkSmall/memdb-1000-100-4-10/query-miss-2 100000 11325 ns/op 23 | BenchmarkSmall/memdb-1000-100-4-10/query-hits-2 300000 9585 ns/op 24 | BenchmarkSmall/memdb-1000-100-4-10/update-2 20000 88439 ns/op 25 | BenchmarkSmall/memdb-1000-100-4-10/tmsp-2 10000 235413 ns/op 26 | BenchmarkSmall/memdb-1000-100-4-10/insert-2 10000 231453 ns/op 27 | BenchmarkSmall/memdb-1000-100-4-10/delete-2 10000 121835 ns/op 28 | Init Tree took 0.47 MB 29 | BenchmarkSmall/goleveldb-1000-100-4-10/query-miss-2 100000 10942 ns/op 30 | BenchmarkSmall/goleveldb-1000-100-4-10/query-hits-2 100000 13878 ns/op 31 | BenchmarkSmall/goleveldb-1000-100-4-10/update-2 10000 125941 ns/op 32 | BenchmarkSmall/goleveldb-1000-100-4-10/tmsp-2 5000 228558 ns/op 33 | BenchmarkSmall/goleveldb-1000-100-4-10/insert-2 10000 439718 ns/op 34 | BenchmarkSmall/goleveldb-1000-100-4-10/delete-2 10000 254810 ns/op 35 | Init Tree took 0.47 MB 36 | BenchmarkSmall/leveldb-1000-100-4-10/query-miss-2 200000 8161 ns/op 37 | BenchmarkSmall/leveldb-1000-100-4-10/query-hits-2 200000 10279 ns/op 38 | BenchmarkSmall/leveldb-1000-100-4-10/update-2 10000 140223 ns/op 39 | BenchmarkSmall/leveldb-1000-100-4-10/tmsp-2 5000 266231 ns/op 40 | BenchmarkSmall/leveldb-1000-100-4-10/insert-2 10000 440206 ns/op 41 | BenchmarkSmall/leveldb-1000-100-4-10/delete-2 10000 262984 ns/op 42 | PASS 43 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 61.918s 44 | Init Tree took 47.20 MB 45 | BenchmarkMedium/nodb-100000-100-16-40/query-miss-2 500000 3026 ns/op 46 | BenchmarkMedium/nodb-100000-100-16-40/query-hits-2 500000 2226 ns/op 47 | BenchmarkMedium/nodb-100000-100-16-40/update-2 30000 47671 ns/op 48 | BenchmarkMedium/nodb-100000-100-16-40/tmsp-2 20000 91781 ns/op 49 | BenchmarkMedium/nodb-100000-100-16-40/insert-2 20000 70797 ns/op 50 | BenchmarkMedium/nodb-100000-100-16-40/delete-2 30000 47361 ns/op 51 | Init Tree took 85.08 MB 52 | BenchmarkMedium/memdb-100000-100-16-40/query-miss-2 50000 22383 ns/op 53 | BenchmarkMedium/memdb-100000-100-16-40/query-hits-2 100000 15686 ns/op 54 | BenchmarkMedium/memdb-100000-100-16-40/update-2 10000 146801 ns/op 55 | BenchmarkMedium/memdb-100000-100-16-40/tmsp-2 5000 301393 ns/op 56 | BenchmarkMedium/memdb-100000-100-16-40/insert-2 10000 123555 ns/op 57 | BenchmarkMedium/memdb-100000-100-16-40/delete-2 10000 130045 ns/op 58 | Init Tree took 45.29 MB 59 | BenchmarkMedium/goleveldb-100000-100-16-40/query-miss-2 10000 331146 ns/op 60 | BenchmarkMedium/goleveldb-100000-100-16-40/query-hits-2 20000 72753 ns/op 61 | BenchmarkMedium/goleveldb-100000-100-16-40/update-2 5000 655807 ns/op 62 | BenchmarkMedium/goleveldb-100000-100-16-40/tmsp-2 1000 1174947 ns/op 63 | BenchmarkMedium/goleveldb-100000-100-16-40/insert-2 1000 1570246 ns/op 64 | BenchmarkMedium/goleveldb-100000-100-16-40/delete-2 1000 1500384 ns/op 65 | Init Tree took 36.45 MB 66 | BenchmarkMedium/leveldb-100000-100-16-40/query-miss-2 3000 382304 ns/op 67 | BenchmarkMedium/leveldb-100000-100-16-40/query-hits-2 30000 68459 ns/op 68 | BenchmarkMedium/leveldb-100000-100-16-40/update-2 5000 407120 ns/op 69 | BenchmarkMedium/leveldb-100000-100-16-40/tmsp-2 3000 962354 ns/op 70 | BenchmarkMedium/leveldb-100000-100-16-40/insert-2 1000 1806111 ns/op 71 | BenchmarkMedium/leveldb-100000-100-16-40/delete-2 1000 1043286 ns/op 72 | PASS 73 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 321.607s 74 | Init Tree took 49.20 MB 75 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-miss-2 500000 3863 ns/op 76 | BenchmarkMemKeySizes/nodb-100000-100-4-80/query-hits-2 500000 2577 ns/op 77 | BenchmarkMemKeySizes/nodb-100000-100-4-80/update-2 30000 54838 ns/op 78 | BenchmarkMemKeySizes/nodb-100000-100-4-80/tmsp-2 10000 110951 ns/op 79 | BenchmarkMemKeySizes/nodb-100000-100-4-80/insert-2 30000 82763 ns/op 80 | BenchmarkMemKeySizes/nodb-100000-100-4-80/delete-2 50000 36173 ns/op 81 | Init Tree took 50.40 MB 82 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-miss-2 500000 2017 ns/op 83 | BenchmarkMemKeySizes/nodb-100000-100-16-80/query-hits-2 1000000 1852 ns/op 84 | BenchmarkMemKeySizes/nodb-100000-100-16-80/update-2 50000 63354 ns/op 85 | BenchmarkMemKeySizes/nodb-100000-100-16-80/tmsp-2 20000 104027 ns/op 86 | BenchmarkMemKeySizes/nodb-100000-100-16-80/insert-2 30000 48495 ns/op 87 | BenchmarkMemKeySizes/nodb-100000-100-16-80/delete-2 50000 37681 ns/op 88 | Init Tree took 52.00 MB 89 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-miss-2 1000000 2594 ns/op 90 | BenchmarkMemKeySizes/nodb-100000-100-32-80/query-hits-2 500000 3050 ns/op 91 | BenchmarkMemKeySizes/nodb-100000-100-32-80/update-2 30000 45242 ns/op 92 | BenchmarkMemKeySizes/nodb-100000-100-32-80/tmsp-2 20000 80852 ns/op 93 | BenchmarkMemKeySizes/nodb-100000-100-32-80/insert-2 30000 48661 ns/op 94 | BenchmarkMemKeySizes/nodb-100000-100-32-80/delete-2 50000 42489 ns/op 95 | Init Tree took 55.20 MB 96 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-miss-2 1000000 2733 ns/op 97 | BenchmarkMemKeySizes/nodb-100000-100-64-80/query-hits-2 1000000 2477 ns/op 98 | BenchmarkMemKeySizes/nodb-100000-100-64-80/update-2 30000 71995 ns/op 99 | BenchmarkMemKeySizes/nodb-100000-100-64-80/tmsp-2 30000 60682 ns/op 100 | BenchmarkMemKeySizes/nodb-100000-100-64-80/insert-2 30000 65346 ns/op 101 | BenchmarkMemKeySizes/nodb-100000-100-64-80/delete-2 50000 32863 ns/op 102 | Init Tree took 61.60 MB 103 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-miss-2 500000 3102 ns/op 104 | BenchmarkMemKeySizes/nodb-100000-100-128-80/query-hits-2 500000 3365 ns/op 105 | BenchmarkMemKeySizes/nodb-100000-100-128-80/update-2 30000 68945 ns/op 106 | BenchmarkMemKeySizes/nodb-100000-100-128-80/tmsp-2 20000 62037 ns/op 107 | BenchmarkMemKeySizes/nodb-100000-100-128-80/insert-2 20000 79507 ns/op 108 | BenchmarkMemKeySizes/nodb-100000-100-128-80/delete-2 50000 48408 ns/op 109 | Init Tree took 74.40 MB 110 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-miss-2 300000 4619 ns/op 111 | BenchmarkMemKeySizes/nodb-100000-100-256-80/query-hits-2 1000000 1975 ns/op 112 | BenchmarkMemKeySizes/nodb-100000-100-256-80/update-2 30000 53841 ns/op 113 | BenchmarkMemKeySizes/nodb-100000-100-256-80/tmsp-2 20000 87730 ns/op 114 | BenchmarkMemKeySizes/nodb-100000-100-256-80/insert-2 30000 49158 ns/op 115 | BenchmarkMemKeySizes/nodb-100000-100-256-80/delete-2 50000 31043 ns/op 116 | PASS 117 | ok github.com/tendermint/merkleeyes/iavl/benchmarks 247.827s 118 | make[1]: Leaving directory '/root/go/src/github.com/tendermint/merkleeyes/iavl' 119 | -------------------------------------------------------------------------------- /benchmarks/setup/INSTALL_ROOT.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | apt-get update 4 | apt-get -y upgrade 5 | apt-get -y install make screen 6 | 7 | GOFILE=go1.7.linux-amd64.tar.gz 8 | 9 | wget https://storage.googleapis.com/golang/${GOFILE} 10 | tar xzf ${GOFILE} 11 | mv go /usr/local/go1.7 12 | ln -s /usr/local/go1.7 /usr/local/go 13 | -------------------------------------------------------------------------------- /benchmarks/setup/INSTALL_USER.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This runs benchmarks, by default from develop branch of 4 | # github.com/tendermint/merkleeyes/iavl 5 | # You can customize this by optional command line args 6 | # 7 | # INSTALL_USER.sh [branch] [repouser] 8 | # 9 | # set repouser as your username to time your fork 10 | 11 | BRANCH=${1:-develop} 12 | REPOUSER=${2:-tendermint} 13 | 14 | cat <<'EOF' > ~/.goenv 15 | export GOROOT=/usr/local/go 16 | export GOPATH=$HOME/go 17 | export PATH=$GOPATH/bin:$GOROOT/bin:$PATH 18 | EOF 19 | 20 | . ~/.goenv 21 | 22 | mkdir -p $GOPATH/src/github.com/tendermint 23 | MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl 24 | git clone https://github.com/${REPOUSER}/merkleeyes/iavl.git $MERKLE 25 | cd $MERKLE 26 | git checkout ${BRANCH} 27 | -------------------------------------------------------------------------------- /benchmarks/setup/RUN_BENCHMARKS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . ~/.goenv 4 | 5 | MERKLE=$GOPATH/src/github.com/tendermint/merkleeyes/iavl 6 | cd $MERKLE 7 | git pull 8 | 9 | make get_deps 10 | make record 11 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | environment: 3 | GOPATH: /home/ubuntu/.go_workspace 4 | REPO: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME 5 | GO15VENDOREXPERIMENT: 1 6 | hosts: 7 | circlehost: 127.0.0.1 8 | localhost: 127.0.0.1 9 | 10 | checkout: 11 | post: 12 | - rm -rf $REPO 13 | - mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME 14 | - mv $HOME/$CIRCLE_PROJECT_REPONAME $REPO 15 | # - git submodule sync 16 | # - git submodule update --init # use submodules 17 | 18 | dependencies: 19 | override: 20 | - "cd $REPO && make get_vendor_deps && make install" 21 | 22 | test: 23 | override: 24 | - "cd $REPO && make test" 25 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package eyes 2 | 3 | import ( 4 | abcicli "github.com/tendermint/abci/client" 5 | abci "github.com/tendermint/abci/types" 6 | "github.com/tendermint/go-wire" 7 | "github.com/tendermint/merkleeyes/app" 8 | cmn "github.com/tendermint/tmlibs/common" 9 | ) 10 | 11 | type Client struct { 12 | abcicli.Client 13 | } 14 | 15 | func NewClient(addr string) (*Client, error) { 16 | abciClient, err := abcicli.NewClient(addr, "socket", false) 17 | if err != nil { 18 | return nil, err 19 | } 20 | client := &Client{ 21 | Client: abciClient, 22 | } 23 | return client, nil 24 | } 25 | 26 | func NewLocalClient(dbName string, cacheSize int) *Client { 27 | eyesApp := app.NewMerkleEyesApp(dbName, cacheSize) 28 | abciClient := abcicli.NewLocalClient(nil, eyesApp) 29 | return &Client{ 30 | Client: abciClient, 31 | } 32 | } 33 | 34 | // Convenience KVStore interface 35 | func (client *Client) Get(key []byte) (value []byte) { 36 | _, value, err := client.GetByKey(key) 37 | if err != nil { 38 | panic("requesting ABCI Query: " + err.Error()) 39 | } 40 | return value 41 | } 42 | 43 | func (client *Client) GetByKey(key []byte) (index int64, value []byte, err error) { 44 | resQuery, err := client.QuerySync(abci.RequestQuery{ 45 | Path: "/key", 46 | Data: key, 47 | Height: 0, 48 | }) 49 | if err != nil { 50 | return 51 | } 52 | return resQuery.Index, resQuery.Value, nil 53 | } 54 | 55 | // TODO: Support returning index too? 56 | func (client *Client) GetByKeyWithProof(key []byte) (value []byte, proof []byte, err error) { 57 | resQuery, err := client.QuerySync(abci.RequestQuery{ 58 | Path: "/key", 59 | Data: key, 60 | Height: 0, 61 | Prove: true, 62 | }) 63 | if err != nil { 64 | return 65 | } 66 | return resQuery.Value, resQuery.Proof, nil 67 | } 68 | 69 | func (client *Client) GetByIndex(index int64) (key []byte, value []byte, err error) { 70 | resQuery, err := client.QuerySync(abci.RequestQuery{ 71 | Path: "/index", 72 | Data: wire.BinaryBytes(index), 73 | Height: 0, 74 | }) 75 | if err != nil { 76 | return 77 | } 78 | return resQuery.Key, resQuery.Value, nil 79 | } 80 | 81 | // Convenience KVStore interface 82 | func (client *Client) Set(key []byte, value []byte) { 83 | tx := make([]byte, app.NonceLength+1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value)) 84 | buf := tx 85 | 86 | // nonce 87 | copy(buf[:12], cmn.RandBytes(12)) 88 | buf = buf[12:] 89 | 90 | buf[0] = app.TxTypeSet // Set TypeByte 91 | buf = buf[1:] 92 | n, err := wire.PutByteSlice(buf, key) 93 | if err != nil { 94 | panic("encoding key byteslice: " + err.Error()) 95 | } 96 | buf = buf[n:] 97 | n, err = wire.PutByteSlice(buf, value) 98 | if err != nil { 99 | panic("encoding value byteslice: " + err.Error()) 100 | } 101 | res := client.DeliverTxSync(tx) 102 | if res.IsErr() { 103 | panic(res.Error()) 104 | } 105 | } 106 | 107 | // Convenience 108 | func (client *Client) Remove(key []byte) { 109 | tx := make([]byte, app.NonceLength+1+wire.ByteSliceSize(key)) 110 | buf := tx 111 | 112 | // nonce 113 | copy(buf[:12], cmn.RandBytes(12)) 114 | buf = buf[12:] 115 | 116 | buf[0] = app.TxTypeRm // Rem TypeByte 117 | buf = buf[1:] 118 | _, err := wire.PutByteSlice(buf, key) 119 | if err != nil { 120 | panic("encoding key byteslice: " + err.Error()) 121 | } 122 | res := client.DeliverTxSync(tx) 123 | if res.IsErr() { 124 | panic(res.Error()) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /cmd/app.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/tendermint/abci/server" 6 | 7 | cmn "github.com/tendermint/tmlibs/common" 8 | 9 | application "github.com/tendermint/merkleeyes/app" 10 | ) 11 | 12 | var ( 13 | address string 14 | abci string 15 | cache int 16 | ) 17 | 18 | var startCmd = &cobra.Command{ 19 | Run: StartServer, 20 | Use: "start", 21 | Short: "Start the MerkleEyes server", 22 | Long: `Startup the MerkleEyes ABCi app`, 23 | } 24 | 25 | func init() { 26 | RootCmd.AddCommand(startCmd) 27 | startCmd.Flags().StringVarP(&address, "address", "l", "unix://data.sock", "MerkleEyes server listen address") 28 | startCmd.Flags().StringVarP(&abci, "abci", "a", "socket", "socket | grpc") 29 | startCmd.Flags().IntVarP(&cache, "cache", "c", 0, "database cache size") 30 | } 31 | 32 | func StartServer(cmd *cobra.Command, args []string) { 33 | app := application.NewMerkleEyesApp(dbName, cache) 34 | server, err := server.NewServer(address, abci, app) 35 | 36 | if err != nil { 37 | cmn.Exit(err.Error()) 38 | } 39 | _, err = server.Start() 40 | if err != nil { 41 | cmn.Exit(err.Error()) 42 | } 43 | 44 | cmn.TrapSignal(func() { 45 | app.CloseDB() 46 | server.Stop() 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /cmd/dump.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | cmn "github.com/tendermint/tmlibs/common" 10 | db "github.com/tendermint/tmlibs/db" 11 | 12 | "github.com/tendermint/merkleeyes/iavl" 13 | ) 14 | 15 | var ( 16 | dbDir string 17 | verbose bool 18 | cacheSize int 19 | ) 20 | 21 | var dumpCmd = &cobra.Command{ 22 | Run: DumpDatabase, 23 | Use: "dump", 24 | Short: "Dump a database", 25 | Long: `Dump all of the data for an underlying persistent database`, 26 | } 27 | 28 | func init() { 29 | RootCmd.AddCommand(dumpCmd) 30 | dumpCmd.Flags().StringVarP(&dbDir, "path", "p", "./", "Dir path to DB") 31 | dumpCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Print everything") 32 | dumpCmd.Flags().IntVarP(&cacheSize, "cache", "c", 10000, "Size of the Cache") 33 | } 34 | 35 | func DumpDatabase(cmd *cobra.Command, args []string) { 36 | if dbName == "" { 37 | dbName = "merkleeyes" 38 | } 39 | 40 | dbPath := path.Join(dbDir, dbName+".db") 41 | 42 | if !cmn.FileExists(dbPath) { 43 | cmn.Exit("No existing database: " + dbPath) 44 | } 45 | 46 | if verbose { 47 | fmt.Printf("Dumping DB %s (%s)...\n", dbName, dbType) 48 | } 49 | 50 | database := db.NewDB(dbName, db.LevelDBBackendStr, "./") 51 | 52 | if verbose { 53 | fmt.Printf("Database: %v\n", database) 54 | } 55 | 56 | tree := iavl.NewIAVLTree(cacheSize, database) 57 | 58 | if verbose { 59 | fmt.Printf("Tree: %v\n", tree) 60 | } 61 | 62 | tree.Dump(verbose, nil) 63 | } 64 | -------------------------------------------------------------------------------- /cmd/loadtest.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/rand" 7 | "os" 8 | "runtime" 9 | "time" 10 | 11 | "github.com/spf13/cobra" 12 | 13 | "github.com/tendermint/merkleeyes/iavl" 14 | "github.com/tendermint/tmlibs/db" 15 | "github.com/tendermint/tmlibs/merkle" 16 | ) 17 | 18 | var loadtestCmd = &cobra.Command{ 19 | Run: LoadTest, 20 | Use: "loadtest", 21 | Short: "Run a load test on the database", 22 | Long: `Do a long running load test on the database to determine timing`, 23 | } 24 | 25 | const reportInterval = 100 26 | 27 | var ( 28 | initSize int 29 | keySize int 30 | dataSize int 31 | blockSize int 32 | ) 33 | 34 | func init() { 35 | RootCmd.AddCommand(loadtestCmd) 36 | loadtestCmd.Flags().IntVarP(&initSize, "initsize", "i", 100000, "Initial DB Size") 37 | loadtestCmd.Flags().IntVarP(&keySize, "keysize", "k", 16, "Length of keys (in bytes)") 38 | loadtestCmd.Flags().IntVarP(&dataSize, "valuesize", "v", 100, "Length of values (in bytes)") 39 | loadtestCmd.Flags().IntVarP(&blockSize, "blocksize", "b", 200, "Number of Txs per block") 40 | } 41 | 42 | func LoadTest(cmd *cobra.Command, args []string) { 43 | 44 | tmpDir, err := ioutil.TempDir("", "loadtest-") 45 | if err != nil { 46 | fmt.Printf("Cannot create temp dir: %s\n", err) 47 | os.Exit(-1) 48 | } 49 | 50 | initMB := memUseMB() 51 | start := time.Now() 52 | 53 | fmt.Printf("Preparing DB (%s with %d keys)...\n", dbType, initSize) 54 | d := db.NewDB("loadtest", dbType, tmpDir) 55 | tree, keys := prepareTree(d, initSize, keySize, dataSize) 56 | 57 | delta := time.Now().Sub(start) 58 | fmt.Printf("Initialization took %0.3f s, used %0.2f MB\n", 59 | delta.Seconds(), memUseMB()-initMB) 60 | fmt.Printf("Keysize: %d, Datasize: %d\n", keySize, dataSize) 61 | 62 | fmt.Printf("Starting loadtest (blocks of %d tx)...\n", blockSize) 63 | loopForever(tree, dataSize, blockSize, keys, initMB) 64 | } 65 | 66 | // blatently copied from benchmarks/bench_test.go 67 | func randBytes(length int) []byte { 68 | key := make([]byte, length) 69 | // math.rand.Read always returns err=nil 70 | rand.Read(key) 71 | return key 72 | } 73 | 74 | // blatently copied from benchmarks/bench_test.go 75 | func prepareTree(db db.DB, size, keyLen, dataLen int) (merkle.Tree, [][]byte) { 76 | t := iavl.NewIAVLTree(size, db) 77 | keys := make([][]byte, size) 78 | 79 | for i := 0; i < size; i++ { 80 | key := randBytes(keyLen) 81 | t.Set(key, randBytes(dataLen)) 82 | keys[i] = key 83 | } 84 | t.Hash() 85 | t.Save() 86 | runtime.GC() 87 | return t, keys 88 | } 89 | 90 | func runBlock(t merkle.Tree, dataLen, blockSize int, keys [][]byte) merkle.Tree { 91 | l := int32(len(keys)) 92 | 93 | real := t.Copy() 94 | check := t.Copy() 95 | 96 | for j := 0; j < blockSize; j++ { 97 | // always update to avoid changing size 98 | key := keys[rand.Int31n(l)] 99 | data := randBytes(dataLen) 100 | 101 | // perform query and write on check and then real 102 | check.Get(key) 103 | check.Set(key, data) 104 | real.Get(key) 105 | real.Set(key, data) 106 | } 107 | 108 | // at the end of a block, move it all along.... 109 | real.Hash() 110 | real.Save() 111 | return real 112 | } 113 | 114 | func loopForever(t merkle.Tree, dataLen, blockSize int, keys [][]byte, initMB float64) { 115 | for { 116 | start := time.Now() 117 | for i := 0; i < reportInterval; i++ { 118 | t = runBlock(t, dataLen, blockSize, keys) 119 | } 120 | // now report 121 | end := time.Now() 122 | delta := end.Sub(start) 123 | timing := delta.Seconds() / reportInterval 124 | usedMB := memUseMB() - initMB 125 | fmt.Printf("%s: blocks of %d tx: %0.3f s/block, using %0.2f MB\n", 126 | end.Format("Jan 2 15:04:05"), blockSize, timing, usedMB) 127 | } 128 | } 129 | 130 | // returns number of MB in use 131 | func memUseMB() float64 { 132 | var mem runtime.MemStats 133 | runtime.ReadMemStats(&mem) 134 | asize := mem.Alloc 135 | mb := float64(asize) / 1000000 136 | return mb 137 | } 138 | -------------------------------------------------------------------------------- /cmd/merkleeyes/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/tendermint/merkleeyes/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | var RootCmd = &cobra.Command{ 13 | Use: "merkleeyes", 14 | Short: "Merkleeyes server", 15 | Long: `Merkleeyes server and other tools 16 | 17 | Including: 18 | - Start the Merkleeyes server 19 | - Benchmark to check the underlying performance of the databases. 20 | - Dump to list the full contents of any persistent go-merkle database. 21 | `, 22 | } 23 | 24 | func Execute() { 25 | if err := RootCmd.Execute(); err != nil { 26 | fmt.Println(err) 27 | os.Exit(-1) 28 | } 29 | } 30 | 31 | var ( 32 | dbType string 33 | dbName string 34 | ) 35 | 36 | func init() { 37 | cobra.OnInitialize(initEnv) 38 | RootCmd.PersistentFlags().StringVarP(&dbType, "dbType", "t", "goleveldb", "type of backing db") 39 | RootCmd.PersistentFlags().StringVarP(&dbName, "dbName", "d", "", "database name") 40 | } 41 | 42 | func initEnv() { 43 | viper.SetEnvPrefix("TM") 44 | viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) 45 | viper.AutomaticEnv() 46 | } 47 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/tendermint/merkleeyes/version" 9 | ) 10 | 11 | var versionCmd = &cobra.Command{ 12 | Use: "version", 13 | Short: "Show version info", 14 | Run: func(cmd *cobra.Command, args []string) { 15 | fmt.Println(version.Version) 16 | }, 17 | } 18 | 19 | func init() { 20 | RootCmd.AddCommand(versionCmd) 21 | } 22 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: a8af57ad9881011183dc9303839ceec09ec7b3745479224f1d4f1cbaf48f546c 2 | updated: 2017-07-12T11:38:54.910095798-04:00 3 | imports: 4 | - name: github.com/btcsuite/btcd 5 | version: 583684b21bfbde9b5fc4403916fd7c807feb0289 6 | subpackages: 7 | - btcec 8 | - name: github.com/fsnotify/fsnotify 9 | version: 4da3e2cfbabc9f751898f250b49f2439785783a1 10 | - name: github.com/go-kit/kit 11 | version: d67bb4c202e3b91377d1079b110a6c9ce23ab2f8 12 | subpackages: 13 | - log 14 | - log/level 15 | - log/term 16 | - name: github.com/go-logfmt/logfmt 17 | version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 18 | - name: github.com/go-stack/stack 19 | version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 20 | - name: github.com/golang/protobuf 21 | version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 22 | subpackages: 23 | - proto 24 | - ptypes/any 25 | - name: github.com/golang/snappy 26 | version: 553a641470496b2327abcac10b36396bd98e45c9 27 | - name: github.com/hashicorp/hcl 28 | version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca 29 | subpackages: 30 | - hcl/ast 31 | - hcl/parser 32 | - hcl/scanner 33 | - hcl/strconv 34 | - hcl/token 35 | - json/parser 36 | - json/scanner 37 | - json/token 38 | - name: github.com/inconshreveable/mousetrap 39 | version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 40 | - name: github.com/jmhodges/levigo 41 | version: c42d9e0ca023e2198120196f842701bb4c55d7b9 42 | - name: github.com/kr/logfmt 43 | version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 44 | - name: github.com/magiconair/properties 45 | version: 51463bfca2576e06c62a8504b5c0f06d61312647 46 | - name: github.com/mitchellh/mapstructure 47 | version: cc8532a8e9a55ea36402aa21efdf403a60d34096 48 | - name: github.com/pelletier/go-buffruneio 49 | version: c37440a7cf42ac63b919c752ca73a85067e05992 50 | - name: github.com/pelletier/go-toml 51 | version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a 52 | - name: github.com/pkg/errors 53 | version: 645ef00459ed84a119197bfb8d8205042c6df63d 54 | - name: github.com/spf13/afero 55 | version: 9be650865eab0c12963d8753212f4f9c66cdcf12 56 | subpackages: 57 | - mem 58 | - name: github.com/spf13/cast 59 | version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 60 | - name: github.com/spf13/cobra 61 | version: 4cdb38c072b86bf795d2c81de50784d9fdd6eb77 62 | - name: github.com/spf13/jwalterweatherman 63 | version: 8f07c835e5cc1450c082fe3a439cf87b0cbb2d99 64 | - name: github.com/spf13/pflag 65 | version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 66 | - name: github.com/spf13/viper 67 | version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2 68 | - name: github.com/syndtr/goleveldb 69 | version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 70 | subpackages: 71 | - leveldb 72 | - leveldb/cache 73 | - leveldb/comparer 74 | - leveldb/errors 75 | - leveldb/filter 76 | - leveldb/iterator 77 | - leveldb/journal 78 | - leveldb/memdb 79 | - leveldb/opt 80 | - leveldb/storage 81 | - leveldb/table 82 | - leveldb/util 83 | - name: github.com/tendermint/abci 84 | version: 7f5f48b6b9ec3964de4b07b6c3cd05d7c91aeee5 85 | subpackages: 86 | - client 87 | - server 88 | - types 89 | - name: github.com/tendermint/ed25519 90 | version: 1f52c6f8b8a5c7908aff4497c186af344b428925 91 | subpackages: 92 | - edwards25519 93 | - extra25519 94 | - name: github.com/tendermint/go-crypto 95 | version: 7dff40942a64cdeefefa9446b2d104750b349f8a 96 | - name: github.com/tendermint/go-wire 97 | version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb 98 | subpackages: 99 | - data 100 | - name: github.com/tendermint/tmlibs 101 | version: efb56aaea7517220bb3f42ff87b8004d554a17ff 102 | subpackages: 103 | - common 104 | - db 105 | - log 106 | - merkle 107 | - test 108 | - name: golang.org/x/crypto 109 | version: c7af5bf2638a1164f2eb5467c39c6cffbd13a02e 110 | subpackages: 111 | - nacl/secretbox 112 | - openpgp/armor 113 | - openpgp/errors 114 | - poly1305 115 | - ripemd160 116 | - salsa20/salsa 117 | - name: golang.org/x/net 118 | version: feeb485667d1fdabe727840fe00adc22431bc86e 119 | subpackages: 120 | - context 121 | - http2 122 | - http2/hpack 123 | - idna 124 | - internal/timeseries 125 | - lex/httplex 126 | - trace 127 | - name: golang.org/x/sys 128 | version: e62c3de784db939836898e5c19ffd41bece347da 129 | subpackages: 130 | - unix 131 | - name: golang.org/x/text 132 | version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4 133 | subpackages: 134 | - secure/bidirule 135 | - transform 136 | - unicode/bidi 137 | - unicode/norm 138 | - name: google.golang.org/genproto 139 | version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 140 | subpackages: 141 | - googleapis/rpc/status 142 | - name: google.golang.org/grpc 143 | version: 844f573616520565fdc6fb4db242321b5456fd6d 144 | subpackages: 145 | - codes 146 | - credentials 147 | - grpclb/grpc_lb_v1 148 | - grpclog 149 | - internal 150 | - keepalive 151 | - metadata 152 | - naming 153 | - peer 154 | - stats 155 | - status 156 | - tap 157 | - transport 158 | - name: gopkg.in/yaml.v2 159 | version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b 160 | testImports: 161 | - name: github.com/davecgh/go-spew 162 | version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 163 | subpackages: 164 | - spew 165 | - name: github.com/pmezard/go-difflib 166 | version: d8ed2627bdf02c080bf22230dbb337003b7aba2d 167 | subpackages: 168 | - difflib 169 | - name: github.com/stretchr/testify 170 | version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 171 | subpackages: 172 | - assert 173 | - require 174 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/tendermint/merkleeyes 2 | import: 3 | - package: github.com/spf13/cobra 4 | - package: github.com/spf13/viper 5 | - package: github.com/tendermint/abci 6 | version: develop 7 | subpackages: 8 | - client 9 | - server 10 | - types 11 | - package: github.com/tendermint/go-wire 12 | version: develop 13 | - package: github.com/tendermint/tmlibs 14 | version: develop 15 | subpackages: 16 | - common 17 | - db 18 | - merkle 19 | - package: golang.org/x/crypto 20 | subpackages: 21 | - ripemd160 22 | testImport: 23 | - package: github.com/stretchr/testify 24 | version: ^1.1.4 25 | subpackages: 26 | - assert 27 | - require 28 | -------------------------------------------------------------------------------- /iavl/PERFORMANCE.md: -------------------------------------------------------------------------------- 1 | # Performance 2 | 3 | After some discussion with Jae on the usability, it seems performance is a big concern. If every write takes around 1ms, that puts a serious upper limit on the speed of the consensus engine (especially since with the check/tx dichotomy, we need at least two writes (to cache, only one to disk) and likely two or more queries to handle any transaction). 4 | 5 | As Jae notes: for CheckTx, a copy of IAVLTree doesn't need to be saved. During CheckTx it'll load inner nodes into the cache. The cache is shared w/ the AppendTx state IAVLTree, so during AppendTx we should save some time. There would only be 1 set of writes. Also, there's quite a bit of free time in between blocks as provided by Tendermint, during which CheckTx can run priming the cache, so hopefully this helps as well. 6 | 7 | Jae: That said, I'm not sure exactly what the tx throughput would be during normal running times. I'm hoping that we can have e.g. 3 second blocks w/ say over a hundred txs per sec per block w/ 1 million items. That will get us through for some time, but that time is limited. 8 | 9 | Ethan: I agree, and think this works now with goleveldb backing on most host machines. For public chains, maybe it is desired to push 1000 tx every 3 sec to a block, with a db size of 1 billion items. 10x the throughput with 1000x the data. That could be a long-term goal, and would scale to the cosmos and beyond. 10 | 11 | ## Plan 12 | 13 | For any goal, we need some clear steps. 14 | 15 | 1) Cleanup code, and write some more benchmark cases to capture "realistic" usage 16 | 2) Run tests on various hardware to see the best performing backing stores 17 | 3) Do profiling on the best performance to see if there are any easy performance gains 18 | 4) (Possibly) Write another implementation of merkle.Tree to improve all the memory overhead, consider CPU cache, etc.... 19 | 5) (Possibly) Write another backend datastore to persist the tree in a more efficient way 20 | 21 | The rest of this document is the planned or completed actions for the above-listed steps. 22 | 23 | ## Cleanup 24 | 25 | Done in branch `cleanup_deps`: 26 | * Fixed up dependeny management (tmlibs/db etc in glide/vendor) 27 | * Updated Makefile (test, bench, get_deps) 28 | * Fixed broken code - `looper.go` and one benchmark didn't run 29 | 30 | Benchmarks should be parameterized on: 31 | 1) storage implementation 32 | 2) initial data size 33 | 3) length of keys 34 | 4) length of data 35 | 5) block size (frequency of copy/hash...) 36 | Thus, we would see the same benchmark run against memdb with 100K items, goleveldb with 100K, leveldb with 100K, memdb with 10K, goleveldb with 10K... 37 | 38 | Scenarios to run after db is set up. 39 | * Pure query time (known/hits, vs. random/misses) 40 | * Write timing (known/updates, vs. random/inserts) 41 | * Delete timing (existing keys only) 42 | * TMSP Usage: 43 | * For each block size: 44 | * 2x copy "last commit" -> check and real 45 | * repeat for each tx: 46 | * (50% update + 50% insert?) 47 | * query + insert/update in check 48 | * query + insert/update in real 49 | * get hash 50 | * save real 51 | * real -> "last commit" 52 | 53 | 54 | ## Benchmarks 55 | 56 | After writing the benchmarks, we can run them under various environments and store the results under benchmarks directory. Some useful environments to test: 57 | 58 | * Dev machines 59 | * Digital ocean small/large machine 60 | * Various AWS setups 61 | 62 | Please run the benchmark on more machines and add the result. Just type: `make record` in the directory and wait a (long) while (with little other load on the machine). 63 | 64 | This will require also a quick setup script to install go and run tests in these environments. Maybe some scripts even. Also, this will produce a lot of files and we may have to graph them to see something useful... 65 | 66 | But for starting, my laptop, and one digital ocean and one aws server should be sufficient. At least to find the winner, before profiling. 67 | 68 | 69 | ## Profiling 70 | 71 | Once we figure out which current implementation looks fastest, let's profile it to make it even faster. It is great to optimize the memdb code to really speed up the hashing and tree-building logic. And then focus on the backend implementation to optimize the disk storage, which will be the next major pain point. 72 | 73 | Some guides: 74 | 75 | * [Profiling benchmarks locally](https://medium.com/@hackintoshrao/daily-code-optimization-using-benchmarks-and-profiling-in-golang-gophercon-india-2016-talk-874c8b4dc3c5#.jmnd8w2qr) 76 | * [On optimizing memory](https://signalfx.com/blog/a-pattern-for-optimizing-go-2/) 77 | * [Profiling running programs](http://blog.ralch.com/tutorial/golang-performance-and-memory-analysis/) 78 | * [Dave Chenny's profiler pkg](https://github.com/pkg/profile) 79 | 80 | Some ideas for speedups: 81 | 82 | * [Speedup SHA256 100x on ARM](https://blog.minio.io/accelerating-sha256-by-100x-in-golang-on-arm-1517225f5ff4#.pybt7bb3w) 83 | * [Faster SHA256 golang implementation](https://github.com/minio/sha256-simd) 84 | * [Data structure alignment](http://stackoverflow.com/questions/39063530/optimising-datastructure-word-alignment-padding-in-golang) 85 | * [Slice alignment](http://blog.chewxy.com/2016/07/25/on-the-memory-alignment-of-go-slice-values/) 86 | * [Tool to analyze your structs](https://github.com/dominikh/go-structlayout) 87 | 88 | ## Tree Re-implementation 89 | 90 | If we want to copy lots of objects, it becomes better to think of using memcpy on large (eg. 4-16KB) buffers than copying individual structs. We also could allocate arrays of structs and align them to remove a lot of memory management and gc overhead. That means going down to some C-level coding... 91 | 92 | Some links for thought: 93 | 94 | * [Array representation of a binary tree](http://www.cse.hut.fi/en/research/SVG/TRAKLA2/tutorials/heap_tutorial/taulukkona.html) 95 | * [Memcpy buffer size timing](http://stackoverflow.com/questions/21038965/why-does-the-speed-of-memcpy-drop-dramatically-every-4kb) 96 | * [Calling memcpy from go](https://github.com/jsgilmore/shm/blob/master/memcpy.go) 97 | * [Unsafe docs](https://godoc.org/unsafe) 98 | * [...and how to use it](https://copyninja.info/blog/workaround-gotypesystems.html) 99 | * [Or maybe just plain copy...](https://godoc.org/builtin#copy) 100 | 101 | ## Backend implementation 102 | 103 | Storing each link in the tree in leveldb treats each node as an isolated item. Since we know some usage patterns (when a parent is hit, very likely one child will be hit), we could try to organize the memory and disk location of the nodes ourselves to make it more efficient. Or course, this could be a long, slippery slope. 104 | 105 | Inspired by the [Array representation](http://www.cse.hut.fi/en/research/SVG/TRAKLA2/tutorials/heap_tutorial/taulukkona.html) link above, we could consider other layouts for the nodes. For example, rather than store them alone, or the entire tree in one big array, the nodes could be placed in groups of 15 based on the parent (parent and 3 generations of children). Then we have 4 levels before jumping to another location. Maybe we just store this larger chunk as one leveldb location, or really try to do the mmap ourselves... 106 | 107 | In any case, assuming around 100 bytes for one non-leaf node (3 sha hashes, plus prefix, plus other data), 15 nodes would be a little less than 2K, maybe even go one more level to 31 nodes and 3-4KB, where we could take best advantage of the memory/disk page size. 108 | 109 | Some links for thought: 110 | 111 | * [Memory mapped files](https://github.com/edsrzf/mmap-go) 112 | -------------------------------------------------------------------------------- /iavl/README.md: -------------------------------------------------------------------------------- 1 | ## IAVL+ Tree 2 | 3 | A snapshottable (immutable) AVL+ tree for persistent data 4 | 5 | **Note** Please make sure you read the [caveat](https://github.com/tendermint/merkleeyes/blob/develop/iavl/iavl_tree.go#L34-L40) on `Copy`. If you have a backing DB and call `Save` to persist the state, all existing copies become potentially invalid and may panic if used. For safe coding, you must throw away all references upon save, and `Copy` again from the new, committed state. 6 | 7 | The purpose of this data structure is to provide persistent storage for key-value pairs (say to store account balances) such that a deterministic merkle root hash can be computed. The tree is balanced using a variant of the [AVL algortihm](http://en.wikipedia.org/wiki/AVL_tree) so all operations are O(log(n)). 8 | 9 | Nodes of this tree are immutable and indexed by its hash. Thus any node serves as an immutable snapshot which lets us stage uncommitted transactions from the mempool cheaply, and we can instantly roll back to the last committed state to process transactions of a newly committed block (which may not be the same set of transactions as those from the mempool). 10 | 11 | In an AVL tree, the heights of the two child subtrees of any node differ by at most one. Whenever this condition is violated upon an update, the tree is rebalanced by creating O(log(n)) new nodes that point to unmodified nodes of the old tree. In the original AVL algorithm, inner nodes can also hold key-value pairs. The AVL+ algorithm (note the plus) modifies the AVL algorithm to keep all values on leaf nodes, while only using branch-nodes to store keys. This simplifies the algorithm while keeping the merkle hash trail short. 12 | 13 | In Ethereum, the analog is [Patricia tries](http://en.wikipedia.org/wiki/Radix_tree). There are tradeoffs. Keys do not need to be hashed prior to insertion in IAVL+ trees, so this provides faster iteration in the key space which may benefit some applications. The logic is simpler to implement, requiring only two types of nodes -- inner nodes and leaf nodes. On the other hand, while IAVL+ trees provide a deterministic merkle root hash, it depends on the order of transactions. In practice this shouldn't be a problem, since you can efficiently encode the tree structure when serializing the tree contents. 14 | -------------------------------------------------------------------------------- /iavl/circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | environment: 3 | GO15VENDOREXPERIMENT: 1 4 | 5 | dependencies: 6 | pre: 7 | - make get_deps 8 | 9 | test: 10 | override: 11 | - make test 12 | -------------------------------------------------------------------------------- /iavl/iavl_node.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | 7 | "golang.org/x/crypto/ripemd160" 8 | 9 | . "github.com/tendermint/tmlibs/common" 10 | "github.com/tendermint/go-wire" 11 | ) 12 | 13 | // Node 14 | 15 | type IAVLNode struct { 16 | key []byte 17 | value []byte 18 | height int8 19 | size int 20 | hash []byte 21 | leftHash []byte 22 | leftNode *IAVLNode 23 | rightHash []byte 24 | rightNode *IAVLNode 25 | persisted bool 26 | } 27 | 28 | func NewIAVLNode(key []byte, value []byte) *IAVLNode { 29 | return &IAVLNode{ 30 | key: key, 31 | value: value, 32 | height: 0, 33 | size: 1, 34 | } 35 | } 36 | 37 | // NOTE: The hash is not saved or set. The caller should set the hash afterwards. 38 | // (Presumably the caller already has the hash) 39 | func MakeIAVLNode(buf []byte, t *IAVLTree) (node *IAVLNode, err error) { 40 | node = &IAVLNode{} 41 | 42 | // node header 43 | node.height = int8(buf[0]) 44 | buf = buf[1:] 45 | var n int 46 | node.size, n, err = wire.GetVarint(buf) 47 | if err != nil { 48 | return nil, err 49 | } 50 | buf = buf[n:] 51 | node.key, n, err = wire.GetByteSlice(buf) 52 | if err != nil { 53 | return nil, err 54 | } 55 | buf = buf[n:] 56 | if node.height == 0 { 57 | // value 58 | node.value, n, err = wire.GetByteSlice(buf) 59 | if err != nil { 60 | return nil, err 61 | } 62 | buf = buf[n:] 63 | } else { 64 | // children 65 | leftHash, n, err := wire.GetByteSlice(buf) 66 | if err != nil { 67 | return nil, err 68 | } 69 | buf = buf[n:] 70 | rightHash, n, err := wire.GetByteSlice(buf) 71 | if err != nil { 72 | return nil, err 73 | } 74 | buf = buf[n:] 75 | node.leftHash = leftHash 76 | node.rightHash = rightHash 77 | } 78 | return node, nil 79 | } 80 | 81 | func (node *IAVLNode) _copy() *IAVLNode { 82 | if node.height == 0 { 83 | PanicSanity("Why are you copying a value node?") 84 | } 85 | return &IAVLNode{ 86 | key: node.key, 87 | height: node.height, 88 | size: node.size, 89 | hash: nil, // Going to be mutated anyways. 90 | leftHash: node.leftHash, 91 | leftNode: node.leftNode, 92 | rightHash: node.rightHash, 93 | rightNode: node.rightNode, 94 | persisted: false, // Going to be mutated, so it can't already be persisted. 95 | } 96 | } 97 | 98 | func (node *IAVLNode) has(t *IAVLTree, key []byte) (has bool) { 99 | if bytes.Compare(node.key, key) == 0 { 100 | return true 101 | } 102 | if node.height == 0 { 103 | return false 104 | } else { 105 | if bytes.Compare(key, node.key) < 0 { 106 | return node.getLeftNode(t).has(t, key) 107 | } else { 108 | return node.getRightNode(t).has(t, key) 109 | } 110 | } 111 | } 112 | 113 | func (node *IAVLNode) get(t *IAVLTree, key []byte) (index int, value []byte, exists bool) { 114 | if node.height == 0 { 115 | cmp := bytes.Compare(node.key, key) 116 | if cmp == 0 { 117 | return 0, node.value, true 118 | } else if cmp == -1 { 119 | return 1, nil, false 120 | } else { 121 | return 0, nil, false 122 | } 123 | } else { 124 | if bytes.Compare(key, node.key) < 0 { 125 | return node.getLeftNode(t).get(t, key) 126 | } else { 127 | rightNode := node.getRightNode(t) 128 | index, value, exists = rightNode.get(t, key) 129 | index += node.size - rightNode.size 130 | return index, value, exists 131 | } 132 | } 133 | } 134 | 135 | func (node *IAVLNode) getByIndex(t *IAVLTree, index int) (key []byte, value []byte) { 136 | if node.height == 0 { 137 | if index == 0 { 138 | return node.key, node.value 139 | } else { 140 | PanicSanity("getByIndex asked for invalid index") 141 | return nil, nil 142 | } 143 | } else { 144 | // TODO: could improve this by storing the 145 | // sizes as well as left/right hash. 146 | leftNode := node.getLeftNode(t) 147 | if index < leftNode.size { 148 | return leftNode.getByIndex(t, index) 149 | } else { 150 | return node.getRightNode(t).getByIndex(t, index-leftNode.size) 151 | } 152 | } 153 | } 154 | 155 | // NOTE: sets hashes recursively 156 | func (node *IAVLNode) hashWithCount(t *IAVLTree) ([]byte, int) { 157 | if node.hash != nil { 158 | return node.hash, 0 159 | } 160 | 161 | hasher := ripemd160.New() 162 | buf := new(bytes.Buffer) 163 | _, hashCount, err := node.writeHashBytes(t, buf) 164 | if err != nil { 165 | PanicCrisis(err) 166 | } 167 | hasher.Write(buf.Bytes()) 168 | node.hash = hasher.Sum(nil) 169 | 170 | return node.hash, hashCount + 1 171 | } 172 | 173 | // NOTE: sets hashes recursively 174 | func (node *IAVLNode) writeHashBytes(t *IAVLTree, w io.Writer) (n int, hashCount int, err error) { 175 | // height & size 176 | wire.WriteInt8(node.height, w, &n, &err) 177 | wire.WriteVarint(node.size, w, &n, &err) 178 | // key is not written for inner nodes, unlike writePersistBytes 179 | 180 | if node.height == 0 { 181 | // key & value 182 | wire.WriteByteSlice(node.key, w, &n, &err) 183 | wire.WriteByteSlice(node.value, w, &n, &err) 184 | } else { 185 | // left 186 | if node.leftNode != nil { 187 | leftHash, leftCount := node.leftNode.hashWithCount(t) 188 | node.leftHash = leftHash 189 | hashCount += leftCount 190 | } 191 | if node.leftHash == nil { 192 | PanicSanity("node.leftHash was nil in writeHashBytes") 193 | } 194 | wire.WriteByteSlice(node.leftHash, w, &n, &err) 195 | // right 196 | if node.rightNode != nil { 197 | rightHash, rightCount := node.rightNode.hashWithCount(t) 198 | node.rightHash = rightHash 199 | hashCount += rightCount 200 | } 201 | if node.rightHash == nil { 202 | PanicSanity("node.rightHash was nil in writeHashBytes") 203 | } 204 | wire.WriteByteSlice(node.rightHash, w, &n, &err) 205 | } 206 | return 207 | } 208 | 209 | // NOTE: clears leftNode/rigthNode recursively 210 | // NOTE: sets hashes recursively 211 | func (node *IAVLNode) save(t *IAVLTree) { 212 | if node.hash == nil { 213 | node.hash, _ = node.hashWithCount(t) 214 | } 215 | if node.persisted { 216 | return 217 | } 218 | 219 | // save children 220 | if node.leftNode != nil { 221 | node.leftNode.save(t) 222 | node.leftNode = nil 223 | } 224 | if node.rightNode != nil { 225 | node.rightNode.save(t) 226 | node.rightNode = nil 227 | } 228 | 229 | // save node 230 | t.ndb.SaveNode(t, node) 231 | return 232 | } 233 | 234 | // NOTE: sets hashes recursively 235 | func (node *IAVLNode) writePersistBytes(t *IAVLTree, w io.Writer) (n int, err error) { 236 | // node header 237 | wire.WriteInt8(node.height, w, &n, &err) 238 | wire.WriteVarint(node.size, w, &n, &err) 239 | // key (unlike writeHashBytes, key is written for inner nodes) 240 | wire.WriteByteSlice(node.key, w, &n, &err) 241 | 242 | if node.height == 0 { 243 | // value 244 | wire.WriteByteSlice(node.value, w, &n, &err) 245 | } else { 246 | // left 247 | if node.leftHash == nil { 248 | PanicSanity("node.leftHash was nil in writePersistBytes") 249 | } 250 | wire.WriteByteSlice(node.leftHash, w, &n, &err) 251 | // right 252 | if node.rightHash == nil { 253 | PanicSanity("node.rightHash was nil in writePersistBytes") 254 | } 255 | wire.WriteByteSlice(node.rightHash, w, &n, &err) 256 | } 257 | return 258 | } 259 | 260 | func (node *IAVLNode) set(t *IAVLTree, key []byte, value []byte) (newSelf *IAVLNode, updated bool) { 261 | if node.height == 0 { 262 | cmp := bytes.Compare(key, node.key) 263 | if cmp < 0 { 264 | return &IAVLNode{ 265 | key: node.key, 266 | height: 1, 267 | size: 2, 268 | leftNode: NewIAVLNode(key, value), 269 | rightNode: node, 270 | }, false 271 | } else if cmp == 0 { 272 | removeOrphan(t, node) 273 | return NewIAVLNode(key, value), true 274 | } else { 275 | return &IAVLNode{ 276 | key: key, 277 | height: 1, 278 | size: 2, 279 | leftNode: node, 280 | rightNode: NewIAVLNode(key, value), 281 | }, false 282 | } 283 | } else { 284 | removeOrphan(t, node) 285 | node = node._copy() 286 | if bytes.Compare(key, node.key) < 0 { 287 | node.leftNode, updated = node.getLeftNode(t).set(t, key, value) 288 | node.leftHash = nil // leftHash is yet unknown 289 | } else { 290 | node.rightNode, updated = node.getRightNode(t).set(t, key, value) 291 | node.rightHash = nil // rightHash is yet unknown 292 | } 293 | if updated { 294 | return node, updated 295 | } else { 296 | node.calcHeightAndSize(t) 297 | return node.balance(t), updated 298 | } 299 | } 300 | } 301 | 302 | // newHash/newNode: The new hash or node to replace node after remove. 303 | // newKey: new leftmost leaf key for tree after successfully removing 'key' if changed. 304 | // value: removed value. 305 | func (node *IAVLNode) remove(t *IAVLTree, key []byte) ( 306 | newHash []byte, newNode *IAVLNode, newKey []byte, value []byte, removed bool) { 307 | if node.height == 0 { 308 | if bytes.Compare(key, node.key) == 0 { 309 | removeOrphan(t, node) 310 | return nil, nil, nil, node.value, true 311 | } else { 312 | return node.hash, node, nil, nil, false 313 | } 314 | } else { 315 | if bytes.Compare(key, node.key) < 0 { 316 | var newLeftHash []byte 317 | var newLeftNode *IAVLNode 318 | newLeftHash, newLeftNode, newKey, value, removed = node.getLeftNode(t).remove(t, key) 319 | if !removed { 320 | return node.hash, node, nil, value, false 321 | } else if newLeftHash == nil && newLeftNode == nil { // left node held value, was removed 322 | return node.rightHash, node.rightNode, node.key, value, true 323 | } 324 | removeOrphan(t, node) 325 | node = node._copy() 326 | node.leftHash, node.leftNode = newLeftHash, newLeftNode 327 | node.calcHeightAndSize(t) 328 | node = node.balance(t) 329 | return node.hash, node, newKey, value, true 330 | } else { 331 | var newRightHash []byte 332 | var newRightNode *IAVLNode 333 | newRightHash, newRightNode, newKey, value, removed = node.getRightNode(t).remove(t, key) 334 | if !removed { 335 | return node.hash, node, nil, value, false 336 | } else if newRightHash == nil && newRightNode == nil { // right node held value, was removed 337 | return node.leftHash, node.leftNode, nil, value, true 338 | } 339 | removeOrphan(t, node) 340 | node = node._copy() 341 | node.rightHash, node.rightNode = newRightHash, newRightNode 342 | if newKey != nil { 343 | node.key = newKey 344 | } 345 | node.calcHeightAndSize(t) 346 | node = node.balance(t) 347 | return node.hash, node, nil, value, true 348 | } 349 | } 350 | } 351 | 352 | func (node *IAVLNode) getLeftNode(t *IAVLTree) *IAVLNode { 353 | if node.leftNode != nil { 354 | return node.leftNode 355 | } else { 356 | return t.ndb.GetNode(t, node.leftHash) 357 | } 358 | } 359 | 360 | func (node *IAVLNode) getRightNode(t *IAVLTree) *IAVLNode { 361 | if node.rightNode != nil { 362 | return node.rightNode 363 | } else { 364 | return t.ndb.GetNode(t, node.rightHash) 365 | } 366 | } 367 | 368 | // NOTE: overwrites node 369 | // TODO: optimize balance & rotate 370 | func (node *IAVLNode) rotateRight(t *IAVLTree) *IAVLNode { 371 | node = node._copy() 372 | l := node.getLeftNode(t) 373 | removeOrphan(t, l) 374 | _l := l._copy() 375 | 376 | _lrHash, _lrCached := _l.rightHash, _l.rightNode 377 | _l.rightHash, _l.rightNode = node.hash, node 378 | node.leftHash, node.leftNode = _lrHash, _lrCached 379 | 380 | node.calcHeightAndSize(t) 381 | _l.calcHeightAndSize(t) 382 | 383 | return _l 384 | } 385 | 386 | // NOTE: overwrites node 387 | // TODO: optimize balance & rotate 388 | func (node *IAVLNode) rotateLeft(t *IAVLTree) *IAVLNode { 389 | node = node._copy() 390 | r := node.getRightNode(t) 391 | removeOrphan(t, r) 392 | _r := r._copy() 393 | 394 | _rlHash, _rlCached := _r.leftHash, _r.leftNode 395 | _r.leftHash, _r.leftNode = node.hash, node 396 | node.rightHash, node.rightNode = _rlHash, _rlCached 397 | 398 | node.calcHeightAndSize(t) 399 | _r.calcHeightAndSize(t) 400 | 401 | return _r 402 | } 403 | 404 | // NOTE: mutates height and size 405 | func (node *IAVLNode) calcHeightAndSize(t *IAVLTree) { 406 | node.height = maxInt8(node.getLeftNode(t).height, node.getRightNode(t).height) + 1 407 | node.size = node.getLeftNode(t).size + node.getRightNode(t).size 408 | } 409 | 410 | func (node *IAVLNode) calcBalance(t *IAVLTree) int { 411 | return int(node.getLeftNode(t).height) - int(node.getRightNode(t).height) 412 | } 413 | 414 | // NOTE: assumes that node can be modified 415 | // TODO: optimize balance & rotate 416 | func (node *IAVLNode) balance(t *IAVLTree) (newSelf *IAVLNode) { 417 | if node.persisted { 418 | panic("Unexpected balance() call on persisted node") 419 | } 420 | balance := node.calcBalance(t) 421 | if balance > 1 { 422 | if node.getLeftNode(t).calcBalance(t) >= 0 { 423 | // Left Left Case 424 | return node.rotateRight(t) 425 | } else { 426 | // Left Right Case 427 | // node = node._copy() 428 | left := node.getLeftNode(t) 429 | removeOrphan(t, left) 430 | node.leftHash, node.leftNode = nil, left.rotateLeft(t) 431 | //node.calcHeightAndSize() 432 | return node.rotateRight(t) 433 | } 434 | } 435 | if balance < -1 { 436 | if node.getRightNode(t).calcBalance(t) <= 0 { 437 | // Right Right Case 438 | return node.rotateLeft(t) 439 | } else { 440 | // Right Left Case 441 | // node = node._copy() 442 | right := node.getRightNode(t) 443 | removeOrphan(t, right) 444 | node.rightHash, node.rightNode = nil, right.rotateRight(t) 445 | //node.calcHeightAndSize() 446 | return node.rotateLeft(t) 447 | } 448 | } 449 | // Nothing changed 450 | return node 451 | } 452 | 453 | // traverse is a wrapper over traverseInRange when we want the whole tree 454 | func (node *IAVLNode) traverse(t *IAVLTree, ascending bool, cb func(*IAVLNode) bool) bool { 455 | return node.traverseInRange(t, nil, nil, ascending, cb) 456 | } 457 | 458 | func (node *IAVLNode) traverseInRange(t *IAVLTree, start, end []byte, ascending bool, cb func(*IAVLNode) bool) bool { 459 | afterStart := (start == nil || bytes.Compare(start, node.key) <= 0) 460 | beforeEnd := (end == nil || bytes.Compare(node.key, end) <= 0) 461 | 462 | stop := false 463 | if afterStart && beforeEnd { 464 | // IterateRange ignores this if not leaf 465 | stop = cb(node) 466 | } 467 | if stop { 468 | return stop 469 | } 470 | 471 | if node.height > 0 { 472 | if ascending { 473 | // check lower nodes, then higher 474 | if afterStart { 475 | stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, cb) 476 | } 477 | if stop { 478 | return stop 479 | } 480 | if beforeEnd { 481 | stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, cb) 482 | } 483 | } else { 484 | // check the higher nodes first 485 | if beforeEnd { 486 | stop = node.getRightNode(t).traverseInRange(t, start, end, ascending, cb) 487 | } 488 | if stop { 489 | return stop 490 | } 491 | if afterStart { 492 | stop = node.getLeftNode(t).traverseInRange(t, start, end, ascending, cb) 493 | } 494 | } 495 | } 496 | 497 | return stop 498 | } 499 | 500 | // Only used in testing... 501 | func (node *IAVLNode) lmd(t *IAVLTree) *IAVLNode { 502 | if node.height == 0 { 503 | return node 504 | } 505 | return node.getLeftNode(t).lmd(t) 506 | } 507 | 508 | // Only used in testing... 509 | func (node *IAVLNode) rmd(t *IAVLTree) *IAVLNode { 510 | if node.height == 0 { 511 | return node 512 | } 513 | return node.getRightNode(t).rmd(t) 514 | } 515 | 516 | //---------------------------------------- 517 | 518 | func removeOrphan(t *IAVLTree, node *IAVLNode) { 519 | if !node.persisted { 520 | return 521 | } 522 | if t.ndb == nil { 523 | return 524 | } 525 | t.ndb.RemoveNode(t, node) 526 | } 527 | -------------------------------------------------------------------------------- /iavl/iavl_proof.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "bytes" 5 | 6 | "golang.org/x/crypto/ripemd160" 7 | 8 | . "github.com/tendermint/tmlibs/common" 9 | "github.com/tendermint/go-wire" 10 | ) 11 | 12 | const proofLimit = 1 << 16 // 64 KB 13 | 14 | type IAVLProof struct { 15 | LeafHash []byte 16 | InnerNodes []IAVLProofInnerNode 17 | RootHash []byte 18 | } 19 | 20 | func (proof *IAVLProof) Verify(key []byte, value []byte, root []byte) bool { 21 | if !bytes.Equal(proof.RootHash, root) { 22 | return false 23 | } 24 | leafNode := IAVLProofLeafNode{KeyBytes: key, ValueBytes: value} 25 | leafHash := leafNode.Hash() 26 | if !bytes.Equal(leafHash, proof.LeafHash) { 27 | return false 28 | } 29 | hash := leafHash 30 | for _, branch := range proof.InnerNodes { 31 | hash = branch.Hash(hash) 32 | } 33 | return bytes.Equal(proof.RootHash, hash) 34 | } 35 | 36 | // Please leave this here! I use it in light-client to fulfill an interface 37 | func (proof *IAVLProof) Root() []byte { 38 | return proof.RootHash 39 | } 40 | 41 | // ReadProof will deserialize a IAVLProof from bytes 42 | func ReadProof(data []byte) (*IAVLProof, error) { 43 | // TODO: make go-wire never panic 44 | n, err := int(0), error(nil) 45 | proof := wire.ReadBinary(&IAVLProof{}, bytes.NewBuffer(data), proofLimit, &n, &err).(*IAVLProof) 46 | return proof, err 47 | } 48 | 49 | type IAVLProofInnerNode struct { 50 | Height int8 51 | Size int 52 | Left []byte 53 | Right []byte 54 | } 55 | 56 | func (branch IAVLProofInnerNode) Hash(childHash []byte) []byte { 57 | hasher := ripemd160.New() 58 | buf := new(bytes.Buffer) 59 | n, err := int(0), error(nil) 60 | wire.WriteInt8(branch.Height, buf, &n, &err) 61 | wire.WriteVarint(branch.Size, buf, &n, &err) 62 | if len(branch.Left) == 0 { 63 | wire.WriteByteSlice(childHash, buf, &n, &err) 64 | wire.WriteByteSlice(branch.Right, buf, &n, &err) 65 | } else { 66 | wire.WriteByteSlice(branch.Left, buf, &n, &err) 67 | wire.WriteByteSlice(childHash, buf, &n, &err) 68 | } 69 | if err != nil { 70 | PanicCrisis(Fmt("Failed to hash IAVLProofInnerNode: %v", err)) 71 | } 72 | // fmt.Printf("InnerNode hash bytes: %X\n", buf.Bytes()) 73 | hasher.Write(buf.Bytes()) 74 | return hasher.Sum(nil) 75 | } 76 | 77 | type IAVLProofLeafNode struct { 78 | KeyBytes []byte 79 | ValueBytes []byte 80 | } 81 | 82 | func (leaf IAVLProofLeafNode) Hash() []byte { 83 | hasher := ripemd160.New() 84 | buf := new(bytes.Buffer) 85 | n, err := int(0), error(nil) 86 | wire.WriteInt8(0, buf, &n, &err) 87 | wire.WriteVarint(1, buf, &n, &err) 88 | wire.WriteByteSlice(leaf.KeyBytes, buf, &n, &err) 89 | wire.WriteByteSlice(leaf.ValueBytes, buf, &n, &err) 90 | if err != nil { 91 | PanicCrisis(Fmt("Failed to hash IAVLProofLeafNode: %v", err)) 92 | } 93 | // fmt.Printf("LeafNode hash bytes: %X\n", buf.Bytes()) 94 | hasher.Write(buf.Bytes()) 95 | return hasher.Sum(nil) 96 | } 97 | 98 | func (node *IAVLNode) constructProof(t *IAVLTree, key []byte, valuePtr *[]byte, proof *IAVLProof) (exists bool) { 99 | if node.height == 0 { 100 | if bytes.Compare(node.key, key) == 0 { 101 | *valuePtr = node.value 102 | proof.LeafHash = node.hash 103 | return true 104 | } else { 105 | return false 106 | } 107 | } else { 108 | if bytes.Compare(key, node.key) < 0 { 109 | exists := node.getLeftNode(t).constructProof(t, key, valuePtr, proof) 110 | if !exists { 111 | return false 112 | } 113 | branch := IAVLProofInnerNode{ 114 | Height: node.height, 115 | Size: node.size, 116 | Left: nil, 117 | Right: node.getRightNode(t).hash, 118 | } 119 | proof.InnerNodes = append(proof.InnerNodes, branch) 120 | return true 121 | } else { 122 | exists := node.getRightNode(t).constructProof(t, key, valuePtr, proof) 123 | if !exists { 124 | return false 125 | } 126 | branch := IAVLProofInnerNode{ 127 | Height: node.height, 128 | Size: node.size, 129 | Left: node.getLeftNode(t).hash, 130 | Right: nil, 131 | } 132 | proof.InnerNodes = append(proof.InnerNodes, branch) 133 | return true 134 | } 135 | } 136 | } 137 | 138 | // Returns nil, nil if key is not in tree. 139 | func (t *IAVLTree) ConstructProof(key []byte) (value []byte, proof *IAVLProof) { 140 | if t.root == nil { 141 | return nil, nil 142 | } 143 | t.root.hashWithCount(t) // Ensure that all hashes are calculated. 144 | proof = &IAVLProof{ 145 | RootHash: t.root.hash, 146 | } 147 | exists := t.root.constructProof(t, key, &value, proof) 148 | if exists { 149 | return value, proof 150 | } else { 151 | return nil, nil 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /iavl/iavl_test.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | mrand "math/rand" 7 | "sort" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | "github.com/tendermint/tmlibs/db" 12 | "github.com/tendermint/go-wire" 13 | . "github.com/tendermint/tmlibs/common" 14 | . "github.com/tendermint/tmlibs/test" 15 | 16 | "runtime" 17 | "testing" 18 | ) 19 | 20 | const testReadLimit = 1 << 20 // Some reasonable limit for wire.Read*() lmt 21 | 22 | func randstr(length int) string { 23 | return RandStr(length) 24 | } 25 | 26 | func i2b(i int) []byte { 27 | bz := make([]byte, 4) 28 | wire.PutInt32(bz, int32(i)) 29 | return bz 30 | } 31 | 32 | func b2i(bz []byte) int { 33 | i := wire.GetInt32(bz) 34 | return int(i) 35 | } 36 | 37 | // Convenience for a new node 38 | func N(l, r interface{}) *IAVLNode { 39 | var left, right *IAVLNode 40 | if _, ok := l.(*IAVLNode); ok { 41 | left = l.(*IAVLNode) 42 | } else { 43 | left = NewIAVLNode(i2b(l.(int)), nil) 44 | } 45 | if _, ok := r.(*IAVLNode); ok { 46 | right = r.(*IAVLNode) 47 | } else { 48 | right = NewIAVLNode(i2b(r.(int)), nil) 49 | } 50 | 51 | n := &IAVLNode{ 52 | key: right.lmd(nil).key, 53 | value: nil, 54 | leftNode: left, 55 | rightNode: right, 56 | } 57 | n.calcHeightAndSize(nil) 58 | return n 59 | } 60 | 61 | // Setup a deep node 62 | func T(n *IAVLNode) *IAVLTree { 63 | d := db.NewDB("test", db.MemDBBackendStr, "") 64 | t := NewIAVLTree(0, d) 65 | 66 | n.hashWithCount(t) 67 | t.root = n 68 | return t 69 | } 70 | 71 | // Convenience for simple printing of keys & tree structure 72 | func P(n *IAVLNode) string { 73 | if n.height == 0 { 74 | return fmt.Sprintf("%v", b2i(n.key)) 75 | } else { 76 | return fmt.Sprintf("(%v %v)", P(n.leftNode), P(n.rightNode)) 77 | } 78 | } 79 | 80 | func TestBasic(t *testing.T) { 81 | var tree *IAVLTree = NewIAVLTree(0, nil) 82 | var up bool 83 | up = tree.Set([]byte("1"), []byte("one")) 84 | if up { 85 | t.Error("Did not expect an update (should have been create)") 86 | } 87 | up = tree.Set([]byte("2"), []byte("two")) 88 | if up { 89 | t.Error("Did not expect an update (should have been create)") 90 | } 91 | up = tree.Set([]byte("2"), []byte("TWO")) 92 | if !up { 93 | t.Error("Expected an update") 94 | } 95 | up = tree.Set([]byte("5"), []byte("five")) 96 | if up { 97 | t.Error("Did not expect an update (should have been create)") 98 | } 99 | 100 | // Test 0x00 101 | { 102 | idx, val, exists := tree.Get([]byte{0x00}) 103 | if exists { 104 | t.Errorf("Expected no value to exist") 105 | } 106 | if idx != 0 { 107 | t.Errorf("Unexpected idx %x", idx) 108 | } 109 | if string(val) != "" { 110 | t.Errorf("Unexpected value %v", string(val)) 111 | } 112 | } 113 | 114 | // Test "1" 115 | { 116 | idx, val, exists := tree.Get([]byte("1")) 117 | if !exists { 118 | t.Errorf("Expected value to exist") 119 | } 120 | if idx != 0 { 121 | t.Errorf("Unexpected idx %x", idx) 122 | } 123 | if string(val) != "one" { 124 | t.Errorf("Unexpected value %v", string(val)) 125 | } 126 | } 127 | 128 | // Test "2" 129 | { 130 | idx, val, exists := tree.Get([]byte("2")) 131 | if !exists { 132 | t.Errorf("Expected value to exist") 133 | } 134 | if idx != 1 { 135 | t.Errorf("Unexpected idx %x", idx) 136 | } 137 | if string(val) != "TWO" { 138 | t.Errorf("Unexpected value %v", string(val)) 139 | } 140 | } 141 | 142 | // Test "4" 143 | { 144 | idx, val, exists := tree.Get([]byte("4")) 145 | if exists { 146 | t.Errorf("Expected no value to exist") 147 | } 148 | if idx != 2 { 149 | t.Errorf("Unexpected idx %x", idx) 150 | } 151 | if string(val) != "" { 152 | t.Errorf("Unexpected value %v", string(val)) 153 | } 154 | } 155 | } 156 | 157 | func TestUnit(t *testing.T) { 158 | 159 | expectHash := func(tree *IAVLTree, hashCount int) { 160 | // ensure number of new hash calculations is as expected. 161 | hash, count := tree.HashWithCount() 162 | if count != hashCount { 163 | t.Fatalf("Expected %v new hashes, got %v", hashCount, count) 164 | } 165 | // nuke hashes and reconstruct hash, ensure it's the same. 166 | tree.root.traverse(tree, true, func(node *IAVLNode) bool { 167 | node.hash = nil 168 | return false 169 | }) 170 | // ensure that the new hash after nuking is the same as the old. 171 | newHash, _ := tree.HashWithCount() 172 | if bytes.Compare(hash, newHash) != 0 { 173 | t.Fatalf("Expected hash %v but got %v after nuking", hash, newHash) 174 | } 175 | } 176 | 177 | expectSet := func(tree *IAVLTree, i int, repr string, hashCount int) { 178 | origNode := tree.root 179 | updated := tree.Set(i2b(i), nil) 180 | // ensure node was added & structure is as expected. 181 | if updated == true || P(tree.root) != repr { 182 | t.Fatalf("Adding %v to %v:\nExpected %v\nUnexpectedly got %v updated:%v", 183 | i, P(origNode), repr, P(tree.root), updated) 184 | } 185 | // ensure hash calculation requirements 186 | expectHash(tree, hashCount) 187 | tree.root = origNode 188 | } 189 | 190 | expectRemove := func(tree *IAVLTree, i int, repr string, hashCount int) { 191 | origNode := tree.root 192 | value, removed := tree.Remove(i2b(i)) 193 | // ensure node was added & structure is as expected. 194 | if len(value) != 0 || !removed || P(tree.root) != repr { 195 | t.Fatalf("Removing %v from %v:\nExpected %v\nUnexpectedly got %v value:%v removed:%v", 196 | i, P(origNode), repr, P(tree.root), value, removed) 197 | } 198 | // ensure hash calculation requirements 199 | expectHash(tree, hashCount) 200 | tree.root = origNode 201 | } 202 | 203 | //////// Test Set cases: 204 | 205 | // Case 1: 206 | t1 := T(N(4, 20)) 207 | 208 | expectSet(t1, 8, "((4 8) 20)", 3) 209 | expectSet(t1, 25, "(4 (20 25))", 3) 210 | 211 | t2 := T(N(4, N(20, 25))) 212 | 213 | expectSet(t2, 8, "((4 8) (20 25))", 3) 214 | expectSet(t2, 30, "((4 20) (25 30))", 4) 215 | 216 | t3 := T(N(N(1, 2), 6)) 217 | 218 | expectSet(t3, 4, "((1 2) (4 6))", 4) 219 | expectSet(t3, 8, "((1 2) (6 8))", 3) 220 | 221 | t4 := T(N(N(1, 2), N(N(5, 6), N(7, 9)))) 222 | 223 | expectSet(t4, 8, "(((1 2) (5 6)) ((7 8) 9))", 5) 224 | expectSet(t4, 10, "(((1 2) (5 6)) (7 (9 10)))", 5) 225 | 226 | //////// Test Remove cases: 227 | 228 | t10 := T(N(N(1, 2), 3)) 229 | 230 | expectRemove(t10, 2, "(1 3)", 1) 231 | expectRemove(t10, 3, "(1 2)", 0) 232 | 233 | t11 := T(N(N(N(1, 2), 3), N(4, 5))) 234 | 235 | expectRemove(t11, 4, "((1 2) (3 5))", 2) 236 | expectRemove(t11, 3, "((1 2) (4 5))", 1) 237 | 238 | } 239 | 240 | func randBytes(length int) []byte { 241 | key := make([]byte, length) 242 | // math.rand.Read always returns err=nil 243 | mrand.Read(key) 244 | return key 245 | } 246 | 247 | func TestRemove(t *testing.T) { 248 | size := 10000 249 | keyLen, dataLen := 16, 40 250 | 251 | d := db.NewDB("test", "memdb", "") 252 | defer d.Close() 253 | t1 := NewIAVLTree(size, d) 254 | 255 | // insert a bunch of random nodes 256 | keys := make([][]byte, size) 257 | l := int32(len(keys)) 258 | for i := 0; i < size; i++ { 259 | key := randBytes(keyLen) 260 | t1.Set(key, randBytes(dataLen)) 261 | keys[i] = key 262 | } 263 | 264 | for i := 0; i < 10; i++ { 265 | step := 50 * i 266 | // remove a bunch of existing keys (may have been deleted twice) 267 | for j := 0; j < step; j++ { 268 | key := keys[mrand.Int31n(l)] 269 | t1.Remove(key) 270 | } 271 | // FIXME: this Save() causes a panic! 272 | t1.Save() 273 | } 274 | 275 | } 276 | 277 | func TestIntegration(t *testing.T) { 278 | 279 | type record struct { 280 | key string 281 | value string 282 | } 283 | 284 | records := make([]*record, 400) 285 | var tree *IAVLTree = NewIAVLTree(0, nil) 286 | 287 | randomRecord := func() *record { 288 | return &record{randstr(20), randstr(20)} 289 | } 290 | 291 | for i := range records { 292 | r := randomRecord() 293 | records[i] = r 294 | //t.Log("New record", r) 295 | //PrintIAVLNode(tree.root) 296 | updated := tree.Set([]byte(r.key), nil) 297 | if updated { 298 | t.Error("should have not been updated") 299 | } 300 | updated = tree.Set([]byte(r.key), []byte(r.value)) 301 | if !updated { 302 | t.Error("should have been updated") 303 | } 304 | if tree.Size() != i+1 { 305 | t.Error("size was wrong", tree.Size(), i+1) 306 | } 307 | } 308 | 309 | for _, r := range records { 310 | if has := tree.Has([]byte(r.key)); !has { 311 | t.Error("Missing key", r.key) 312 | } 313 | if has := tree.Has([]byte(randstr(12))); has { 314 | t.Error("Table has extra key") 315 | } 316 | if _, val, _ := tree.Get([]byte(r.key)); string(val) != string(r.value) { 317 | t.Error("wrong value") 318 | } 319 | } 320 | 321 | for i, x := range records { 322 | if val, removed := tree.Remove([]byte(x.key)); !removed { 323 | t.Error("Wasn't removed") 324 | } else if string(val) != string(x.value) { 325 | t.Error("Wrong value") 326 | } 327 | for _, r := range records[i+1:] { 328 | if has := tree.Has([]byte(r.key)); !has { 329 | t.Error("Missing key", r.key) 330 | } 331 | if has := tree.Has([]byte(randstr(12))); has { 332 | t.Error("Table has extra key") 333 | } 334 | _, val, _ := tree.Get([]byte(r.key)) 335 | if string(val) != string(r.value) { 336 | t.Error("wrong value") 337 | } 338 | } 339 | if tree.Size() != len(records)-(i+1) { 340 | t.Error("size was wrong", tree.Size(), (len(records) - (i + 1))) 341 | } 342 | } 343 | } 344 | 345 | func TestIterateRange(t *testing.T) { 346 | type record struct { 347 | key string 348 | value string 349 | } 350 | 351 | records := []record{ 352 | {"abc", "123"}, 353 | {"low", "high"}, 354 | {"fan", "456"}, 355 | {"foo", "a"}, 356 | {"foobaz", "c"}, 357 | {"good", "bye"}, 358 | {"foobang", "d"}, 359 | {"foobar", "b"}, 360 | {"food", "e"}, 361 | {"foml", "f"}, 362 | } 363 | keys := make([]string, len(records)) 364 | for i, r := range records { 365 | keys[i] = r.key 366 | } 367 | sort.Strings(keys) 368 | 369 | var tree *IAVLTree = NewIAVLTree(0, nil) 370 | 371 | // insert all the data 372 | for _, r := range records { 373 | updated := tree.Set([]byte(r.key), []byte(r.value)) 374 | if updated { 375 | t.Error("should have not been updated") 376 | } 377 | } 378 | 379 | // test traversing the whole node works... in order 380 | viewed := []string{} 381 | tree.Iterate(func(key []byte, value []byte) bool { 382 | viewed = append(viewed, string(key)) 383 | return false 384 | }) 385 | if len(viewed) != len(keys) { 386 | t.Error("not the same number of keys as expected") 387 | } 388 | for i, v := range viewed { 389 | if v != keys[i] { 390 | t.Error("Keys out of order", v, keys[i]) 391 | } 392 | } 393 | 394 | trav := traverser{} 395 | tree.IterateRange([]byte("foo"), []byte("goo"), true, trav.view) 396 | expectTraverse(t, trav, "foo", "food", 5) 397 | 398 | trav = traverser{} 399 | tree.IterateRange(nil, []byte("flap"), true, trav.view) 400 | expectTraverse(t, trav, "abc", "fan", 2) 401 | 402 | trav = traverser{} 403 | tree.IterateRange([]byte("foob"), nil, true, trav.view) 404 | expectTraverse(t, trav, "foobang", "low", 6) 405 | 406 | trav = traverser{} 407 | tree.IterateRange([]byte("very"), nil, true, trav.view) 408 | expectTraverse(t, trav, "", "", 0) 409 | 410 | // make sure backwards also works... 411 | trav = traverser{} 412 | tree.IterateRange([]byte("fooba"), []byte("food"), false, trav.view) 413 | expectTraverse(t, trav, "food", "foobang", 4) 414 | 415 | // make sure backwards also works... 416 | trav = traverser{} 417 | tree.IterateRange([]byte("g"), nil, false, trav.view) 418 | expectTraverse(t, trav, "low", "good", 2) 419 | 420 | } 421 | 422 | type traverser struct { 423 | first string 424 | last string 425 | count int 426 | } 427 | 428 | func (t *traverser) view(key, value []byte) bool { 429 | if t.first == "" { 430 | t.first = string(key) 431 | } 432 | t.last = string(key) 433 | t.count += 1 434 | return false 435 | } 436 | 437 | func expectTraverse(t *testing.T, trav traverser, start, end string, count int) { 438 | if trav.first != start { 439 | t.Error("Bad start", start, trav.first) 440 | } 441 | if trav.last != end { 442 | t.Error("Bad end", end, trav.last) 443 | } 444 | if trav.count != count { 445 | t.Error("Bad count", count, trav.count) 446 | } 447 | } 448 | 449 | func TestPersistence(t *testing.T) { 450 | db := db.NewMemDB() 451 | 452 | // Create some random key value pairs 453 | records := make(map[string]string) 454 | for i := 0; i < 10000; i++ { 455 | records[randstr(20)] = randstr(20) 456 | } 457 | 458 | // Construct some tree and save it 459 | t1 := NewIAVLTree(0, db) 460 | for key, value := range records { 461 | t1.Set([]byte(key), []byte(value)) 462 | } 463 | t1.Save() 464 | 465 | hash, _ := t1.HashWithCount() 466 | 467 | // Load a tree 468 | t2 := NewIAVLTree(0, db) 469 | t2.Load(hash) 470 | for key, value := range records { 471 | _, t2value, _ := t2.Get([]byte(key)) 472 | if string(t2value) != value { 473 | t.Fatalf("Invalid value. Expected %v, got %v", value, t2value) 474 | } 475 | } 476 | } 477 | 478 | func testProof(t *testing.T, proof *IAVLProof, keyBytes, valueBytes, rootHashBytes []byte) { 479 | // Proof must verify. 480 | require.True(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) 481 | 482 | // Write/Read then verify. 483 | proofBytes := wire.BinaryBytes(proof) 484 | proof2, err := ReadProof(proofBytes) 485 | require.Nil(t, err, "Failed to read IAVLProof from bytes: %v", err) 486 | require.True(t, proof2.Verify(keyBytes, valueBytes, proof.RootHash)) 487 | 488 | // Random mutations must not verify 489 | for i := 0; i < 10; i++ { 490 | badProofBytes := MutateByteSlice(proofBytes) 491 | badProof, err := ReadProof(badProofBytes) 492 | // may be invalid... errors are okay 493 | if err == nil { 494 | assert.False(t, badProof.Verify(keyBytes, valueBytes, rootHashBytes), 495 | "Proof was still valid after a random mutation:\n%X\n%X", 496 | proofBytes, badProofBytes) 497 | } 498 | } 499 | 500 | // targetted changes fails... 501 | proof.RootHash = MutateByteSlice(proof.RootHash) 502 | assert.False(t, proof.Verify(keyBytes, valueBytes, rootHashBytes)) 503 | proof2.LeafHash = MutateByteSlice(proof2.LeafHash) 504 | assert.False(t, proof2.Verify(keyBytes, valueBytes, rootHashBytes)) 505 | } 506 | 507 | func TestIAVLProof(t *testing.T) { 508 | // Construct some random tree 509 | db := db.NewMemDB() 510 | var tree *IAVLTree = NewIAVLTree(100, db) 511 | for i := 0; i < 1000; i++ { 512 | key, value := randstr(20), randstr(20) 513 | tree.Set([]byte(key), []byte(value)) 514 | } 515 | 516 | // Persist the items so far 517 | tree.Save() 518 | 519 | // Add more items so it's not all persisted 520 | for i := 0; i < 100; i++ { 521 | key, value := randstr(20), randstr(20) 522 | tree.Set([]byte(key), []byte(value)) 523 | } 524 | 525 | // Now for each item, construct a proof and verify 526 | tree.Iterate(func(key []byte, value []byte) bool { 527 | value2, proof := tree.ConstructProof(key) 528 | assert.Equal(t, value, value2) 529 | if assert.NotNil(t, proof) { 530 | testProof(t, proof, key, value, tree.Hash()) 531 | } 532 | return false 533 | }) 534 | } 535 | 536 | func TestIAVLTreeProof(t *testing.T) { 537 | db := db.NewMemDB() 538 | var tree *IAVLTree = NewIAVLTree(100, db) 539 | 540 | // should get false for proof with nil root 541 | _, _, exists := tree.Proof([]byte("foo")) 542 | assert.False(t, exists) 543 | 544 | // insert lots of info and store the bytes 545 | keys := make([][]byte, 200) 546 | for i := 0; i < 200; i++ { 547 | key, value := randstr(20), randstr(200) 548 | tree.Set([]byte(key), []byte(value)) 549 | keys[i] = []byte(key) 550 | } 551 | 552 | // query random key fails 553 | _, _, exists = tree.Proof([]byte("foo")) 554 | assert.False(t, exists) 555 | 556 | // valid proof for real keys 557 | root := tree.Hash() 558 | for _, key := range keys { 559 | value, proofBytes, exists := tree.Proof(key) 560 | if assert.True(t, exists) { 561 | proof, err := ReadProof(proofBytes) 562 | require.Nil(t, err, "Failed to read IAVLProof from bytes: %v", err) 563 | assert.True(t, proof.Verify(key, value, root)) 564 | } 565 | } 566 | } 567 | 568 | func BenchmarkImmutableAvlTreeCLevelDB(b *testing.B) { 569 | b.StopTimer() 570 | 571 | db := db.NewDB("test", db.CLevelDBBackendStr, "./") 572 | t := NewIAVLTree(100000, db) 573 | // for i := 0; i < 10000000; i++ { 574 | for i := 0; i < 1000000; i++ { 575 | // for i := 0; i < 1000; i++ { 576 | t.Set(i2b(int(RandInt32())), nil) 577 | if i > 990000 && i%1000 == 999 { 578 | t.Save() 579 | } 580 | } 581 | t.Save() 582 | 583 | fmt.Println("ok, starting") 584 | 585 | runtime.GC() 586 | 587 | b.StartTimer() 588 | for i := 0; i < b.N; i++ { 589 | ri := i2b(int(RandInt32())) 590 | t.Set(ri, nil) 591 | t.Remove(ri) 592 | if i%100 == 99 { 593 | t.Save() 594 | } 595 | } 596 | 597 | db.Close() 598 | } 599 | 600 | func BenchmarkImmutableAvlTreeMemDB(b *testing.B) { 601 | b.StopTimer() 602 | 603 | db := db.NewDB("test", db.MemDBBackendStr, "") 604 | t := NewIAVLTree(100000, db) 605 | // for i := 0; i < 10000000; i++ { 606 | for i := 0; i < 1000000; i++ { 607 | // for i := 0; i < 1000; i++ { 608 | t.Set(i2b(int(RandInt32())), nil) 609 | if i > 990000 && i%1000 == 999 { 610 | t.Save() 611 | } 612 | } 613 | t.Save() 614 | 615 | fmt.Println("ok, starting") 616 | 617 | runtime.GC() 618 | 619 | b.StartTimer() 620 | for i := 0; i < b.N; i++ { 621 | ri := i2b(int(RandInt32())) 622 | t.Set(ri, nil) 623 | t.Remove(ri) 624 | if i%100 == 99 { 625 | t.Save() 626 | } 627 | } 628 | 629 | db.Close() 630 | } 631 | -------------------------------------------------------------------------------- /iavl/iavl_tree.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "bytes" 5 | "container/list" 6 | "sync" 7 | 8 | wire "github.com/tendermint/go-wire" 9 | . "github.com/tendermint/tmlibs/common" 10 | dbm "github.com/tendermint/tmlibs/db" 11 | "github.com/tendermint/tmlibs/merkle" 12 | ) 13 | 14 | /* 15 | Immutable AVL Tree (wraps the Node root) 16 | This tree is not goroutine safe. 17 | */ 18 | type IAVLTree struct { 19 | root *IAVLNode 20 | ndb *nodeDB 21 | } 22 | 23 | // NewIAVLTree creates both im-memory and persistent instances 24 | func NewIAVLTree(cacheSize int, db dbm.DB) *IAVLTree { 25 | if db == nil { 26 | // In-memory IAVLTree 27 | return &IAVLTree{} 28 | } else { 29 | // Persistent IAVLTree 30 | ndb := newNodeDB(cacheSize, db) 31 | return &IAVLTree{ 32 | ndb: ndb, 33 | } 34 | } 35 | } 36 | 37 | // The returned tree and the original tree are goroutine independent. 38 | // That is, they can each run in their own goroutine. 39 | // However, upon Save(), any other trees that share a db will become 40 | // outdated, as some nodes will become orphaned. 41 | // Note that Save() clears leftNode and rightNode. Otherwise, 42 | // two copies would not be goroutine independent. 43 | func (t *IAVLTree) Copy() merkle.Tree { 44 | if t.root == nil { 45 | return &IAVLTree{ 46 | root: nil, 47 | ndb: t.ndb, 48 | } 49 | } 50 | if t.ndb != nil && !t.root.persisted { 51 | // Saving a tree finalizes all the nodes. 52 | // It sets all the hashes recursively, 53 | // clears all the leftNode/rightNode values recursively, 54 | // and all the .persisted flags get set. 55 | PanicSanity("It is unsafe to Copy() an unpersisted tree.") 56 | } else if t.ndb == nil && t.root.hash == nil { 57 | // An in-memory IAVLTree is finalized when the hashes are 58 | // calculated. 59 | t.root.hashWithCount(t) 60 | } 61 | return &IAVLTree{ 62 | root: t.root, 63 | ndb: t.ndb, 64 | } 65 | } 66 | 67 | func (t *IAVLTree) Size() int { 68 | if t.root == nil { 69 | return 0 70 | } 71 | return t.root.size 72 | } 73 | 74 | func (t *IAVLTree) Height() int8 { 75 | if t.root == nil { 76 | return 0 77 | } 78 | return t.root.height 79 | } 80 | 81 | func (t *IAVLTree) Has(key []byte) bool { 82 | if t.root == nil { 83 | return false 84 | } 85 | return t.root.has(t, key) 86 | } 87 | 88 | func (t *IAVLTree) Proof(key []byte) (value []byte, proofBytes []byte, exists bool) { 89 | value, proof := t.ConstructProof(key) 90 | if proof == nil { 91 | return nil, nil, false 92 | } 93 | proofBytes = wire.BinaryBytes(proof) 94 | return value, proofBytes, true 95 | } 96 | 97 | func (t *IAVLTree) Set(key []byte, value []byte) (updated bool) { 98 | if t.root == nil { 99 | t.root = NewIAVLNode(key, value) 100 | return false 101 | } 102 | t.root, updated = t.root.set(t, key, value) 103 | return updated 104 | } 105 | 106 | func (t *IAVLTree) Hash() []byte { 107 | if t.root == nil { 108 | return nil 109 | } 110 | hash, _ := t.root.hashWithCount(t) 111 | return hash 112 | } 113 | 114 | func (t *IAVLTree) HashWithCount() ([]byte, int) { 115 | if t.root == nil { 116 | return nil, 0 117 | } 118 | return t.root.hashWithCount(t) 119 | } 120 | 121 | func (t *IAVLTree) Save() []byte { 122 | if t.root == nil { 123 | return nil 124 | } 125 | if t.ndb != nil { 126 | t.root.save(t) 127 | t.ndb.Commit() 128 | } 129 | return t.root.hash 130 | } 131 | 132 | // Sets the root node by reading from db. 133 | // If the hash is empty, then sets root to nil. 134 | func (t *IAVLTree) Load(hash []byte) { 135 | if len(hash) == 0 { 136 | t.root = nil 137 | } else { 138 | t.root = t.ndb.GetNode(t, hash) 139 | } 140 | } 141 | 142 | func (t *IAVLTree) Get(key []byte) (index int, value []byte, exists bool) { 143 | if t.root == nil { 144 | return 0, nil, false 145 | } 146 | return t.root.get(t, key) 147 | } 148 | 149 | func (t *IAVLTree) GetByIndex(index int) (key []byte, value []byte) { 150 | if t.root == nil { 151 | return nil, nil 152 | } 153 | return t.root.getByIndex(t, index) 154 | } 155 | 156 | func (t *IAVLTree) Remove(key []byte) (value []byte, removed bool) { 157 | if t.root == nil { 158 | return nil, false 159 | } 160 | newRootHash, newRoot, _, value, removed := t.root.remove(t, key) 161 | if !removed { 162 | return nil, false 163 | } 164 | if newRoot == nil && newRootHash != nil { 165 | t.root = t.ndb.GetNode(t, newRootHash) 166 | } else { 167 | t.root = newRoot 168 | } 169 | return value, true 170 | } 171 | 172 | func (t *IAVLTree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) { 173 | if t.root == nil { 174 | return false 175 | } 176 | return t.root.traverse(t, true, func(node *IAVLNode) bool { 177 | if node.height == 0 { 178 | return fn(node.key, node.value) 179 | } else { 180 | return false 181 | } 182 | }) 183 | } 184 | 185 | // IterateRange makes a callback for all nodes with key between start and end inclusive 186 | // If either are nil, then it is open on that side (nil, nil is the same as Iterate) 187 | func (t *IAVLTree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) { 188 | if t.root == nil { 189 | return false 190 | } 191 | return t.root.traverseInRange(t, start, end, ascending, func(node *IAVLNode) bool { 192 | if node.height == 0 { 193 | return fn(node.key, node.value) 194 | } else { 195 | return false 196 | } 197 | }) 198 | } 199 | 200 | //----------------------------------------------------------------------------- 201 | 202 | type nodeDB struct { 203 | mtx sync.Mutex 204 | cache map[string]*list.Element 205 | cacheSize int 206 | cacheQueue *list.List 207 | db dbm.DB 208 | batch dbm.Batch 209 | orphans map[string]struct{} 210 | orphansPrev map[string]struct{} 211 | } 212 | 213 | func newNodeDB(cacheSize int, db dbm.DB) *nodeDB { 214 | ndb := &nodeDB{ 215 | cache: make(map[string]*list.Element), 216 | cacheSize: cacheSize, 217 | cacheQueue: list.New(), 218 | db: db, 219 | batch: db.NewBatch(), 220 | orphans: make(map[string]struct{}), 221 | orphansPrev: make(map[string]struct{}), 222 | } 223 | return ndb 224 | } 225 | 226 | func (ndb *nodeDB) GetNode(t *IAVLTree, hash []byte) *IAVLNode { 227 | ndb.mtx.Lock() 228 | defer ndb.mtx.Unlock() 229 | // Check the cache. 230 | elem, ok := ndb.cache[string(hash)] 231 | if ok { 232 | // Already exists. Move to back of cacheQueue. 233 | ndb.cacheQueue.MoveToBack(elem) 234 | return elem.Value.(*IAVLNode) 235 | } else { 236 | // Doesn't exist, load. 237 | buf := ndb.db.Get(hash) 238 | if len(buf) == 0 { 239 | // ndb.db.Print() 240 | PanicSanity(Fmt("Value missing for key %X", hash)) 241 | } 242 | node, err := MakeIAVLNode(buf, t) 243 | if err != nil { 244 | PanicCrisis(Fmt("Error reading IAVLNode. bytes: %X error: %v", buf, err)) 245 | } 246 | node.hash = hash 247 | node.persisted = true 248 | ndb.cacheNode(node) 249 | return node 250 | } 251 | } 252 | 253 | func (ndb *nodeDB) SaveNode(t *IAVLTree, node *IAVLNode) { 254 | ndb.mtx.Lock() 255 | defer ndb.mtx.Unlock() 256 | if node.hash == nil { 257 | PanicSanity("Expected to find node.hash, but none found.") 258 | } 259 | if node.persisted { 260 | PanicSanity("Shouldn't be calling save on an already persisted node.") 261 | } 262 | /*if _, ok := ndb.cache[string(node.hash)]; ok { 263 | panic("Shouldn't be calling save on an already cached node.") 264 | }*/ 265 | // Save node bytes to db 266 | buf := bytes.NewBuffer(nil) 267 | _, err := node.writePersistBytes(t, buf) 268 | if err != nil { 269 | PanicCrisis(err) 270 | } 271 | ndb.batch.Set(node.hash, buf.Bytes()) 272 | node.persisted = true 273 | ndb.cacheNode(node) 274 | // Re-creating the orphan, 275 | // Do not garbage collect. 276 | delete(ndb.orphans, string(node.hash)) 277 | delete(ndb.orphansPrev, string(node.hash)) 278 | } 279 | 280 | func (ndb *nodeDB) RemoveNode(t *IAVLTree, node *IAVLNode) { 281 | ndb.mtx.Lock() 282 | defer ndb.mtx.Unlock() 283 | if node.hash == nil { 284 | PanicSanity("Expected to find node.hash, but none found.") 285 | } 286 | if !node.persisted { 287 | PanicSanity("Shouldn't be calling remove on a non-persisted node.") 288 | } 289 | elem, ok := ndb.cache[string(node.hash)] 290 | if ok { 291 | ndb.cacheQueue.Remove(elem) 292 | delete(ndb.cache, string(node.hash)) 293 | } 294 | ndb.orphans[string(node.hash)] = struct{}{} 295 | } 296 | 297 | func (ndb *nodeDB) cacheNode(node *IAVLNode) { 298 | // Create entry in cache and append to cacheQueue. 299 | elem := ndb.cacheQueue.PushBack(node) 300 | ndb.cache[string(node.hash)] = elem 301 | // Maybe expire an item. 302 | if ndb.cacheQueue.Len() > ndb.cacheSize { 303 | hash := ndb.cacheQueue.Remove(ndb.cacheQueue.Front()).(*IAVLNode).hash 304 | delete(ndb.cache, string(hash)) 305 | } 306 | } 307 | 308 | func (ndb *nodeDB) Commit() { 309 | ndb.mtx.Lock() 310 | defer ndb.mtx.Unlock() 311 | // Delete orphans from previous block 312 | for orphanHashStr, _ := range ndb.orphansPrev { 313 | ndb.batch.Delete([]byte(orphanHashStr)) 314 | } 315 | // Write saves & orphan deletes 316 | ndb.batch.Write() 317 | ndb.db.SetSync(nil, nil) 318 | ndb.batch = ndb.db.NewBatch() 319 | // Shift orphans 320 | ndb.orphansPrev = ndb.orphans 321 | ndb.orphans = make(map[string]struct{}) 322 | } 323 | -------------------------------------------------------------------------------- /iavl/iavl_tree_dump.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | //"strings" 7 | 8 | wire "github.com/tendermint/go-wire" 9 | ) 10 | 11 | type Formatter func(in []byte) (out string) 12 | 13 | type KeyValueMapping struct { 14 | Key Formatter 15 | Value Formatter 16 | } 17 | 18 | // Flip back and forth between ascii and hex. 19 | func mixedDisplay(value []byte) string { 20 | 21 | var buffer bytes.Buffer 22 | var last []byte 23 | 24 | ascii := true 25 | for i := 0; i < len(value); i++ { 26 | if value[i] < 32 || value[i] > 126 { 27 | if ascii && len(last) > 0 { 28 | // only if there are 6 or more chars 29 | if len(last) > 5 { 30 | buffer.WriteString(fmt.Sprintf("%s", last)) 31 | last = nil 32 | } 33 | ascii = false 34 | } 35 | } 36 | last = append(last, value[i]) 37 | } 38 | if ascii { 39 | buffer.WriteString(fmt.Sprintf("%s", last)) 40 | } else { 41 | buffer.WriteString(fmt.Sprintf("%X", last)) 42 | } 43 | return buffer.String() 44 | } 45 | 46 | // This is merkleeyes state, that it is writing to a specific key 47 | type state struct { 48 | Hash []byte 49 | Height uint64 50 | } 51 | 52 | // Try to interpet as merkleeyes state 53 | func stateMapping(value []byte) string { 54 | var s state 55 | err := wire.ReadBinaryBytes(value, &s) 56 | if err != nil || s.Height > 500 { 57 | return mixedDisplay(value) 58 | } 59 | return fmt.Sprintf("Height:%d, [%X]", s.Height, s.Hash) 60 | } 61 | 62 | // This is basecoin accounts, that it is writing to a specific key 63 | type account struct { 64 | PubKey []byte 65 | Sequence int 66 | Balance []coin 67 | } 68 | 69 | type wrapper struct { 70 | bytes []byte 71 | } 72 | 73 | type coin struct { 74 | Denom string 75 | Amount int64 76 | } 77 | 78 | // Perhaps this is an IAVL tree node? 79 | func nodeMapping(node *IAVLNode) string { 80 | 81 | formattedKey := mixedDisplay(node.key) 82 | 83 | var formattedValue string 84 | var acc account 85 | 86 | err := wire.ReadBinaryBytes(node.value, &acc) 87 | if err != nil { 88 | formattedValue = mixedDisplay(node.value) 89 | } else { 90 | formattedValue = fmt.Sprintf("%v", acc) 91 | } 92 | 93 | if node.height == 0 { 94 | return fmt.Sprintf(" LeafNode[height: %d, size %d, key: %s, value: %s]", 95 | node.height, node.size, formattedKey, formattedValue) 96 | } else { 97 | return fmt.Sprintf("InnerNode[height: %d, size %d, key: %s, leftHash: %X, rightHash: %X]", 98 | node.height, node.size, formattedKey, node.leftHash, node.rightHash) 99 | } 100 | } 101 | 102 | // Try everything and see what sticks... 103 | func overallMapping(value []byte) (str string) { 104 | // underneath make node, wire can throw a panic 105 | defer func() { 106 | if recover() != nil { 107 | str = fmt.Sprintf("%X", value) 108 | return 109 | } 110 | }() 111 | 112 | // test to see if this is a node 113 | node, err := MakeIAVLNode(value, nil) 114 | 115 | if err == nil && node.height < 100 && node.key != nil { 116 | return nodeMapping(node) 117 | } 118 | 119 | // Unknown value type 120 | return stateMapping(value) 121 | } 122 | 123 | // Dump everything from the database 124 | func (t *IAVLTree) Dump(verbose bool, mapping *KeyValueMapping) { 125 | if verbose && t.root == nil { 126 | fmt.Printf("No root loaded into memory\n") 127 | } 128 | 129 | if mapping == nil { 130 | mapping = &KeyValueMapping{Key: mixedDisplay, Value: overallMapping} 131 | } 132 | 133 | if verbose { 134 | stats := t.ndb.db.Stats() 135 | for key, value := range stats { 136 | fmt.Printf("%s:\n\t%s\n", key, value) 137 | } 138 | } 139 | 140 | iter := t.ndb.db.Iterator() 141 | for iter.Next() { 142 | fmt.Printf("%s: %s\n", mapping.Key(iter.Key()), mapping.Value(iter.Value())) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /iavl/util.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Prints the in-memory children recursively. 8 | func PrintIAVLNode(node *IAVLNode) { 9 | fmt.Println("==== NODE") 10 | if node != nil { 11 | printIAVLNode(node, 0) 12 | } 13 | fmt.Println("==== END") 14 | } 15 | 16 | func printIAVLNode(node *IAVLNode, indent int) { 17 | indentPrefix := "" 18 | for i := 0; i < indent; i++ { 19 | indentPrefix += " " 20 | } 21 | 22 | if node.rightNode != nil { 23 | printIAVLNode(node.rightNode, indent+1) 24 | } else if node.rightHash != nil { 25 | fmt.Printf("%s %X\n", indentPrefix, node.rightHash) 26 | } 27 | 28 | fmt.Printf("%s%v:%v\n", indentPrefix, node.key, node.height) 29 | 30 | if node.leftNode != nil { 31 | printIAVLNode(node.leftNode, indent+1) 32 | } else if node.leftHash != nil { 33 | fmt.Printf("%s %X\n", indentPrefix, node.leftHash) 34 | } 35 | 36 | } 37 | 38 | func maxInt8(a, b int8) int8 { 39 | if a > b { 40 | return a 41 | } 42 | return b 43 | } 44 | -------------------------------------------------------------------------------- /iavl/version.go: -------------------------------------------------------------------------------- 1 | package iavl 2 | 3 | const Version = "0.4.0" // benchmarking, update proof 4 | -------------------------------------------------------------------------------- /scripts/dist.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | REPO_NAME="merkleeyes" 5 | 6 | # Get the version from the environment, or try to figure it out. 7 | if [ -z $VERSION ]; then 8 | VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) 9 | fi 10 | if [ -z "$VERSION" ]; then 11 | echo "Please specify a version." 12 | exit 1 13 | fi 14 | echo "==> Building version $VERSION..." 15 | 16 | # Get the parent directory of where this script is. 17 | SOURCE="${BASH_SOURCE[0]}" 18 | while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done 19 | DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" 20 | 21 | # Change into that dir because we expect that. 22 | cd "$DIR" 23 | 24 | 25 | # Delete the old dir 26 | echo "==> Removing old directory..." 27 | rm -rf build/pkg 28 | mkdir -p build/pkg 29 | 30 | # Do a hermetic build inside a Docker container. 31 | docker build -t tendermint/${REPO_NAME}-builder scripts/${REPO_NAME}-builder/ 32 | docker run --rm -e "BUILD_TAGS=$BUILD_TAGS" -v "$(pwd)":/go/src/github.com/tendermint/${REPO_NAME} tendermint/${REPO_NAME}-builder ./scripts/dist_build.sh 33 | 34 | # Add $REPO_NAME and $VERSION prefix to package name. 35 | rm -rf ./build/dist 36 | mkdir -p ./build/dist 37 | for FILENAME in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type f); do 38 | FILENAME=$(basename "$FILENAME") 39 | cp "./build/pkg/${FILENAME}" "./build/dist/${REPO_NAME}_${VERSION}_${FILENAME}" 40 | done 41 | 42 | # Make the checksums. 43 | pushd ./build/dist 44 | shasum -a256 ./* > "./${REPO_NAME}_${VERSION}_SHA256SUMS" 45 | popd 46 | 47 | # Done 48 | echo 49 | echo "==> Results:" 50 | ls -hl ./build/dist 51 | 52 | exit 0 53 | -------------------------------------------------------------------------------- /scripts/dist_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # Get the parent directory of where this script is. 5 | SOURCE="${BASH_SOURCE[0]}" 6 | while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done 7 | DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" 8 | 9 | # Change into that dir because we expect that. 10 | cd "$DIR" 11 | 12 | # Get the git commit 13 | GIT_COMMIT="$(git rev-parse --short HEAD)" 14 | GIT_DESCRIBE="$(git describe --tags --always)" 15 | GIT_IMPORT="github.com/tendermint/merkleeyes/version" 16 | 17 | # Determine the arch/os combos we're building for 18 | XC_ARCH=${XC_ARCH:-"386 amd64 arm"} 19 | XC_OS=${XC_OS:-"solaris darwin freebsd linux windows"} 20 | 21 | # Make sure build tools are available. 22 | make tools 23 | 24 | # Get VENDORED dependencies 25 | make get_vendor_deps 26 | 27 | # Build! 28 | echo "==> Building..." 29 | "$(which gox)" \ 30 | -os="${XC_OS}" \ 31 | -arch="${XC_ARCH}" \ 32 | -osarch="!darwin/arm !solaris/amd64 !freebsd/amd64" \ 33 | -ldflags "-X ${GIT_IMPORT}.GitCommit='${GIT_COMMIT}' -X ${GIT_IMPORT}.GitDescribe='${GIT_DESCRIBE}'" \ 34 | -output "build/pkg/{{.OS}}_{{.Arch}}/merkleeyes" \ 35 | -tags="${BUILD_TAGS}" \ 36 | github.com/tendermint/merkleeyes/cmd/merkleeyes 37 | 38 | # Zip all the files. 39 | echo "==> Packaging..." 40 | for PLATFORM in $(find ./build/pkg -mindepth 1 -maxdepth 1 -type d); do 41 | OSARCH=$(basename "${PLATFORM}") 42 | echo "--> ${OSARCH}" 43 | 44 | pushd "$PLATFORM" >/dev/null 2>&1 45 | zip "../${OSARCH}.zip" ./* 46 | popd >/dev/null 2>&1 47 | done 48 | 49 | 50 | 51 | exit 0 52 | -------------------------------------------------------------------------------- /scripts/looper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | . "github.com/tendermint/tmlibs/common" 7 | "github.com/tendermint/tmlibs/db" 8 | "github.com/tendermint/merkleeyes/iavl" 9 | ) 10 | 11 | func main() { 12 | db := db.NewMemDB() 13 | t := iavl.NewIAVLTree(0, db) 14 | // 23000ns/op, 43000ops/s 15 | // for i := 0; i < 10000000; i++ { 16 | // for i := 0; i < 1000000; i++ { 17 | for i := 0; i < 1000; i++ { 18 | t.Set(RandBytes(12), nil) 19 | } 20 | t.Save() 21 | 22 | fmt.Println("ok, starting") 23 | 24 | for i := 0; ; i++ { 25 | key := RandBytes(12) 26 | t.Set(key, nil) 27 | t.Remove(key) 28 | if i%1000 == 0 { 29 | t.Save() 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /scripts/merkleeyes-builder/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.7.4 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | zip \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | # We want to ensure that release builds never have any cgo dependencies so we 8 | # switch that off at the highest level. 9 | ENV CGO_ENABLED 0 10 | 11 | RUN mkdir -p $GOPATH/src/github.com/tendermint/merkleeyes 12 | WORKDIR $GOPATH/src/github.com/tendermint/merkleeyes 13 | -------------------------------------------------------------------------------- /scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Get the version from the environment, or try to figure it out. 4 | if [ -z $VERSION ]; then 5 | VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) 6 | fi 7 | aws s3 cp --recursive build/dist s3://tendermint/binaries/merkleeyes/v${VERSION} --acl public-read 8 | -------------------------------------------------------------------------------- /tests/client_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | "github.com/tendermint/abci/server" 10 | "github.com/tendermint/merkleeyes/app" 11 | eyes "github.com/tendermint/merkleeyes/client" 12 | . "github.com/tendermint/tmlibs/common" 13 | ) 14 | 15 | var tmspAddr = "tcp://127.0.0.1:46659" 16 | 17 | func TestNonPersistent(t *testing.T) { 18 | testProcedure(t, tmspAddr, "", 0, false, true) 19 | } 20 | 21 | func TestPersistent(t *testing.T) { 22 | dbName := "testDb" 23 | os.RemoveAll(dbName) //remove the database if exists for any reason 24 | testProcedure(t, tmspAddr, dbName, 0, false, false) 25 | testProcedure(t, tmspAddr, dbName, 0, true, true) 26 | os.RemoveAll(dbName) //cleanup, remove database that was created by testProcedure 27 | } 28 | 29 | func testProcedure(t *testing.T, addr, dbName string, cache int, testPersistence, clearRecords bool) { 30 | 31 | checkErr := func(err error) { 32 | if err != nil { 33 | t.Fatal(err.Error()) 34 | return 35 | } 36 | } 37 | 38 | // Create the app and start the listener 39 | mApp := app.NewMerkleEyesApp(dbName, cache) 40 | defer mApp.CloseDB() 41 | 42 | s, err := server.NewServer(addr, "socket", mApp) 43 | checkErr(err) 44 | 45 | s.Start() 46 | defer s.Stop() 47 | 48 | // Create client 49 | cli, err := eyes.NewClient(addr) 50 | checkErr(err) 51 | cli.Start() 52 | defer cli.Stop() 53 | 54 | if !testPersistence { 55 | // Empty 56 | commit(t, cli, "") 57 | get(t, cli, "foo", "") 58 | get(t, cli, "bar", "") 59 | // Set foo=FOO 60 | set(t, cli, "foo", "FOO") 61 | 62 | commit(t, cli, "68DECA470D80183B5E979D167E3DD0956631A952") 63 | get(t, cli, "foo", "FOO") 64 | get(t, cli, "foa", "") 65 | get(t, cli, "foz", "") 66 | rem(t, cli, "foo") 67 | 68 | // Not empty until commit.... 69 | get(t, cli, "foo", "FOO") 70 | commit(t, cli, "") 71 | get(t, cli, "foo", "") 72 | 73 | // Set foo1, foo2, foo3... 74 | set(t, cli, "foo1", "1") 75 | set(t, cli, "foo2", "2") 76 | set(t, cli, "foo3", "3") 77 | set(t, cli, "foo1", "4") 78 | // nothing commited yet... 79 | get(t, cli, "foo1", "") 80 | commit(t, cli, "45B7F856A16CB2F8BB9A4A25587FC71D062BD631") 81 | // now we got info 82 | get(t, cli, "foo1", "4") 83 | get(t, cli, "foo2", "2") 84 | get(t, cli, "foo3", "3") 85 | } else { 86 | get(t, cli, "foo1", "4") 87 | get(t, cli, "foo2", "2") 88 | get(t, cli, "foo3", "3") 89 | } 90 | 91 | if clearRecords { 92 | rem(t, cli, "foo3") 93 | rem(t, cli, "foo2") 94 | rem(t, cli, "foo1") 95 | // Empty 96 | commit(t, cli, "") 97 | } 98 | } 99 | 100 | func get(t *testing.T, cli *eyes.Client, key string, value string) { 101 | valExp := []byte(nil) 102 | if value != "" { 103 | valExp = []byte(value) 104 | } 105 | valGot := cli.Get([]byte(key)) 106 | require.EqualValues(t, valExp, valGot) 107 | } 108 | 109 | func set(t *testing.T, cli *eyes.Client, key string, value string) { 110 | cli.Set([]byte(key), []byte(value)) 111 | } 112 | 113 | func rem(t *testing.T, cli *eyes.Client, key string) { 114 | cli.Remove([]byte(key)) 115 | } 116 | 117 | func commit(t *testing.T, cli *eyes.Client, hash string) { 118 | res := cli.CommitSync() 119 | require.False(t, res.IsErr(), res.Error()) 120 | assert.Equal(t, hash, Fmt("%X", res.Data.Bytes())) 121 | } 122 | -------------------------------------------------------------------------------- /testutil/testutil.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/tendermint/abci/server" 7 | wire "github.com/tendermint/go-wire" 8 | "github.com/tendermint/merkleeyes/app" 9 | eyes "github.com/tendermint/merkleeyes/client" 10 | . "github.com/tendermint/tmlibs/common" 11 | ) 12 | 13 | // NOTE: don't forget to close the client & server. 14 | func CreateEyes(t *testing.T) (svr Service, cli *eyes.Client) { 15 | addr := "unix://eyes.sock" 16 | 17 | // Start the listener 18 | mApp := app.NewMerkleEyesApp("", 0) 19 | svr, err := server.NewServer(addr, "socket", mApp) 20 | if err != nil { 21 | (err.Error()) 22 | return 23 | } 24 | 25 | // Create client 26 | cli, err = eyes.NewClient(addr) 27 | if err != nil { 28 | t.Fatal(err.Error()) 29 | return 30 | } 31 | 32 | return svr, cli 33 | } 34 | 35 | // MakeTxKV returns a text transaction, allong with expected key, value pair 36 | func MakeTxKV() ([]byte, []byte, []byte) { 37 | k := []byte(RandStr(8)) 38 | v := []byte(RandStr(8)) 39 | return k, v, makeSet(k, v) 40 | } 41 | 42 | // blatently copied from merkleeyes/app/app_test.go 43 | // constructs a "set" transaction 44 | func makeSet(key, value []byte) []byte { 45 | tx := make([]byte, 1+wire.ByteSliceSize(key)+wire.ByteSliceSize(value)) 46 | buf := tx 47 | buf[0] = app.TxTypeSet // Set TypeByte 48 | buf = buf[1:] 49 | n, err := wire.PutByteSlice(buf, key) 50 | if err != nil { 51 | panic(err) 52 | } 53 | buf = buf[n:] 54 | n, err = wire.PutByteSlice(buf, value) 55 | if err != nil { 56 | panic(err) 57 | } 58 | return tx 59 | } 60 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import abci "github.com/tendermint/abci/types" 4 | 5 | type MerkleEyser interface { 6 | GetSync(key []byte) abci.Result 7 | SetSync(key []byte, value []byte) abci.Result 8 | RemSync(key []byte) abci.Result 9 | CommitSync() abci.Result 10 | } 11 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | const Maj = "0" 4 | const Min = "2" 5 | const Fix = "2" 6 | 7 | const Version = "0.2.2-ngc.3" 8 | --------------------------------------------------------------------------------