├── mkpkg.sh ├── convert ├── height.go ├── init.go ├── meta.go ├── part.go ├── config.go ├── validator.go ├── blockstore.go ├── commit.go └── state.go ├── run.sh ├── util ├── file.go └── blockchain.go ├── cmd ├── state │ └── main.go ├── migrator │ └── main.go ├── validator │ └── main.go ├── genesis │ └── main.go └── viewer │ ├── blockchain.go │ └── main.go ├── LICENSE ├── old └── tm_v0.7.3.go └── README.md /mkpkg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | VERSION=1.1.0.0 5 | ROOT_PATH=$(cd $(dirname $0) && pwd) 6 | PKG=tm_tools_v${VERSION} 7 | 8 | cd "$ROOT_PATH" 9 | bash build.sh 10 | 11 | mkdir -p $PKG 12 | cp run.sh build/* $PKG 13 | 14 | tar -zvcf ${PKG}.tar.gz $PKG 15 | rm -rf $PKG 16 | -------------------------------------------------------------------------------- /convert/height.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hxzqlh/tm-tools/util" 7 | ) 8 | 9 | func TotalHeight() { 10 | blockStore := util.LoadOldBlockStoreStateJSON(oBlockDb) 11 | totalHeight = blockStore.Height 12 | 13 | fmt.Println("total height", totalHeight) 14 | } 15 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | [ $# -lt 3 ] && echo "$0 old_tm new_tm priv_dir" && exit 1 5 | 6 | ROOT_PATH=$(cd $(dirname $0) && pwd) 7 | 8 | OLD_TM=$(cd $(dirname $1) && pwd) 9 | NEW_TM=$(cd $(dirname $2) && pwd) 10 | PRIV_DIR=$(cd $(dirname $3) && pwd) 11 | 12 | # on your own config 13 | TM=/path/to/tendermint 14 | APP=appname 15 | 16 | $ROOT_PATH/tm_migrator -old $OLD_TM -new $NEW_TM -priv $PRIV_DIR 17 | TMROOT="$NEW_TM" $TM node --proxy_app=$APP 18 | -------------------------------------------------------------------------------- /util/file.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | func FileNameNoExt(fpath string) string { 11 | base := filepath.Base(fpath) 12 | return strings.TrimSuffix(base, filepath.Ext(fpath)) 13 | } 14 | 15 | func CopyFile(dstName, srcName string) (written int64, err error) { 16 | src, err := os.Open(srcName) 17 | if err != nil { 18 | return 0, err 19 | } 20 | defer src.Close() 21 | 22 | dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644) 23 | if err != nil { 24 | return 0, err 25 | } 26 | defer dst.Close() 27 | return io.Copy(dst, src) 28 | } 29 | -------------------------------------------------------------------------------- /convert/init.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | dbm "github.com/tendermint/tmlibs/db" 5 | ) 6 | 7 | var oBlockDb, oStateDb dbm.DB 8 | var nBlockDb, nStateDb dbm.DB 9 | var totalHeight int 10 | 11 | func OnStart(oTmRoot, nTmRoot string) { 12 | oBlockDb = dbm.NewDB("blockstore", "leveldb", oTmRoot+"/data") 13 | oStateDb = dbm.NewDB("state", "leveldb", oTmRoot+"/data") 14 | nBlockDb = dbm.NewDB("blockstore", "leveldb", nTmRoot+"/data") 15 | nStateDb = dbm.NewDB("state", "leveldb", nTmRoot+"/data") 16 | } 17 | 18 | func OnStop() { 19 | oBlockDb.Close() 20 | oStateDb.Close() 21 | nBlockDb.Close() 22 | nStateDb.Close() 23 | } 24 | -------------------------------------------------------------------------------- /cmd/state/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/hex" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/hxzqlh/tm-tools/util" 11 | dbm "github.com/tendermint/tmlibs/db" 12 | ) 13 | 14 | var ( 15 | stateKey = "stateKey" 16 | ) 17 | 18 | var stateDir = flag.String("state", os.ExpandEnv("$HOME/.tendermint")+"/data/state.db", "tendermint state db") 19 | var hash = flag.String("hash", "", "tendermint app hash") 20 | var stateDb dbm.DB 21 | 22 | func main() { 23 | if len(os.Args) <= 4 { 24 | fmt.Printf("Usage: %s -state db -hash appHash\n", os.Args[0]) 25 | os.Exit(0) 26 | } 27 | 28 | flag.Parse() 29 | 30 | stateDb = dbm.NewDB(util.FileNameNoExt(*stateDir), "leveldb", filepath.Dir(*stateDir)) 31 | defer stateDb.Close() 32 | 33 | bytes, _ := hex.DecodeString(*hash) 34 | s := util.LoadNewState(stateDb) 35 | s.AppHash = bytes 36 | util.SaveNewState(stateDb, s) 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 hxzqlh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /convert/meta.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "github.com/hxzqlh/tm-tools/old" 5 | "github.com/hxzqlh/tm-tools/util" 6 | "github.com/tendermint/tendermint/types" 7 | ) 8 | 9 | // bugy 10 | func convertBlockMeta(height int, lastBlockId *types.BlockID) *types.BlockID { 11 | oMeta := util.LoadOldBlockMeta(oBlockDb, height) 12 | 13 | nMeta := &types.BlockMeta{} 14 | // BlockID 15 | // TODO: was Hash need recompute? 16 | nMeta.BlockID.Hash = oMeta.Hash 17 | nMeta.BlockID.PartsHeader.Hash = oMeta.PartsHeader.Hash 18 | nMeta.BlockID.PartsHeader.Total = oMeta.PartsHeader.Total 19 | 20 | // Header 21 | nMeta.Header = NewHeader(oMeta.Header, lastBlockId) 22 | util.SaveNewBlockMeta(nBlockDb, height, nMeta) 23 | 24 | return &nMeta.BlockID 25 | } 26 | 27 | func NewHeader(o *old.Header, lastBlockId *types.BlockID) *types.Header { 28 | n := &types.Header{} 29 | // TODO: AppHash need reset? 30 | n.AppHash = o.AppHash 31 | n.ChainID = o.ChainID 32 | n.DataHash = o.DataHash 33 | n.Height = o.Height 34 | if lastBlockId != nil { 35 | n.LastBlockID = *lastBlockId 36 | } 37 | n.LastCommitHash = o.LastCommitHash 38 | n.NumTxs = o.NumTxs 39 | n.Time = o.Time 40 | n.ValidatorsHash = o.ValidatorsHash 41 | 42 | return n 43 | } 44 | -------------------------------------------------------------------------------- /convert/part.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "github.com/hxzqlh/tm-tools/old" 5 | "github.com/hxzqlh/tm-tools/util" 6 | "github.com/tendermint/tendermint/types" 7 | ) 8 | 9 | func convertPart(height int, lastBlockID *types.BlockID) { 10 | nBlock := NewBlockFromOld(height, lastBlockID) 11 | util.SaveNewBlockParts(nBlockDb, height, nBlock) 12 | } 13 | 14 | func NewBlockFromOld(height int, lastBlockID *types.BlockID) *types.Block { 15 | oBlock := util.LoadOldBlock(oBlockDb, height) 16 | 17 | nBlock := &types.Block{} 18 | nBlock.Data = NewData(oBlock.Data) 19 | nBlock.LastCommit = NewCommit(oBlock.LastCommit, lastBlockID) 20 | nBlock.Header = NewHeader(oBlock.Header, lastBlockID) 21 | 22 | return nBlock 23 | } 24 | 25 | func NewBlockFromOld2(height int, lastBlockID *types.BlockID) (*types.Block, *types.PartSet) { 26 | oBlock := util.LoadOldBlock(oBlockDb, height) 27 | commit := NewCommit(oBlock.LastCommit, lastBlockID) 28 | txs := []types.Tx{} 29 | for _, tx := range oBlock.Txs { 30 | txs = append(txs, []byte(tx)) 31 | } 32 | 33 | return types.MakeBlock(height, oBlock.ChainID, txs, commit, 34 | *lastBlockID, oBlock.ValidatorsHash, oBlock.AppHash, types.DefaultBlockPartSize) 35 | } 36 | 37 | func NewData(o *old.Data) *types.Data { 38 | nData := &types.Data{} 39 | txs := []types.Tx{} 40 | for _, tx := range o.Txs { 41 | txs = append(txs, []byte(tx)) 42 | } 43 | nData.Txs = txs 44 | return nData 45 | } 46 | -------------------------------------------------------------------------------- /cmd/migrator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/hxzqlh/tm-tools/convert" 10 | cmn "github.com/tendermint/tmlibs/common" 11 | ) 12 | 13 | var oldData = flag.String("old", os.ExpandEnv("$HOME/.tendermint"), "old tendermint dir") 14 | var newData = flag.String("new", os.ExpandEnv("$HOME/.tendermint.new"), "new tendermint dir") 15 | var privData = flag.String("priv", "", "other priv_validator.json configs dir") 16 | var startHeight = flag.Int("s", 1, "start from height") 17 | 18 | func main() { 19 | if len(os.Args) <= 6 { 20 | fmt.Printf("Usage: %s -old tmroot -new tmroot -priv priv_dir [-s startHeight]\n", os.Args[0]) 21 | os.Exit(0) 22 | } 23 | 24 | flag.Parse() 25 | 26 | //pwd, _ := filepath.Abs(filepath.Dir(os.Args[0])) 27 | oTmRoot, _ := filepath.Abs(*oldData) 28 | nTmRoot, _ := filepath.Abs(*newData) 29 | privDir, _ := filepath.Abs(*privData) 30 | cmn.EnsureDir(nTmRoot, 0755) 31 | 32 | convert.OnStart(oTmRoot, nTmRoot) 33 | 34 | // gen config.toml 35 | convert.OnConfigToml(nTmRoot + "/config.toml") 36 | 37 | // genesis 38 | convert.OnGenesisJSON(oTmRoot+"/genesis.json", nTmRoot+"/genesis.json") 39 | 40 | // me priv 41 | convert.OnPrivValidatorJSON(oTmRoot+"/priv_validator.json", nTmRoot+"/priv_validator.json") 42 | 43 | // old privalitors 44 | convert.LoadPrivValidators(privDir) 45 | 46 | convert.TotalHeight() 47 | 48 | convert.OnBlockStore(*startHeight) 49 | 50 | convert.OnStop() 51 | } 52 | -------------------------------------------------------------------------------- /cmd/validator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/hxzqlh/tm-tools/old" 9 | "github.com/tendermint/tendermint/types" 10 | ) 11 | 12 | // convert tendermint v0.7.3 priv_validator.json to version of v0.10.0 and output result to stdout 13 | // Usage: ./tm_priv_validator old_priv_validator_path 14 | /* 15 | Replace [TypeByte, Xxx] with {"type": "some-type", "data": Xxx} in RPC and all .json files by using go-wire/data. 16 | 17 | For instance, a pubkey old verison is: 18 | "pub_key": { 19 | 1, 20 | "83DDF8775937A4A12A2704269E2729FCFCD491B933C4B0A7FFE37FE41D7760D0" 21 | } 22 | 23 | now is: 24 | "pub_key": { 25 | "type": "ed25519", 26 | "data": "83DDF8775937A4A12A2704269E2729FCFCD491B933C4B0A7FFE37FE41D7760D0" 27 | } 28 | */ 29 | 30 | func main() { 31 | if len(os.Args) <= 1 { 32 | fmt.Printf("Usage: %s old_priv_validator.json \n", os.Args[0]) 33 | os.Exit(0) 34 | } 35 | 36 | nVali := NewPrivValidator(os.Args[1]) 37 | bytes, _ := json.Marshal(nVali) 38 | fmt.Println(string(bytes)) 39 | } 40 | 41 | func NewPrivValidator(oPath string) *types.PrivValidator { 42 | privVali := &types.PrivValidator{} 43 | old := old.LoadPrivValidator(oPath) 44 | privVali.Address = old.Address 45 | privVali.LastHeight = old.LastHeight 46 | privVali.LastRound = old.LastRound 47 | privVali.LastSignature = old.LastSignature 48 | privVali.LastSignBytes = old.LastSignBytes 49 | privVali.LastStep = old.LastStep 50 | privVali.PrivKey = old.PrivKey 51 | privVali.PubKey = old.PubKey 52 | return privVali 53 | } 54 | -------------------------------------------------------------------------------- /convert/config.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "github.com/hxzqlh/tm-tools/old" 7 | "github.com/tendermint/tendermint/types" 8 | cmn "github.com/tendermint/tmlibs/common" 9 | ) 10 | 11 | func OnConfigToml(configFilePath string) { 12 | var configTmpl = `# This is a TOML config file. 13 | # For more information, see https://github.com/toml-lang/toml 14 | 15 | proxy_app = "tcp://0.0.0.0:46658" 16 | moniker = "anonymous" 17 | node_laddr = "tcp://0.0.0.0:46656" 18 | seeds = "" 19 | fast_sync = true 20 | db_backend = "leveldb" 21 | log_level = "info" 22 | rpc_laddr = "tcp://0.0.0.0:46657" 23 | ` 24 | cmn.WriteFile(configFilePath, []byte(configTmpl), 0644) 25 | } 26 | 27 | func OnGenesisJSON(oPath, nPath string) { 28 | jsonBytes, err := ioutil.ReadFile(oPath) 29 | if err != nil { 30 | panic(err) 31 | } 32 | 33 | oGen := old.GenesisDocFromJSON(jsonBytes) 34 | nGen := NewGenesisDoc(oGen) 35 | nGen.SaveAs(nPath) 36 | } 37 | 38 | func NewPrivValidator(oPath string) *types.PrivValidator { 39 | privVali := &types.PrivValidator{} 40 | old := old.LoadPrivValidator(oPath) 41 | privVali.Address = old.Address 42 | privVali.LastHeight = old.LastHeight 43 | privVali.LastRound = old.LastRound 44 | privVali.LastSignature = old.LastSignature 45 | privVali.LastSignBytes = old.LastSignBytes 46 | privVali.LastStep = old.LastStep 47 | privVali.PrivKey = old.PrivKey 48 | privVali.PubKey = old.PubKey 49 | return privVali 50 | } 51 | 52 | func OnPrivValidatorJSON(oPath, nPath string) { 53 | privVali := NewPrivValidator(oPath) 54 | privVali.SetFile(nPath) 55 | privVali.Save() 56 | } 57 | -------------------------------------------------------------------------------- /convert/validator.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "encoding/hex" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | "strings" 10 | 11 | "github.com/tendermint/tendermint/types" 12 | cmn "github.com/tendermint/tmlibs/common" 13 | ) 14 | 15 | var privValidators *PrivValidators 16 | var priv_files []string 17 | 18 | type PrivValidators struct { 19 | Validators []*types.PrivValidator 20 | } 21 | 22 | func (arr *PrivValidators) Len() int { 23 | return len(arr.Validators) 24 | } 25 | 26 | func (arr *PrivValidators) Less(i, j int) bool { 27 | a := hex.EncodeToString(arr.Validators[i].Address) 28 | b := hex.EncodeToString(arr.Validators[j].Address) 29 | return strings.Compare(a, b) < 1 30 | } 31 | 32 | func (arr *PrivValidators) Swap(i, j int) { 33 | arr.Validators[i], arr.Validators[j] = arr.Validators[j], arr.Validators[i] 34 | } 35 | 36 | func LoadPrivValidators(folder string) { 37 | err := filepath.Walk(folder, walkFunc) 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | privValidators = &PrivValidators{ 43 | Validators: []*types.PrivValidator{}, 44 | } 45 | 46 | for _, path := range priv_files { 47 | priv := NewPrivValidator(path) 48 | privValidators.Validators = append(privValidators.Validators, priv) 49 | } 50 | 51 | // sort priv_validators 52 | sort.Sort(privValidators) 53 | } 54 | 55 | func walkFunc(path string, f os.FileInfo, err error) error { 56 | if !cmn.FileExists(path) || f.IsDir() { 57 | return nil 58 | } 59 | 60 | fmt.Println("priv: ", path) 61 | if strings.HasPrefix(filepath.Base(path), "priv_validator") { 62 | priv_files = append(priv_files, path) 63 | } 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /cmd/genesis/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/hxzqlh/tm-tools/old" 10 | "github.com/tendermint/tendermint/types" 11 | ) 12 | 13 | // convert tendermint v0.7.3 genesis.json to version of v0.10.0 and output result to stdout 14 | // Usage: ./tm_genesis old_genesis_path 15 | /* 16 | Replace [TypeByte, Xxx] with {"type": "some-type", "data": Xxx} in RPC and all .json files by using go-wire/data. 17 | 18 | For instance, a pubkey old verison is: 19 | "pub_key": { 20 | 1, 21 | "83DDF8775937A4A12A2704269E2729FCFCD491B933C4B0A7FFE37FE41D7760D0" 22 | } 23 | 24 | now is: 25 | "pub_key": { 26 | "type": "ed25519", 27 | "data": "83DDF8775937A4A12A2704269E2729FCFCD491B933C4B0A7FFE37FE41D7760D0" 28 | } 29 | */ 30 | 31 | func main() { 32 | if len(os.Args) <= 1 { 33 | fmt.Printf("Usage: %s old_genesis.json \n", os.Args[0]) 34 | os.Exit(0) 35 | } 36 | 37 | jsonBytes, err := ioutil.ReadFile(os.Args[1]) 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | oGen := old.GenesisDocFromJSON(jsonBytes) 43 | nGen := NewGenesisDoc(oGen) 44 | bytes, _ := json.Marshal(nGen) 45 | fmt.Println(string(bytes)) 46 | } 47 | 48 | func NewGenesisDoc(old *old.GenesisDoc) *types.GenesisDoc { 49 | newGenesisDoc := &types.GenesisDoc{ 50 | AppHash: old.AppHash, 51 | ChainID: old.ChainID, 52 | GenesisTime: old.GenesisTime, 53 | Validators: []types.GenesisValidator{}, 54 | } 55 | for _, val := range old.Validators { 56 | one := types.GenesisValidator{} 57 | one.Amount = val.Amount 58 | one.Name = val.Name 59 | one.PubKey = val.PubKey 60 | 61 | newGenesisDoc.Validators = append(newGenesisDoc.Validators, one) 62 | } 63 | 64 | return newGenesisDoc 65 | } 66 | -------------------------------------------------------------------------------- /convert/blockstore.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/hxzqlh/tm-tools/util" 7 | "github.com/tendermint/tendermint/types" 8 | dbm "github.com/tendermint/tmlibs/db" 9 | ) 10 | 11 | // simulate the BlockStore api of blockchain 12 | func OnBlockStore(startHeight int) { 13 | if startHeight < 1 { 14 | panic("Invalid start height") 15 | } 16 | 17 | InitState() 18 | 19 | var lastBlockID *types.BlockID 20 | if startHeight == 1 { 21 | lastBlockID = &types.BlockID{} 22 | } else { 23 | nMeta := util.LoadNewBlockMeta(nBlockDb, startHeight-1) 24 | lastBlockID = &nMeta.BlockID 25 | } 26 | 27 | cnt := 0 28 | limit := 1000 29 | batch := nBlockDb.NewBatch() 30 | for i := startHeight; i <= totalHeight; i++ { 31 | cnt++ 32 | 33 | nBlock := NewBlockFromOld(i, lastBlockID) 34 | blockParts := nBlock.MakePartSet(types.DefaultBlockPartSize) 35 | nMeta := types.NewBlockMeta(nBlock, blockParts) 36 | // seen this BlockId's commit 37 | seenCommit := NewSeenCommit(i, &nMeta.BlockID) 38 | 39 | SaveBlock(batch, nBlock, nMeta, seenCommit) 40 | if cnt%limit == 0 { 41 | log.Printf("batch write %v/%v\n", cnt, totalHeight) 42 | batch.Write() 43 | batch = nBlockDb.NewBatch() 44 | } 45 | 46 | // update lastBlockID 47 | lastBlockID = &nMeta.BlockID 48 | } 49 | if cnt%limit != 0 { 50 | log.Printf("batch write %v/%v\n", cnt, totalHeight) 51 | batch.Write() 52 | } 53 | 54 | SaveState(lastBlockID) 55 | } 56 | 57 | func SaveBlock(batch dbm.Batch, block *types.Block, blockMeta *types.BlockMeta, seenCommit *types.Commit) { 58 | height := block.Height 59 | util.SaveNewBlockMeta2(batch, height, blockMeta) 60 | util.SaveNewBlockParts2(batch, height, block) 61 | util.SaveNewCommit2(batch, height-1, "C", block.LastCommit) 62 | util.SaveNewCommit2(batch, height, "SC", seenCommit) 63 | util.SaveNewBlockStoreStateJSON2(batch, height) 64 | } 65 | -------------------------------------------------------------------------------- /convert/commit.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "github.com/hxzqlh/tm-tools/old" 5 | "github.com/hxzqlh/tm-tools/util" 6 | "github.com/tendermint/go-crypto" 7 | "github.com/tendermint/tendermint/types" 8 | ) 9 | 10 | func NewSeenCommit(height int, lastBlockID *types.BlockID) *types.Commit { 11 | oCommit := util.LoadOldBlockCommit(oBlockDb, height, "SC") 12 | return NewCommit(oCommit, lastBlockID) 13 | } 14 | 15 | func NewCommit(oCommit *old.Commit, lastBlockID *types.BlockID) *types.Commit { 16 | nCommit := &types.Commit{} 17 | 18 | preCommits := []*types.Vote{} 19 | for i := 0; i < len(oCommit.Precommits); i++ { 20 | v := oCommit.Precommits[i] 21 | // node's commit may be nil 22 | if v == nil { 23 | preCommits = append(preCommits, nil) 24 | continue 25 | } 26 | 27 | one := &types.Vote{} 28 | one.BlockID = *lastBlockID 29 | one.Height = v.Height 30 | one.Round = v.Round 31 | one.Type = v.Type 32 | one.ValidatorIndex = i 33 | one.ValidatorAddress = nState.Validators.Validators[i].Address 34 | 35 | one.Signature = SignaVote(i, nState.ChainID, one) 36 | preCommits = append(preCommits, one) 37 | } 38 | 39 | nCommit.BlockID = *lastBlockID 40 | nCommit.Precommits = preCommits 41 | 42 | return nCommit 43 | } 44 | 45 | func convertSeenCommit(height int, lastBlockID *types.BlockID) { 46 | oCommit := util.LoadOldBlockCommit(oBlockDb, height, "SC") 47 | nCommit := NewCommit(oCommit, lastBlockID) 48 | util.SaveNewCommit(nBlockDb, height, "SC", nCommit) 49 | } 50 | 51 | func convertCommit(height int, lastBlockID *types.BlockID) { 52 | oCommit := util.LoadOldBlockCommit(oBlockDb, height, "C") 53 | nCommit := NewCommit(oCommit, lastBlockID) 54 | util.SaveNewCommit(nBlockDb, height, "C", nCommit) 55 | } 56 | 57 | func SignaVote(index int, chainID string, vote *types.Vote) crypto.Signature { 58 | if index >= len(privValidators.Validators) { 59 | panic("privValidators index overflow") 60 | } 61 | 62 | bytez := types.SignBytes(chainID, vote) 63 | return privValidators.Validators[index].PrivKey.Sign(bytez) 64 | } 65 | -------------------------------------------------------------------------------- /convert/state.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "github.com/hxzqlh/tm-tools/old" 5 | "github.com/hxzqlh/tm-tools/util" 6 | "github.com/tendermint/tendermint/state" 7 | "github.com/tendermint/tendermint/types" 8 | ) 9 | 10 | var nState *state.State 11 | 12 | func InitState() { 13 | oState := util.LoadOldState(oStateDb) 14 | nState = &state.State{} 15 | nState.AppHash = oState.AppHash 16 | nState.ChainID = oState.ChainID 17 | nState.GenesisDoc = NewGenesisDoc(oState.GenesisDoc) 18 | nState.LastBlockHeight = oState.LastBlockHeight 19 | nState.LastBlockTime = oState.LastBlockTime 20 | nState.LastValidators = NewValidatorSet(oState.LastValidators) 21 | nState.Validators = NewValidatorSet(oState.Validators) 22 | // need set LastBlockID before save state 23 | } 24 | 25 | func SaveState(lastBlockID *types.BlockID) { 26 | nState.LastBlockID = *lastBlockID 27 | util.SaveNewState(nStateDb, nState) 28 | } 29 | 30 | func NewGenesisDoc(old *old.GenesisDoc) *types.GenesisDoc { 31 | newGenesisDoc := &types.GenesisDoc{ 32 | AppHash: old.AppHash, 33 | ChainID: old.ChainID, 34 | GenesisTime: old.GenesisTime, 35 | Validators: []types.GenesisValidator{}, 36 | } 37 | for _, val := range old.Validators { 38 | one := types.GenesisValidator{} 39 | one.Amount = val.Amount 40 | one.Name = val.Name 41 | one.PubKey = val.PubKey 42 | 43 | newGenesisDoc.Validators = append(newGenesisDoc.Validators, one) 44 | } 45 | 46 | return newGenesisDoc 47 | } 48 | 49 | func NewValidatorSet(oValidatorSet *old.ValidatorSet) *types.ValidatorSet { 50 | nValidatorSet := &types.ValidatorSet{ 51 | Validators: []*types.Validator{}, 52 | } 53 | 54 | // Validators 55 | for _, val := range oValidatorSet.Validators { 56 | one := &types.Validator{} 57 | one.Accum = val.Accum 58 | one.Address = val.Address 59 | one.PubKey = val.PubKey 60 | one.VotingPower = val.VotingPower 61 | 62 | nValidatorSet.Validators = append(nValidatorSet.Validators, one) 63 | } 64 | 65 | // Proposer 66 | // NOTE: no this fiedl in old version, we find by new api 67 | nValidatorSet.Proposer = nValidatorSet.GetProposer() 68 | 69 | return nValidatorSet 70 | } 71 | -------------------------------------------------------------------------------- /cmd/viewer/blockchain.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/hxzqlh/tm-tools/util" 8 | ) 9 | 10 | func LoadBlock(height int) { 11 | var res []byte 12 | 13 | switch *ver { 14 | case "new": 15 | block := util.LoadNewBlock(ldb, height) 16 | res, _ = json.Marshal(block) 17 | case "old": 18 | block := util.LoadOldBlock(ldb, height) 19 | res, _ = json.Marshal(block) 20 | default: 21 | panic(ver) 22 | } 23 | 24 | fmt.Println(string(res)) 25 | } 26 | 27 | //state 28 | func LoadState() { 29 | var res []byte 30 | 31 | switch *ver { 32 | case "new": 33 | s := util.LoadNewState(ldb) 34 | res, _ = json.Marshal(s) 35 | case "old": 36 | s := util.LoadOldState(ldb) 37 | res, _ = json.Marshal(s) 38 | default: 39 | panic(ver) 40 | } 41 | 42 | fmt.Println(string(res)) 43 | } 44 | 45 | //meta 46 | func LoadBlockMeta(height int) { 47 | var res []byte 48 | 49 | switch *ver { 50 | case "new": 51 | meta := util.LoadNewBlockMeta(ldb, height) 52 | res, _ = json.Marshal(meta) 53 | case "old": 54 | meta := util.LoadOldBlockMeta(ldb, height) 55 | res, _ = json.Marshal(meta) 56 | default: 57 | panic(ver) 58 | } 59 | 60 | fmt.Println(string(res)) 61 | } 62 | 63 | //Part 64 | func LoadBlockPart(height int, index int) { 65 | var res []byte 66 | 67 | switch *ver { 68 | case "new": 69 | part := util.LoadNewBlockPart(ldb, height, index) 70 | res, _ = json.Marshal(part) 71 | case "old": 72 | part := util.LoadOldBlockPart(ldb, height, index) 73 | res, _ = json.Marshal(part) 74 | default: 75 | panic(ver) 76 | } 77 | 78 | fmt.Println(string(res)) 79 | } 80 | 81 | //commit 82 | func LoadBlockCommit(height int, prefix string) { 83 | var res []byte 84 | 85 | switch *ver { 86 | case "new": 87 | commit := util.LoadNewBlockCommit(ldb, height, prefix) 88 | res, _ = json.Marshal(commit) 89 | case "old": 90 | commit := util.LoadOldBlockCommit(ldb, height, prefix) 91 | res, _ = json.Marshal(commit) 92 | default: 93 | panic(ver) 94 | } 95 | 96 | fmt.Println(string(res)) 97 | } 98 | 99 | func LoadAbciResponses() { 100 | abciResps := util.LoadAbciResps(ldb) 101 | res, _ := json.Marshal(abciResps) 102 | fmt.Println(string(res)) 103 | } 104 | -------------------------------------------------------------------------------- /cmd/viewer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/hxzqlh/tm-tools/util" 12 | dbm "github.com/tendermint/tmlibs/db" 13 | ) 14 | 15 | /* 16 | Usage: $ tm_view -db /path/of/db [-a get|getall|block] [-q key] [-d] [-v new|old] [-t height] 17 | 18 | // -db : db，Note: the db path cannot end with "/" 19 | // [-a get|getall|block]： read the value of a key | output all keyes | read block info 20 | // [-q key] ：key format 21 | // [-d]: whether decode value，default is "false" 22 | // [-v new|old] ：new(0.10.0), old(0.7.3), default is "new" 23 | // [-t height]: block height，workes with "-a block" arg to read block info at height "N" 24 | 25 | examples： 26 | $ tm_view -db /path/of/blockstore.db -a getall 27 | $ tm_view -db /path/of/blockstore.db -a block -t 1 -d 28 | $ tm_view -db /path/of/blockstore.db -q "H:1" -d -v old 29 | $ tm_view -db /path/of/state.db -q "stateKey" -d -v old 30 | 31 | | key format | value type | examples | 32 | | ---- |-----| ---- | 33 | | `stateKey` | raw byte of state | | 34 | | `abciResponsesKey` | raw byte of ABCI Responses | | 35 | | `blockStore` | raw json | "blockStore": {"Height":32} | 36 | | `H:{height}` | raw byte of block meta | H:1 | 37 | | `P:{height}:{index}`| raw byte of block part | P:1:0, P:32:0, P:32:1 | 38 | | `SC:{height}` | raw byte of block seen commit | SC:1, SC:32 | 39 | | `C:{height-1}` | raw byte of block commit | C:0, SC:31 | 40 | */ 41 | 42 | var dbpath = flag.String("db", os.ExpandEnv("$HOME/.tendermint")+"/trade.db", "database db") 43 | var action = flag.String("a", "get", "get key from database") 44 | var key = flag.String("q", "", "the query string of database") 45 | var decode = flag.Bool("d", false, "whether decode data") 46 | var limit = flag.Int("l", 0, "limit of query list") 47 | var ver = flag.String("v", "new", "version of tendermint") 48 | var height = flag.Int("t", 1, "block height") 49 | var ldb dbm.DB 50 | 51 | func main() { 52 | if len(os.Args) < 5 { 53 | fmt.Printf("Usage: %s -db /path/of/db [-a get|getall|block] [-q key] [-d] [-v new|old] [-t height]\n", os.Args[0]) 54 | os.Exit(0) 55 | } 56 | 57 | flag.Parse() 58 | 59 | ldb = dbm.NewDB(util.FileNameNoExt(*dbpath), "leveldb", filepath.Dir(*dbpath)) 60 | defer ldb.Close() 61 | 62 | if *action == "get" { 63 | get() 64 | } else if *action == "getall" { 65 | getall() 66 | } else if *action == "block" { 67 | LoadBlock(*height) 68 | } 69 | } 70 | 71 | func get() { 72 | data := ldb.Get([]byte(*key)) 73 | if len(data) == 0 { 74 | fmt.Println(*key, "not exist") 75 | return 76 | } 77 | 78 | if !*decode { 79 | fmt.Println(string(data)) 80 | return 81 | } 82 | 83 | if *key == util.StateKey { 84 | LoadState() 85 | } else if *key == util.AbciResponsesKey { 86 | LoadAbciResponses() 87 | } else if (*key)[0] == 'H' { 88 | height, _ := strconv.Atoi(strings.Split(*key, ":")[1]) 89 | LoadBlockMeta(height) 90 | } else if (*key)[0] == 'P' { 91 | height, _ := strconv.Atoi(strings.Split(*key, ":")[1]) 92 | index, _ := strconv.Atoi(strings.Split(*key, ":")[2]) 93 | LoadBlockPart(height, index) 94 | } else if (*key)[0] == 'C' || (*key)[:2] == "SC" { 95 | prefix := strings.Split(*key, ":")[0] 96 | height, _ := strconv.Atoi(strings.Split(*key, ":")[1]) 97 | LoadBlockCommit(height, prefix) 98 | } else { 99 | fmt.Println(string(data)) 100 | } 101 | } 102 | 103 | func getall() { 104 | prefix := *key 105 | level := ldb.(*dbm.GoLevelDB).DB() 106 | query := level.NewIterator(nil, nil) 107 | //query := level.NewIterator(util.BytesPrefix([]byte(prefix)), nil) 108 | defer query.Release() 109 | query.Seek([]byte(prefix)) 110 | i := 0 111 | for { 112 | fmt.Printf("%s\n", string(query.Key())) 113 | i++ 114 | if !query.Next() { 115 | break 116 | } 117 | if i == *limit { 118 | break 119 | } 120 | } 121 | if query.Error() != nil { 122 | panic(query.Error()) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /old/tm_v0.7.3.go: -------------------------------------------------------------------------------- 1 | package old 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "sync" 7 | "time" 8 | 9 | crypto "github.com/tendermint/go-crypto" 10 | wire "github.com/tendermint/go-wire" 11 | dbm "github.com/tendermint/tmlibs/db" 12 | "github.com/tendermint/tmlibs/merkle" 13 | ) 14 | 15 | // following structs are based on tendermint v0.7.3 16 | 17 | type Block struct { 18 | *Header `json:"header"` 19 | *Data `json:"data"` 20 | LastCommit *Commit `json:"last_commit"` 21 | } 22 | 23 | type BlockStoreStateJSON struct { 24 | Height int 25 | } 26 | 27 | type BlockMeta struct { 28 | Hash []byte `json:"hash"` // The block hash 29 | Header *Header `json:"header"` // The block's Header 30 | PartsHeader PartSetHeader `json:"parts_header"` // The PartSetHeader, for transfer 31 | } 32 | 33 | type Header struct { 34 | ChainID string `json:"chain_id"` 35 | Height int `json:"height"` 36 | Time time.Time `json:"time"` 37 | NumTxs int `json:"num_txs"` 38 | LastBlockHash []byte `json:"last_block_hash"` 39 | LastBlockParts PartSetHeader `json:"last_block_parts"` 40 | LastCommitHash []byte `json:"last_commit_hash"` 41 | DataHash []byte `json:"data_hash"` 42 | ValidatorsHash []byte `json:"validators_hash"` 43 | AppHash []byte `json:"app_hash"` // state merkle root of txs from the previous block 44 | } 45 | 46 | type PartSetHeader struct { 47 | Total int `json:"total"` 48 | Hash []byte `json:"hash"` 49 | } 50 | 51 | type Data struct { 52 | // Txs that will be applied by state @ block.Height+1. 53 | // NOTE: not all txs here are valid. We're just agreeing on the order first. 54 | // This means that block.AppHash does not include these txs. 55 | Txs Txs `json:"txs"` 56 | // Volatile 57 | hash []byte 58 | } 59 | 60 | type Tx []byte 61 | type Txs []Tx 62 | 63 | type Commit struct { 64 | // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. 65 | // Any peer with a block can gossip precommits by index with a peer without recalculating the 66 | // active ValidatorSet. 67 | Precommits []*Vote `json:"precommits"` 68 | // Volatile 69 | firstPrecommit *Vote 70 | hash []byte 71 | bitArray *BitArray 72 | } 73 | 74 | type BitArray struct { 75 | mtx sync.Mutex 76 | Bits int `json:"bits"` // NOTE: persisted via reflect, must be exported 77 | Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported 78 | } 79 | 80 | type Vote struct { 81 | Height int `json:"height"` 82 | Round int `json:"round"` 83 | Type byte `json:"type"` 84 | BlockHash []byte `json:"block_hash"` // empty if vote is nil. 85 | BlockPartsHeader PartSetHeader `json:"block_parts_header"` // zero if vote is nil. 86 | Signature crypto.SignatureEd25519 `json:"signature"` 87 | } 88 | 89 | type Part struct { 90 | Index int `json:"index"` 91 | Bytes []byte `json:"bytes"` 92 | Proof merkle.SimpleProof `json:"proof"` 93 | // Cache 94 | hash []byte 95 | } 96 | 97 | type State struct { 98 | mtx sync.Mutex 99 | db dbm.DB 100 | GenesisDoc *GenesisDoc 101 | ChainID string 102 | LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. 103 | LastBlockHash []byte 104 | LastBlockParts PartSetHeader 105 | LastBlockTime time.Time 106 | Validators *ValidatorSet 107 | LastValidators *ValidatorSet 108 | AppHash []byte 109 | } 110 | 111 | type GenesisDoc struct { 112 | GenesisTime time.Time `json:"genesis_time"` 113 | ChainID string `json:"chain_id"` 114 | Validators []GenesisValidator `json:"validators"` 115 | AppHash []byte `json:"app_hash"` 116 | } 117 | 118 | type GenesisValidator struct { 119 | PubKey crypto.PubKey `json:"pub_key"` 120 | Amount int64 `json:"amount"` 121 | Name string `json:"name"` 122 | } 123 | 124 | type ValidatorSet struct { 125 | Validators []*Validator // NOTE: persisted via reflect, must be exported. 126 | // cached (unexported) 127 | proposer *Validator 128 | totalVotingPower int64 129 | } 130 | 131 | type Validator struct { 132 | Address []byte `json:"address"` 133 | PubKey crypto.PubKey `json:"pub_key"` 134 | LastCommitHeight int `json:"last_commit_height"` 135 | VotingPower int64 `json:"voting_power"` 136 | Accum int64 `json:"accum"` 137 | } 138 | 139 | type PrivValidator struct { 140 | Address []byte `json:"address"` 141 | PubKey crypto.PubKey `json:"pub_key"` 142 | LastHeight int `json:"last_height"` 143 | LastRound int `json:"last_round"` 144 | LastStep int8 `json:"last_step"` 145 | LastSignature crypto.Signature `json:"last_signature"` // so we dont lose signatures 146 | LastSignBytes []byte `json:"last_signbytes"` // so we dont lose signatures 147 | // PrivKey should be empty if a Signer other than the default is being used. 148 | PrivKey crypto.PrivKey `json:"priv_key"` 149 | Signer `json:"-"` 150 | // For persistence. 151 | // Overloaded for testing. 152 | filePath string 153 | mtx sync.Mutex 154 | } 155 | 156 | type Signer interface { 157 | Sign(msg []byte) crypto.Signature 158 | } 159 | 160 | // Implements Signer 161 | type DefaultSigner struct { 162 | priv crypto.PrivKey 163 | } 164 | 165 | func NewDefaultSigner(priv crypto.PrivKey) *DefaultSigner { 166 | return &DefaultSigner{priv: priv} 167 | } 168 | 169 | // Implements Signer 170 | func (ds *DefaultSigner) Sign(msg []byte) crypto.Signature { 171 | return ds.priv.Sign(msg) 172 | } 173 | 174 | func GenesisDocFromJSON(jsonBlob []byte) (genState *GenesisDoc) { 175 | var err error 176 | wire.ReadJSONPtr(&genState, jsonBlob, &err) 177 | if err != nil { 178 | panic(fmt.Sprintf("Couldn't read GenesisDoc: %v", err)) 179 | } 180 | return 181 | } 182 | 183 | func LoadPrivValidator(filePath string) *PrivValidator { 184 | privValJSONBytes, err := ioutil.ReadFile(filePath) 185 | if err != nil { 186 | panic(err.Error()) 187 | } 188 | privVal := wire.ReadJSON(&PrivValidator{}, privValJSONBytes, &err).(*PrivValidator) 189 | if err != nil { 190 | panic(fmt.Sprintf("Error reading PrivValidator from %v: %v\n", filePath, err)) 191 | } 192 | privVal.filePath = filePath 193 | privVal.Signer = NewDefaultSigner(privVal.PrivKey) 194 | return privVal 195 | } 196 | 197 | // cswal 198 | type ConsensusLogMessage struct { 199 | Time time.Time `json:"time"` 200 | Msg ConsensusLogMessageInterface `json:"msg"` 201 | } 202 | 203 | type ConsensusLogMessageInterface interface{} 204 | 205 | // 0x01 206 | type EventDataRoundState struct { 207 | Height int `json:"height"` 208 | Round int `json:"round"` 209 | Step string `json:"step"` 210 | // private, not exposed to websockets 211 | RoundState interface{} `json:"-"` 212 | } 213 | 214 | //0x02 215 | type MsgInfo struct { 216 | Msg ConsensusMessage `json:"msg"` 217 | PeerKey string `json:"peer_key"` 218 | } 219 | 220 | type ConsensusMessage interface{} 221 | 222 | // 0x03 223 | // internally generated messages which may update the state 224 | type TimeoutInfo struct { 225 | Duration time.Duration `json:"duration"` 226 | Height int `json:"height"` 227 | Round int `json:"round"` 228 | Step RoundStepType `json:"step"` 229 | } 230 | 231 | type RoundStepType uint8 // These must be numeric, ordered. 232 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tm-tools 2 | 3 | There are 2 tools for tendermint data: 4 | 5 | * **tm-migrator**: migrate tendermint data from `v0.7.3` to `v0.10.0` 6 | * **tm-viewer**: view tendermint data in `blockstore.db` or `state.db` 7 | 8 | ## tm-migrator 9 | 10 | ``` 11 | Usage: tm_migrator -old tmroot -new tmroot -priv priv_dir [-s startHeight] 12 | 13 | -old tmroot: dir of old tendermint root 14 | -new tmroot: dir of new tendermint root to store converted data 15 | -priv priv_dir: dir to place other validators's old `priv_validator.json` 16 | -s startHeight: from which height to convert tendermint data, default is `1` 17 | ``` 18 | 19 | Q: Why need `priv` arg? 20 | 21 | A: A blockchain may consistes of one or more nodes. For every block, each node will verify `SeenCommit` and `Commit` which was produced by all validators before adding to blockchain. While `SeenCommit` and `Commit` were signed by validator, without other validators' `priv_validator.json` config info, this validaotr cannot reconstruct `SeenCommit` and `Commit` infos of the block. 22 | 23 | ## tm-viewer 24 | 25 | ``` 26 | Usage: $ tm_view -db /path/of/db [-a get|getall|block] [-q key] [-d] [-v new|old] [-t height] 27 | 28 | // -db : db，Note: the db path cannot end with "/" 29 | // [-a get|getall|block]： read the value of a key | output all keyes | read block info 30 | // [-q key] ：key format, please see following "Tendermint data" section 31 | // [-d]: whether decode value，default is "false" 32 | // [-v new|old] ：new(0.10.0), old(0.7.3), default is "new" 33 | // [-t height]: block height，workes with "-a block" arg to read block info at height "N" 34 | 35 | examples： 36 | $ tm_view -db /path/of/blockstore.db -a getall 37 | $ tm_view -db /path/of/blockstore.db -a block -t 1 -d 38 | $ tm_view -db /path/of/blockstore.db -q "H:1" -d -v old 39 | $ tm_view -db /path/of/state.db -q "stateKey" -d -v old 40 | ``` 41 | 42 | ## Tendermint data 43 | 44 | In tendermint root dir(by default, is `~/.tendermint`), tendermint data is placed in its subdir `data/blockstore.db` or `data/state.db`. 45 | 46 | ``` 47 | Allen@MacBook-Pro:~ ls -l ~/.tendermint.v0.10.0/data/ 48 | drwxr-xr-x 8 Allen staff 272 Oct 15 20:23 blockstore.db 49 | drwx------ 3 Allen staff 102 Oct 15 20:23 cs.wal 50 | drwx------ 3 Allen staff 102 Oct 15 20:23 mempool.wal 51 | drwxr-xr-x 8 Allen staff 272 Oct 15 20:23 state.db 52 | drwxr-xr-x 7 Allen staff 238 Oct 15 20:23 tx_index.db 53 | ``` 54 | 55 | ``` 56 | Allen@MacBook-Pro:~ ls -l ~/.tendermint.v0.7.3/data/ 57 | drwxr-xr-x 8 Allen staff 272 Oct 15 20:22 blockstore.db 58 | -rw------- 1 Allen staff 829784 Oct 15 20:12 cswal 59 | -rw------- 1 Allen staff 3 Oct 15 20:10 mempool_wal 60 | drwxr-xr-x 10 Allen staff 340 Oct 15 20:22 state.db 61 | ``` 62 | 63 | `state.db` is used to store `state` info, the leveldb keyes are: 64 | 65 | * "stateKey" 66 | * "abciResponsesKey"(v0.10.0 only) 67 | 68 | `blockStore.db` is used to store `block` info, the leveldb keyes are(assuming the blockchain height is 32): 69 | 70 | * "blockStore": blockchain height info. {"Height":32} 71 | * "H:1" ... "H:32": block meta info, 72 | * "P:1:0" ... "P:32:0": block part info. Block may be sliced to several parts, for each part, the key is "P:{height}:{partIndex}", partIndex start from `0`. 73 | * "SC:1" ... "SC:32": block seen commit info. 74 | * "C:0" ... "C:31": block commit info. 75 | 76 | | key format | value type | examples | 77 | | ---- |-----| ---- | 78 | | `stateKey` | raw byte of state | | 79 | | `abciResponsesKey` | raw byte of ABCI Responses | | 80 | | `blockStore` | raw json | "blockStore": {"Height":32} | 81 | | `H:{height}` | raw byte of block meta | H:1 | 82 | | `P:{height}:{index}`| raw byte of block part | P:1:0, P:32:0, P:32:1 | 83 | | `SC:{height}` | raw byte of block seen commit | SC:1, SC:32 | 84 | | `C:{height-1}` | raw byte of block commit | C:0, SC:31 | 85 | 86 | 87 | ## Tendermint Data Struct Differences 88 | 89 | ### Commit & SeenCommit 90 | 91 | ``` 92 | // v0.10.0 93 | type Commit struct { 94 | // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. 95 | // Any peer with a block can gossip precommits by index with a peer without recalculating the 96 | // active ValidatorSet. 97 | BlockID BlockID `json:"blockID"` 98 | Precommits []*Vote `json:"precommits"` 99 | 100 | // Volatile 101 | firstPrecommit *Vote 102 | hash data.Bytes 103 | bitArray *BitArray 104 | } 105 | ``` 106 | 107 | ``` 108 | // v0.7.3 109 | type Commit struct { 110 | // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. 111 | // Any peer with a block can gossip precommits by index with a peer without recalculating the 112 | // active ValidatorSet. 113 | Precommits []*Vote `json:"precommits"` 114 | 115 | // Volatile 116 | firstPrecommit *Vote 117 | hash []byte 118 | bitArray *BitArray 119 | } 120 | ``` 121 | 122 | ### BlockMeta 123 | 124 | ``` 125 | // v0.10.0 126 | type BlockMeta struct { 127 | BlockID BlockID `json:"block_id"` // the block hash and partsethash 128 | Header *Header `json:"header"` // The block's Header 129 | } 130 | 131 | type BlockID struct { 132 | Hash data.Bytes `json:"hash"` 133 | PartsHeader PartSetHeader `json:"parts"` 134 | } 135 | ``` 136 | 137 | ``` 138 | // v0.7.3 139 | type BlockMeta struct { 140 | Hash []byte `json:"hash"` // The block hash 141 | Header *Header `json:"header"` // The block's Header 142 | PartsHeader PartSetHeader `json:"parts_header"` // The PartSetHeader, for transfer 143 | } 144 | ``` 145 | 146 | ### Part 147 | 148 | ``` 149 | // v0.10.0 150 | type Part struct { 151 | Index int `json:"index"` 152 | Bytes data.Bytes `json:"bytes"` 153 | Proof merkle.SimpleProof `json:"proof"` 154 | 155 | // Cache 156 | hash []byte 157 | } 158 | ``` 159 | 160 | ``` 161 | // v0.7.3 162 | type Part struct { 163 | Index int `json:"index"` 164 | Bytes []byte `json:"bytes"` 165 | Proof merkle.SimpleProof `json:"proof"` 166 | 167 | // Cache 168 | hash []byte 169 | } 170 | ``` 171 | 172 | ### Block 173 | 174 | ``` 175 | // v0.10.0 176 | type Block struct { 177 | *Header `json:"header"` 178 | *Data `json:"data"` 179 | LastCommit *Commit `json:"last_commit"` 180 | } 181 | ``` 182 | 183 | ``` 184 | // v0.7.3 185 | type Block struct { 186 | *Header `json:"header"` 187 | *Data `json:"data"` 188 | LastCommit *Commit `json:"last_commit"` 189 | } 190 | ``` 191 | 192 | ### State 193 | 194 | ``` 195 | // v0.10.0 196 | type State struct { 197 | // mtx for writing to db 198 | mtx sync.Mutex 199 | db dbm.DB 200 | 201 | // should not change 202 | GenesisDoc *types.GenesisDoc 203 | ChainID string 204 | 205 | // updated at end of SetBlockAndValidators 206 | LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. 207 | LastBlockID types.BlockID 208 | LastBlockTime time.Time 209 | Validators *types.ValidatorSet 210 | LastValidators *types.ValidatorSet // block.LastCommit validated against this 211 | 212 | // AppHash is updated after Commit 213 | AppHash []byte 214 | 215 | TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. 216 | 217 | // Intermediate results from processing 218 | // Persisted separately from the state 219 | abciResponses *ABCIResponses 220 | 221 | logger log.Logger 222 | } 223 | ``` 224 | 225 | ``` 226 | // v0.7.3 227 | type State struct { 228 | mtx sync.Mutex 229 | db dbm.DB 230 | GenesisDoc *types.GenesisDoc 231 | ChainID string 232 | LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. 233 | LastBlockHash []byte 234 | LastBlockParts types.PartSetHeader 235 | LastBlockTime time.Time 236 | Validators *types.ValidatorSet 237 | LastValidators *types.ValidatorSet 238 | AppHash []byte 239 | } 240 | ``` 241 | 242 | ## Further Improvement 243 | 244 | Wating To Be Done. -------------------------------------------------------------------------------- /util/blockchain.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/hxzqlh/tm-tools/old" 9 | wire "github.com/tendermint/go-wire" 10 | "github.com/tendermint/tendermint/blockchain" 11 | "github.com/tendermint/tendermint/state" 12 | "github.com/tendermint/tendermint/types" 13 | dbm "github.com/tendermint/tmlibs/db" 14 | ) 15 | 16 | var ( 17 | BlockStoreKey = "blockStore" 18 | StateKey = "stateKey" 19 | AbciResponsesKey = "abciResponsesKey" 20 | ) 21 | 22 | // blockstore json 23 | func LoadOldBlockStoreStateJSON(ldb dbm.DB) old.BlockStoreStateJSON { 24 | bytes := ldb.Get([]byte(BlockStoreKey)) 25 | if bytes == nil { 26 | return old.BlockStoreStateJSON{ 27 | Height: 0, 28 | } 29 | } 30 | bsj := old.BlockStoreStateJSON{} 31 | err := json.Unmarshal(bytes, &bsj) 32 | if err != nil { 33 | panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes)) 34 | } 35 | return bsj 36 | } 37 | 38 | func SaveNewBlockStoreStateJSON(ldb dbm.DB, totalHeight int) { 39 | bsj := blockchain.BlockStoreStateJSON{ 40 | Height: totalHeight, 41 | } 42 | bytes, err := json.Marshal(bsj) 43 | if err != nil { 44 | panic(fmt.Sprintf("Could not marshal state bytes: %v", err)) 45 | } 46 | ldb.Set([]byte(BlockStoreKey), bytes) 47 | } 48 | 49 | func SaveNewBlockStoreStateJSON2(batch dbm.Batch, totalHeight int) { 50 | bsj := blockchain.BlockStoreStateJSON{ 51 | Height: totalHeight, 52 | } 53 | bytes, err := json.Marshal(bsj) 54 | if err != nil { 55 | panic(fmt.Sprintf("Could not marshal state bytes: %v", err)) 56 | } 57 | batch.Set([]byte(BlockStoreKey), bytes) 58 | } 59 | 60 | // block 61 | func LoadNewBlock(ldb dbm.DB, height int) *types.Block { 62 | var n int 63 | var err error 64 | bytez := []byte{} 65 | meta := LoadNewBlockMeta(ldb, height) 66 | for i := 0; i < meta.BlockID.PartsHeader.Total; i++ { 67 | part := LoadNewBlockPart(ldb, height, i) 68 | bytez = append(bytez, part.Bytes...) 69 | } 70 | block := wire.ReadBinary(&types.Block{}, bytes.NewReader(bytez), 0, &n, &err).(*types.Block) 71 | if err != nil { 72 | panic(err) 73 | } 74 | return block 75 | } 76 | 77 | func LoadOldBlock(ldb dbm.DB, height int) *old.Block { 78 | var n int 79 | var err error 80 | bytez := []byte{} 81 | meta := LoadOldBlockMeta(ldb, height) 82 | for i := 0; i < meta.PartsHeader.Total; i++ { 83 | part := LoadOldBlockPart(ldb, height, i) 84 | bytez = append(bytez, part.Bytes...) 85 | } 86 | block := wire.ReadBinary(&old.Block{}, bytes.NewReader(bytez), 0, &n, &err).(*old.Block) 87 | if err != nil { 88 | panic(err) 89 | } 90 | return block 91 | } 92 | 93 | // header 94 | func LoadNewBlockMeta(ldb dbm.DB, height int) *types.BlockMeta { 95 | var n int 96 | var err error 97 | buf := ldb.Get(calcBlockMetaKey(height)) 98 | meta := wire.ReadBinary(&types.BlockMeta{}, bytes.NewReader(buf), 0, &n, &err).(*types.BlockMeta) 99 | if err != nil { 100 | panic(err) 101 | } 102 | return meta 103 | } 104 | 105 | func LoadOldBlockMeta(ldb dbm.DB, height int) *old.BlockMeta { 106 | var n int 107 | var err error 108 | buf := ldb.Get(calcBlockMetaKey(height)) 109 | meta := wire.ReadBinary(&old.BlockMeta{}, bytes.NewReader(buf), 0, &n, &err).(*old.BlockMeta) 110 | if err != nil { 111 | panic(err) 112 | } 113 | return meta 114 | } 115 | 116 | func SaveNewBlockMeta(ldb dbm.DB, height int, blockMeta *types.BlockMeta) { 117 | metaBytes := wire.BinaryBytes(blockMeta) 118 | ldb.Set(calcBlockMetaKey(height), metaBytes) 119 | } 120 | 121 | func SaveNewBlockMeta2(batch dbm.Batch, height int, blockMeta *types.BlockMeta) { 122 | metaBytes := wire.BinaryBytes(blockMeta) 123 | batch.Set(calcBlockMetaKey(height), metaBytes) 124 | } 125 | 126 | // part 127 | func LoadNewBlockPart(ldb dbm.DB, height int, index int) *types.Part { 128 | buf := ldb.Get(calcBlockPartKey(height, index)) 129 | r, n, err := bytes.NewReader(buf), new(int), new(error) 130 | part := wire.ReadBinary(&types.Part{}, r, 0, n, err).(*types.Part) 131 | if *err != nil { 132 | panic(*err) 133 | } 134 | return part 135 | } 136 | 137 | func LoadOldBlockPart(ldb dbm.DB, height int, index int) *old.Part { 138 | buf := ldb.Get(calcBlockPartKey(height, index)) 139 | r, n, err := bytes.NewReader(buf), new(int), new(error) 140 | part := wire.ReadBinary(&old.Part{}, r, 0, n, err).(*old.Part) 141 | if *err != nil { 142 | panic(*err) 143 | } 144 | return part 145 | } 146 | 147 | func SaveNewBlockParts(ldb dbm.DB, height int, block *types.Block) { 148 | blockParts := block.MakePartSet(types.DefaultBlockPartSize) 149 | for index := 0; index < blockParts.Total(); index++ { 150 | SaveNewBlockPart(ldb, height, index, blockParts.GetPart(index)) 151 | } 152 | } 153 | 154 | func SaveNewBlockParts2(batch dbm.Batch, height int, block *types.Block) { 155 | blockParts := block.MakePartSet(types.DefaultBlockPartSize) 156 | for index := 0; index < blockParts.Total(); index++ { 157 | SaveNewBlockPart2(batch, height, index, blockParts.GetPart(index)) 158 | } 159 | } 160 | 161 | func SaveNewBlockPart(ldb dbm.DB, height int, index int, part *types.Part) { 162 | partBytes := wire.BinaryBytes(part) 163 | ldb.Set(calcBlockPartKey(height, index), partBytes) 164 | } 165 | 166 | func SaveNewBlockPart2(batch dbm.Batch, height int, index int, part *types.Part) { 167 | partBytes := wire.BinaryBytes(part) 168 | batch.Set(calcBlockPartKey(height, index), partBytes) 169 | } 170 | 171 | // commit 172 | func LoadNewBlockCommit(ldb dbm.DB, height int, prefix string) *types.Commit { 173 | var buf []byte 174 | 175 | if prefix == "C" { 176 | buf = ldb.Get(calcBlockCommitKey(height)) 177 | } else if prefix == "SC" { 178 | buf = ldb.Get(calcSeenCommitKey(height)) 179 | } 180 | 181 | blockCommit := &types.Commit{} 182 | r, n, err := bytes.NewReader(buf), new(int), new(error) 183 | wire.ReadBinaryPtr(&blockCommit, r, 0, n, err) 184 | if *err != nil { 185 | panic(err) 186 | } 187 | return blockCommit 188 | } 189 | 190 | func LoadOldBlockCommit(ldb dbm.DB, height int, prefix string) *old.Commit { 191 | var buf []byte 192 | 193 | if prefix == "C" { 194 | buf = ldb.Get(calcBlockCommitKey(height)) 195 | } else if prefix == "SC" { 196 | buf = ldb.Get(calcSeenCommitKey(height)) 197 | } 198 | 199 | blockCommit := &old.Commit{} 200 | 201 | r, n, err := bytes.NewReader(buf), new(int), new(error) 202 | wire.ReadBinaryPtr(&blockCommit, r, 0, n, err) 203 | if *err != nil { 204 | panic(err) 205 | } 206 | return blockCommit 207 | } 208 | 209 | func SaveNewCommit(ldb dbm.DB, height int, prefix string, commit *types.Commit) { 210 | var key []byte 211 | switch prefix { 212 | case "C": 213 | key = calcBlockCommitKey(height) 214 | case "SC": 215 | key = calcSeenCommitKey(height) 216 | default: 217 | panic(prefix) 218 | } 219 | 220 | buf := wire.BinaryBytes(commit) 221 | ldb.Set(key, buf) 222 | } 223 | 224 | func SaveNewCommit2(batch dbm.Batch, height int, prefix string, commit *types.Commit) { 225 | var key []byte 226 | switch prefix { 227 | case "C": 228 | key = calcBlockCommitKey(height) 229 | case "SC": 230 | key = calcSeenCommitKey(height) 231 | default: 232 | panic(prefix) 233 | } 234 | 235 | buf := wire.BinaryBytes(commit) 236 | batch.Set(key, buf) 237 | } 238 | 239 | // state 240 | func LoadOldState(ldb dbm.DB) *old.State { 241 | buf := ldb.Get([]byte(StateKey)) 242 | s := &old.State{} 243 | r, n, err := bytes.NewReader(buf), new(int), new(error) 244 | wire.ReadBinaryPtr(&s, r, 0, n, err) 245 | if *err != nil { 246 | panic(err) 247 | } 248 | return s 249 | } 250 | 251 | func LoadNewState(ldb dbm.DB) *state.State { 252 | buf := ldb.Get([]byte(StateKey)) 253 | s := &state.State{} 254 | r, n, err := bytes.NewReader(buf), new(int), new(error) 255 | wire.ReadBinaryPtr(&s, r, 0, n, err) 256 | if *err != nil { 257 | panic(err) 258 | } 259 | return s 260 | } 261 | 262 | func SaveNewState(ldb dbm.DB, s *state.State) { 263 | buf := s.Bytes() 264 | ldb.Set([]byte(StateKey), buf) 265 | } 266 | 267 | // abciResps 268 | func LoadAbciResps(ldb dbm.DB) *state.ABCIResponses { 269 | buf := ldb.Get([]byte(AbciResponsesKey)) 270 | resps := &state.ABCIResponses{} 271 | 272 | r, n, err := bytes.NewReader(buf), new(int), new(error) 273 | wire.ReadBinaryPtr(resps, r, 0, n, err) 274 | if *err != nil { 275 | panic(err) 276 | } 277 | return resps 278 | } 279 | 280 | //============================================================================== 281 | 282 | func calcBlockMetaKey(height int) []byte { 283 | return []byte(fmt.Sprintf("H:%v", height)) 284 | } 285 | 286 | func calcBlockPartKey(height int, partIndex int) []byte { 287 | return []byte(fmt.Sprintf("P:%v:%v", height, partIndex)) 288 | } 289 | 290 | func calcBlockCommitKey(height int) []byte { 291 | return []byte(fmt.Sprintf("C:%v", height)) 292 | } 293 | 294 | func calcSeenCommitKey(height int) []byte { 295 | return []byte(fmt.Sprintf("SC:%v", height)) 296 | } 297 | --------------------------------------------------------------------------------