├── .gitignore ├── README.md ├── contrib ├── Dockerfile ├── README.md └── build ├── importer.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | state/* 2 | extra/* 3 | blockchain/* 4 | main 5 | ecp 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ##### Ethereum chain parser 2 | 3 | **E**thereum **C**hain **P**arser listens for Ethereum (block) chain events and turns them into MongoDB 4 | records (currently). This could be easily extended to allow importing to 5 | various database formats. These records can be accessed from a third client application. 6 | 7 | It's currently being used for the Ethereum chain explorer. 8 | 9 | This is still pre-Alpha software, not ready for production. 10 | -------------------------------------------------------------------------------- /contrib/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.4.2 2 | MAINTAINER maran@ethdev.com 3 | 4 | RUN go get -u github.com/tools/godep 5 | RUN go get -d github.com/ethereum/go-ethereum/cmd/geth 6 | RUN go get -u gopkg.in/mgo.v2 7 | RUN cd /go/src/github.com/ethereum/go-ethereum && godep restore 8 | RUN apt-get update && apt-get install -y libgmp-dev 9 | -------------------------------------------------------------------------------- /contrib/README.md: -------------------------------------------------------------------------------- 1 | This is a small docker container to build ECP linux binaries from OSX. 2 | -------------------------------------------------------------------------------- /contrib/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build --no-cache -t blockparser-linux . 3 | cd ../ && docker run --rm -v "$PWD":/usr/src/ecp -w /usr/src/ecp blockparser-linux go build -v 4 | -------------------------------------------------------------------------------- /importer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/codegangsta/cli" 5 | "github.com/ethereum/go-ethereum/cmd/utils" 6 | "github.com/ethereum/go-ethereum/core/types" 7 | "github.com/ethereum/go-ethereum/eth" 8 | "github.com/ethereum/go-ethereum/logger" 9 | "github.com/ethereum/go-ethereum/logger/glog" 10 | "gopkg.in/mgo.v2" 11 | "gopkg.in/mgo.v2/bson" 12 | ) 13 | 14 | type ImportMaster struct { 15 | session *mgo.Session 16 | ethereum *eth.Ethereum 17 | txCollection *mgo.Collection 18 | blockCollection *mgo.Collection 19 | } 20 | 21 | type Transaction struct { 22 | TxHash string `bson:"tx_hash"` 23 | Recipient string 24 | From string 25 | Amount string 26 | Price string 27 | GasLimit string `bson:"gas_limit"` 28 | Payload []byte 29 | BlockId *bson.ObjectId `bson:"block_id,omitempty"` 30 | } 31 | 32 | func (self *ImportMaster) parseTx(tx *types.Transaction, blockId *bson.ObjectId) *Transaction { 33 | hash := tx.Hash().Hex() 34 | from, err := tx.From() 35 | if err != nil { 36 | utils.Fatalf("Could not parse from address: %v", err) 37 | } 38 | var recipient string 39 | if tx.Recipient != nil { 40 | recipient = tx.Recipient.Hex() 41 | } 42 | txx := &Transaction{hash, recipient, from.Hex(), tx.Amount.String(), tx.Price.String(), tx.GasLimit.String(), tx.Payload, blockId} 43 | return txx 44 | } 45 | 46 | type Block struct { 47 | BlockHash string `bson:"block_hash"` 48 | ParentHash string `bson:"parent_hash"` 49 | UncleHash string `bson:"uncle_hash"` 50 | Coinbase string `bson:"coin_base"` 51 | Root string `bson:"root"` 52 | TxHash string `bson:"tx_hash"` 53 | ReceiptHash string `bson:"receipt_hash"` 54 | Number string 55 | Difficulty string 56 | GasLimit string `bson:"gas_limit"` 57 | GasUsed string `bson:"gas_used"` 58 | Time uint64 59 | TxAmount uint64 `bson:"tx_amount"` 60 | Extra string `bson:"extra"` 61 | Nonce string 62 | StorageSize string `bson:"storage_size"` 63 | MixDigest string `bson:"mix_digest"` 64 | Processed bool `bson:"processed"` 65 | Id *bson.ObjectId `bson:"_id,omitempty"` 66 | } 67 | 68 | func NewImporter(ctx *cli.Context) *ImportMaster { 69 | importer := new(ImportMaster) 70 | mongoUrl := ctx.String("mongo-url") 71 | db := ctx.String("mongo-database") 72 | 73 | glog.V(logger.Info).Infoln("Connecting to MongoDB db '", db, "'using", mongoUrl) 74 | 75 | session, err := mgo.Dial(mongoUrl) 76 | if err != nil { 77 | panic(err) 78 | } 79 | importer.session = session 80 | importer.txCollection = session.DB(db).C("transactions") 81 | importer.blockCollection = session.DB(db).C("blocks") 82 | 83 | return importer 84 | } 85 | 86 | func (self *ImportMaster) importBlock(block *types.Block) { 87 | blockHash := block.Header().Hash().Hex() 88 | txAmount := uint64(len(block.Transactions())) 89 | 90 | glog.V(logger.Info).Infoln("Importing block", blockHash, "Hash with ", txAmount, "transactions") 91 | extData := string(block.Header().Extra[:]) 92 | 93 | err := self.blockCollection.Insert(&Block{blockHash, block.ParentHash().Hex(), block.Header().UncleHash.Hex(), block.Header().Coinbase.Hex(), block.Header().Root.Hex(), block.Header().TxHash.Hex(), block.Header().ReceiptHash.Hex(), block.Header().Number.String(), block.Header().Difficulty.String(), block.Header().GasLimit.String(), block.Header().GasUsed.String(), block.Header().Time, txAmount, extData, string(block.Nonce()), block.Size().String(), block.Header().MixDigest.Hex(), false, nil}) 94 | if err != nil { 95 | clilogger.Infoln(err) 96 | } 97 | result := Block{} 98 | err = self.blockCollection.Find(bson.M{"block_hash": blockHash}).One(&result) 99 | if err != nil { 100 | utils.Fatalf("Could not find the block we just added, saving faild: %v", err) 101 | } 102 | for _, tx := range block.Transactions() { 103 | self.importTx(tx, result.Id) 104 | } 105 | } 106 | func (self *ImportMaster) importTx(tx *types.Transaction, blockId *bson.ObjectId) { 107 | if glog.V(logger.Info) { 108 | glog.Infoln("Importing tx", tx.Hash().Hex()) 109 | } 110 | err := self.txCollection.Insert(self.parseTx(tx, blockId)) 111 | if err != nil { 112 | clilogger.Infoln(err) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/codegangsta/cli" 6 | "os" 7 | "runtime" 8 | 9 | "github.com/ethereum/go-ethereum/cmd/utils" 10 | "github.com/ethereum/go-ethereum/core" 11 | 12 | "github.com/ethereum/go-ethereum/eth" 13 | "github.com/ethereum/go-ethereum/logger" 14 | ) 15 | 16 | const ( 17 | Version = "0.0.2" 18 | ) 19 | 20 | var ( 21 | clilogger = logger.NewLogger("Parser") 22 | app = utils.NewApp(Version, "Ethereum chain parser") 23 | ) 24 | 25 | func init() { 26 | app.Action = run 27 | app.Name = "BlockParser" 28 | app.Flags = []cli.Flag{ 29 | cli.StringFlag{ 30 | Name: "mongo-url", 31 | Value: "mongodb://localhost", 32 | Usage: "MongoDB connection url", 33 | }, 34 | cli.StringFlag{ 35 | Name: "mongo-database", 36 | Value: "chain_explorer", 37 | Usage: "MongoDB database", 38 | }, 39 | utils.UnlockedAccountFlag, 40 | utils.PasswordFileFlag, 41 | utils.BootnodesFlag, 42 | utils.DataDirFlag, 43 | utils.JSpathFlag, 44 | utils.ListenPortFlag, 45 | utils.LogFileFlag, 46 | utils.LogJSONFlag, 47 | utils.VerbosityFlag, 48 | utils.MaxPeersFlag, 49 | utils.EtherbaseFlag, 50 | utils.BlockchainVersionFlag, 51 | utils.MinerThreadsFlag, 52 | utils.MiningEnabledFlag, 53 | utils.NATFlag, 54 | utils.NodeKeyFileFlag, 55 | utils.NodeKeyHexFlag, 56 | utils.RPCEnabledFlag, 57 | utils.RPCListenAddrFlag, 58 | utils.RPCPortFlag, 59 | utils.VMDebugFlag, 60 | utils.ProtocolVersionFlag, 61 | utils.NetworkIdFlag, 62 | utils.RPCCORSDomainFlag, 63 | utils.BacktraceAtFlag, 64 | utils.LogToStdErrFlag, 65 | } 66 | } 67 | 68 | func main() { 69 | runtime.GOMAXPROCS(runtime.NumCPU()) 70 | defer logger.Flush() 71 | if err := app.Run(os.Args); err != nil { 72 | fmt.Fprintln(os.Stderr, err) 73 | os.Exit(1) 74 | } 75 | } 76 | func run(ctx *cli.Context) { 77 | importer := NewImporter(ctx) 78 | utils.HandleInterrupt() 79 | 80 | cfg := utils.MakeEthConfig("EthChainParser", Version, ctx) 81 | 82 | ethereum, err := eth.New(cfg) 83 | if err != nil { 84 | utils.Fatalf("%v", err) 85 | } 86 | utils.StartEthereum(ethereum) 87 | 88 | if ctx.GlobalBool(utils.RPCEnabledFlag.Name) { 89 | utils.StartRPC(ethereum, ctx) 90 | } 91 | 92 | events := ethereum.EventMux().Subscribe( 93 | core.ChainEvent{}, 94 | core.TxPreEvent{}, 95 | ) 96 | 97 | defer events.Unsubscribe() 98 | for { 99 | select { 100 | case ev, isopen := <-events.Chan(): 101 | if !isopen { 102 | return 103 | } 104 | switch ev := ev.(type) { 105 | case core.ChainEvent: 106 | importer.importBlock(ev.Block) 107 | case core.TxPreEvent: 108 | // Not dealing with incoming txes for now 109 | //importer.importTx(ev.Tx) 110 | } 111 | } 112 | } 113 | 114 | ethereum.WaitForShutdown() 115 | logger.Flush() 116 | fmt.Printf("Shutting down\n") 117 | } 118 | --------------------------------------------------------------------------------