├── .gitignore ├── Dockerfile ├── utils ├── number.go ├── hex.go └── logger.go ├── rpc ├── interface.go ├── eth_rpc_with_retry.go └── eth_rpc.go ├── go.mod ├── plugin ├── tx_plugin.go ├── block_plugin.go ├── receipt_log_plugin.go └── tx_receipt_plugin.go ├── watcher_with_tx_plugin_test.go ├── watcher_with_block_plugin_test.go ├── watcher_with_receipt_logs_plugin_test.go ├── blockchain ├── crypto.go └── blockchain.go ├── structs └── structs.go ├── .github └── workflows │ └── go.yml ├── receipt_log_handler.go ├── receipt_log_handler_test.go ├── watcher_with_tx_receipt_test.go ├── cli └── main.go ├── receipt_log_watcher.go ├── go.sum ├── LICENSE ├── readme.md └── watcher.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | .cover 3 | .idea 4 | debug.test 5 | debug 6 | *.db 7 | main 8 | vendor 9 | .env 10 | .vscode 11 | 12 | .DS_Store 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13 2 | 3 | WORKDIR /go/src 4 | COPY . /go/src 5 | 6 | RUN go build -o bin/ethereum-watcher cli/main.go 7 | 8 | FROM alpine 9 | RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 10 | RUN apk --no-cache add ca-certificates 11 | 12 | COPY --from=0 /go/src/bin/* /bin/ 13 | 14 | CMD ["/bin/ethereum-watcher"] 15 | -------------------------------------------------------------------------------- /utils/number.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/shopspring/decimal" 5 | "math/big" 6 | ) 7 | 8 | // To Decimal 9 | func StringToDecimal(str string) decimal.Decimal { 10 | if len(str) >= 2 && str[:2] == "0x" { 11 | b := new(big.Int) 12 | b.SetString(str[2:], 16) 13 | d := decimal.NewFromBigInt(b, 0) 14 | return d 15 | } else { 16 | v, err := decimal.NewFromString(str) 17 | if err != nil { 18 | panic(err) 19 | } 20 | return v 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /rpc/interface.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 5 | ) 6 | 7 | type IBlockChainRPC interface { 8 | GetCurrentBlockNum() (uint64, error) 9 | 10 | GetBlockByNum(uint64) (blockchain.Block, error) 11 | GetLiteBlockByNum(uint64) (blockchain.Block, error) 12 | GetTransactionReceipt(txHash string) (blockchain.TransactionReceipt, error) 13 | 14 | GetLogs(from, to uint64, address string, topics []string) ([]blockchain.IReceiptLog, error) 15 | } 16 | -------------------------------------------------------------------------------- /utils/hex.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/hex" 5 | "strings" 6 | ) 7 | 8 | func Bytes2Hex(bytes []byte) string { 9 | return hex.EncodeToString(bytes) 10 | } 11 | 12 | func Hex2Bytes(str string) []byte { 13 | if strings.HasPrefix(str, "0x") || strings.HasPrefix(str, "0X") { 14 | str = str[2:] 15 | } 16 | 17 | if len(str)%2 == 1 { 18 | str = "0" + str 19 | } 20 | 21 | h, _ := hex.DecodeString(str) 22 | return h 23 | } 24 | 25 | // with prefix '0x' 26 | func Bytes2HexP(bytes []byte) string { 27 | return "0x" + hex.EncodeToString(bytes) 28 | } 29 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/HydroProtocol/ethereum-watcher 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d 7 | github.com/jarcoal/httpmock v1.0.4 // indirect 8 | github.com/labstack/gommon v0.2.8 9 | github.com/mattn/go-colorable v0.1.4 // indirect 10 | github.com/mattn/go-isatty v0.0.11 // indirect 11 | github.com/onrik/ethrpc v0.0.0-20190305112807-6b8e9c0e9a8f 12 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 13 | github.com/sirupsen/logrus v1.4.1 14 | github.com/spf13/cobra v0.0.5 15 | github.com/spf13/pflag v1.0.5 // indirect 16 | github.com/tidwall/gjson v1.3.5 // indirect 17 | github.com/valyala/fasttemplate v1.1.0 // indirect 18 | golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a 19 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /utils/logger.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "os" 6 | ) 7 | 8 | func init() { 9 | switch os.Getenv("HSK_LOG_LEVEL") { 10 | case "FATAL": 11 | log.SetLevel(log.FatalLevel) 12 | case "ERROR": 13 | log.SetLevel(log.ErrorLevel) 14 | case "WARN": 15 | log.SetLevel(log.WarnLevel) 16 | case "INFO": 17 | log.SetLevel(log.InfoLevel) 18 | case "DEBUG": 19 | log.SetLevel(log.DebugLevel) 20 | default: 21 | log.SetLevel(log.InfoLevel) 22 | } 23 | 24 | formatter := &log.TextFormatter{ 25 | FullTimestamp: true, 26 | } 27 | 28 | log.SetFormatter(formatter) 29 | } 30 | 31 | func Debugf(format string, v ...interface{}) { 32 | log.Debugf(format, v...) 33 | } 34 | 35 | func Infof(format string, v ...interface{}) { 36 | log.Infof(format, v...) 37 | } 38 | 39 | func Errorf(format string, v ...interface{}) { 40 | log.Errorf(format, v...) 41 | } 42 | -------------------------------------------------------------------------------- /plugin/tx_plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/structs" 5 | ) 6 | 7 | type ITxPlugin interface { 8 | AcceptTx(transaction structs.RemovableTx) 9 | } 10 | 11 | type TxHashPlugin struct { 12 | callback func(txHash string, isRemoved bool) 13 | } 14 | 15 | func (p TxHashPlugin) AcceptTx(transaction structs.RemovableTx) { 16 | if p.callback != nil { 17 | p.callback(transaction.GetHash(), transaction.IsRemoved) 18 | } 19 | } 20 | 21 | func NewTxHashPlugin(callback func(txHash string, isRemoved bool)) TxHashPlugin { 22 | return TxHashPlugin{ 23 | callback: callback, 24 | } 25 | } 26 | 27 | type TxPlugin struct { 28 | callback func(tx structs.RemovableTx) 29 | } 30 | 31 | func (p TxPlugin) AcceptTx(transaction structs.RemovableTx) { 32 | if p.callback != nil { 33 | p.callback(transaction) 34 | } 35 | } 36 | 37 | func NewTxPlugin(callback func(tx structs.RemovableTx)) TxPlugin { 38 | return TxPlugin{ 39 | callback: callback, 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /watcher_with_tx_plugin_test.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher/plugin" 7 | "github.com/HydroProtocol/ethereum-watcher/structs" 8 | "github.com/sirupsen/logrus" 9 | "testing" 10 | ) 11 | 12 | func TestTxHashPlugin(t *testing.T) { 13 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 14 | w := NewHttpBasedEthWatcher(context.Background(), api) 15 | 16 | w.RegisterTxPlugin(plugin.NewTxHashPlugin(func(txHash string, isRemoved bool) { 17 | fmt.Println(">>", txHash, isRemoved) 18 | })) 19 | 20 | w.RunTillExit() 21 | } 22 | 23 | func TestTxPlugin(t *testing.T) { 24 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 25 | w := NewHttpBasedEthWatcher(context.Background(), api) 26 | 27 | w.RegisterTxPlugin(plugin.NewTxPlugin(func(tx structs.RemovableTx) { 28 | logrus.Printf(">> block: %d, txHash: %s", tx.GetBlockNumber(), tx.GetHash()) 29 | })) 30 | 31 | w.RunTillExit() 32 | } 33 | -------------------------------------------------------------------------------- /watcher_with_block_plugin_test.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "github.com/HydroProtocol/ethereum-watcher/plugin" 6 | "github.com/HydroProtocol/ethereum-watcher/structs" 7 | "github.com/sirupsen/logrus" 8 | "testing" 9 | ) 10 | 11 | func TestNewBlockNumPlugin(t *testing.T) { 12 | logrus.SetLevel(logrus.InfoLevel) 13 | 14 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 15 | w := NewHttpBasedEthWatcher(context.Background(), api) 16 | 17 | logrus.Println("waiting for new block...") 18 | w.RegisterBlockPlugin(plugin.NewBlockNumPlugin(func(i uint64, b bool) { 19 | logrus.Printf(">> found new block: %d, is removed: %t", i, b) 20 | })) 21 | 22 | w.RunTillExit() 23 | } 24 | 25 | func TestSimpleBlockPlugin(t *testing.T) { 26 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 27 | w := NewHttpBasedEthWatcher(context.Background(), api) 28 | 29 | w.RegisterBlockPlugin(plugin.NewSimpleBlockPlugin(func(block *structs.RemovableBlock) { 30 | logrus.Infof(">> %+v", block.Block) 31 | })) 32 | 33 | w.RunTillExit() 34 | } 35 | -------------------------------------------------------------------------------- /plugin/block_plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/structs" 5 | ) 6 | 7 | type IBlockPlugin interface { 8 | AcceptBlock(block *structs.RemovableBlock) 9 | } 10 | 11 | type BlockNumPlugin struct { 12 | callback func(blockNum uint64, isRemoved bool) 13 | } 14 | 15 | func (p BlockNumPlugin) AcceptBlock(b *structs.RemovableBlock) { 16 | if p.callback != nil { 17 | p.callback(b.Number(), b.IsRemoved) 18 | } 19 | } 20 | 21 | func NewBlockNumPlugin(callback func(uint64, bool)) BlockNumPlugin { 22 | return BlockNumPlugin{ 23 | callback: callback, 24 | } 25 | } 26 | 27 | //provide block info with tx basic infos 28 | type SimpleBlockPlugin struct { 29 | callback func(block *structs.RemovableBlock) 30 | } 31 | 32 | func (p SimpleBlockPlugin) AcceptBlock(b *structs.RemovableBlock) { 33 | if p.callback != nil { 34 | p.callback(b) 35 | } 36 | } 37 | 38 | func NewSimpleBlockPlugin(callback func(block *structs.RemovableBlock)) SimpleBlockPlugin { 39 | return SimpleBlockPlugin{ 40 | callback: callback, 41 | } 42 | } 43 | 44 | // block info with tx full infos 45 | //type BlockWithTxInfoPlugin struct { 46 | //} 47 | -------------------------------------------------------------------------------- /watcher_with_receipt_logs_plugin_test.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher/plugin" 7 | "github.com/HydroProtocol/ethereum-watcher/structs" 8 | "github.com/sirupsen/logrus" 9 | "testing" 10 | ) 11 | 12 | func TestReceiptLogsPlugin(t *testing.T) { 13 | logrus.SetLevel(logrus.DebugLevel) 14 | 15 | api := "https://kovan.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 16 | w := NewHttpBasedEthWatcher(context.Background(), api) 17 | 18 | contract := "0x63bB8a255a8c045122EFf28B3093Cc225B711F6D" 19 | // Match 20 | topics := []string{"0x6bf96fcc2cec9e08b082506ebbc10114578a497ff1ea436628ba8996b750677c"} 21 | 22 | w.RegisterReceiptLogPlugin(plugin.NewReceiptLogPlugin(contract, topics, func(receipt *structs.RemovableReceiptLog) { 23 | if receipt.IsRemoved { 24 | logrus.Infof("Removed >> %+v", receipt) 25 | } else { 26 | logrus.Infof("Adding >> %+v, tx: %s, logIdx: %d", receipt, receipt.IReceiptLog.GetTransactionHash(), receipt.IReceiptLog.GetLogIndex()) 27 | } 28 | })) 29 | 30 | //startBlock := 12304546 31 | startBlock := 12101723 32 | err := w.RunTillExitFromBlock(uint64(startBlock)) 33 | 34 | fmt.Println("err:", err) 35 | } 36 | -------------------------------------------------------------------------------- /blockchain/crypto.go: -------------------------------------------------------------------------------- 1 | package blockchain 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "crypto/elliptic" 6 | "fmt" 7 | "github.com/HydroProtocol/ethereum-watcher/utils" 8 | "github.com/btcsuite/btcd/btcec" 9 | "golang.org/x/crypto/sha3" 10 | ) 11 | 12 | var bitCurve = btcec.S256() 13 | 14 | func Keccak256(data ...[]byte) []byte { 15 | d := sha3.NewLegacyKeccak256() 16 | for _, b := range data { 17 | d.Write(b) 18 | } 19 | return d.Sum(nil) 20 | } 21 | 22 | func PersonalEcRecover(data []byte, sig []byte) (string, error) { 23 | if len(sig) != 65 { 24 | return "", fmt.Errorf("signature must be 65 bytes long") 25 | } 26 | if sig[64] >= 27 { 27 | sig[64] -= 27 28 | } 29 | 30 | rpk, err := SigToPub(hashPersonalMessage(data), sig) 31 | 32 | if err != nil { 33 | return "", err 34 | } 35 | 36 | if rpk == nil || rpk.X == nil || rpk.Y == nil { 37 | return "", fmt.Errorf("") 38 | } 39 | pubBytes := elliptic.Marshal(bitCurve, rpk.X, rpk.Y) 40 | return utils.Bytes2Hex(Keccak256(pubBytes[1:])[12:]), nil 41 | } 42 | 43 | func hashPersonalMessage(data []byte) []byte { 44 | msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) 45 | return Keccak256([]byte(msg)) 46 | } 47 | 48 | func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { 49 | // Convert to btcec input format with 'recovery id' v at the beginning. 50 | btcSig := make([]byte, 65) 51 | btcSig[0] = sig[64] + 27 52 | copy(btcSig[1:], sig) 53 | 54 | pub, _, err := btcec.RecoverCompact(btcec.S256(), btcSig, hash) 55 | return (*ecdsa.PublicKey)(pub), err 56 | } 57 | -------------------------------------------------------------------------------- /structs/structs.go: -------------------------------------------------------------------------------- 1 | package structs 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 5 | ) 6 | 7 | type RemovableBlock struct { 8 | blockchain.Block 9 | IsRemoved bool 10 | } 11 | 12 | func NewRemovableBlock(block blockchain.Block, isRemoved bool) *RemovableBlock { 13 | return &RemovableBlock{ 14 | block, 15 | isRemoved, 16 | } 17 | } 18 | 19 | type TxAndReceipt struct { 20 | Tx blockchain.Transaction 21 | Receipt blockchain.TransactionReceipt 22 | } 23 | 24 | type RemovableTxAndReceipt struct { 25 | *TxAndReceipt 26 | IsRemoved bool 27 | TimeStamp uint64 28 | } 29 | 30 | type RemovableReceiptLog struct { 31 | blockchain.IReceiptLog 32 | IsRemoved bool 33 | } 34 | 35 | func NewRemovableTxAndReceipt(tx blockchain.Transaction, receipt blockchain.TransactionReceipt, removed bool, timeStamp uint64) *RemovableTxAndReceipt { 36 | return &RemovableTxAndReceipt{ 37 | &TxAndReceipt{ 38 | tx, 39 | receipt, 40 | }, 41 | removed, 42 | timeStamp, 43 | } 44 | } 45 | 46 | type RemovableTx struct { 47 | blockchain.Transaction 48 | IsRemoved bool 49 | } 50 | 51 | func NewRemovableTx(tx blockchain.Transaction, removed bool) RemovableTx { 52 | return RemovableTx{ 53 | tx, 54 | removed, 55 | } 56 | } 57 | 58 | // 59 | //type RemovableReceipt struct { 60 | // sdk.TransactionReceipt 61 | // IsRemoved bool 62 | //} 63 | // 64 | //func NewRemovableReceipt(receipt sdk.TransactionReceipt, removed bool) RemovableReceipt { 65 | // return RemovableReceipt{ 66 | // receipt, 67 | // removed, 68 | // } 69 | //} 70 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | env: 4 | # DOCKER_REG: docker.pkg.github.com 5 | DOCKER_REG: docker.io 6 | jobs: 7 | 8 | build: 9 | name: Build 10 | runs-on: ubuntu-latest 11 | steps: 12 | 13 | - name: Set up Go 1.13 14 | uses: actions/setup-go@v1 15 | with: 16 | go-version: 1.13 17 | id: go 18 | 19 | - name: Check out code into the Go module directory 20 | uses: actions/checkout@v1 21 | 22 | # - name: Build Docker Image and Push @ Github-Registry 23 | # run: | 24 | # REPO=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'` 25 | # REF=`echo ${{github.ref}} | awk -F/ '{print $NF}'` 26 | 27 | # CONTAINER_IMAGE=$DOCKER_REG/$REPO/img:${{ github.sha }} 28 | # CONTAINER_IMAGE_ALIAS=$DOCKER_REG/$REPO/img:$REF 29 | 30 | # echo '${{ secrets.GITHUB_TOKEN }}' | docker login --username=${{ github.actor }} $DOCKER_REG --password-stdin 31 | 32 | # docker build . -t $CONTAINER_IMAGE 33 | # docker push $CONTAINER_IMAGE 34 | # docker tag $CONTAINER_IMAGE $CONTAINER_IMAGE_ALIAS 35 | # docker push $CONTAINER_IMAGE_ALIAS 36 | 37 | - name: Build Docker Image and Push @ Dockerhub 38 | run: | 39 | REPO="hydroprotocolio/ethereum-watcher" 40 | REF=`echo ${{github.ref}} | awk -F/ '{print $NF}'` 41 | 42 | CONTAINER_IMAGE=$DOCKER_REG/$REPO:${{ github.sha }} 43 | CONTAINER_IMAGE_ALIAS=$DOCKER_REG/$REPO:$REF 44 | 45 | echo '${{ secrets.DOCKERHUB_TOKEN }}' | docker login --username=${{ secrets.DOCKERHUB_USERNAME }} $DOCKER_REG --password-stdin 46 | 47 | docker build . -t $CONTAINER_IMAGE 48 | docker push $CONTAINER_IMAGE 49 | docker tag $CONTAINER_IMAGE $CONTAINER_IMAGE_ALIAS 50 | docker push $CONTAINER_IMAGE_ALIAS 51 | -------------------------------------------------------------------------------- /plugin/receipt_log_plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/structs" 5 | "strings" 6 | ) 7 | 8 | type IReceiptLogPlugin interface { 9 | FromContract() string 10 | InterestedTopics() []string 11 | NeedReceiptLog(receiptLog *structs.RemovableReceiptLog) bool 12 | Accept(receiptLog *structs.RemovableReceiptLog) 13 | } 14 | 15 | type ReceiptLogPlugin struct { 16 | contract string 17 | topics []string 18 | callback func(receiptLog *structs.RemovableReceiptLog) 19 | } 20 | 21 | func NewReceiptLogPlugin( 22 | contract string, 23 | topics []string, 24 | callback func(receiptLog *structs.RemovableReceiptLog), 25 | ) *ReceiptLogPlugin { 26 | return &ReceiptLogPlugin{ 27 | contract: contract, 28 | topics: topics, 29 | callback: callback, 30 | } 31 | } 32 | 33 | func (p *ReceiptLogPlugin) FromContract() string { 34 | return p.contract 35 | } 36 | 37 | func (p *ReceiptLogPlugin) InterestedTopics() []string { 38 | return p.topics 39 | } 40 | 41 | func (p *ReceiptLogPlugin) Accept(receiptLog *structs.RemovableReceiptLog) { 42 | if p.callback != nil { 43 | p.callback(receiptLog) 44 | } 45 | } 46 | 47 | // simplified version of specifying topic filters 48 | // https://github.com/ethereum/wiki/wiki/JSON-RPC#a-note-on-specifying-topic-filters 49 | func (p *ReceiptLogPlugin) NeedReceiptLog(receiptLog *structs.RemovableReceiptLog) bool { 50 | contract := receiptLog.GetAddress() 51 | if strings.ToLower(p.contract) != strings.ToLower(contract) { 52 | return false 53 | } 54 | 55 | var firstTopic string 56 | if len(receiptLog.GetTopics()) > 0 { 57 | firstTopic = receiptLog.GetTopics()[0] 58 | } 59 | 60 | for _, interestedTopic := range p.topics { 61 | if strings.ToLower(firstTopic) == strings.ToLower(interestedTopic) { 62 | return true 63 | } 64 | } 65 | 66 | return false 67 | } 68 | -------------------------------------------------------------------------------- /receipt_log_handler.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "github.com/HydroProtocol/ethereum-watcher/rpc" 6 | "github.com/HydroProtocol/ethereum-watcher/structs" 7 | "github.com/sirupsen/logrus" 8 | "time" 9 | ) 10 | 11 | const DefaultStepSizeForBigLag = 10 12 | 13 | //deprecated, please use receipt_log_watcher instead. 14 | func ListenForReceiptLogTillExit( 15 | ctx context.Context, 16 | api string, 17 | startBlock int, 18 | contract string, 19 | interestedTopics []string, 20 | handler func(receiptLog structs.RemovableReceiptLog), 21 | steps ...int, 22 | ) int { 23 | var stepSizeForBigLag int 24 | if len(steps) > 0 && steps[0] > 0 { 25 | stepSizeForBigLag = steps[0] 26 | } else { 27 | stepSizeForBigLag = DefaultStepSizeForBigLag 28 | } 29 | 30 | rpc := rpc.NewEthRPCWithRetry(api, 5) 31 | 32 | var blockNumToBeProcessedNext = startBlock 33 | 34 | for { 35 | select { 36 | case <-ctx.Done(): 37 | return blockNumToBeProcessedNext - 1 38 | default: 39 | highestBlock, err := rpc.GetCurrentBlockNum() 40 | if err != nil { 41 | return blockNumToBeProcessedNext - 1 42 | } 43 | 44 | if blockNumToBeProcessedNext < 0 { 45 | blockNumToBeProcessedNext = int(highestBlock) 46 | } 47 | 48 | numOfBlocksToProcess := int(highestBlock) - blockNumToBeProcessedNext + 1 49 | if numOfBlocksToProcess <= 0 { 50 | logrus.Debugf("no ready block after %d, sleep 3 seconds", highestBlock) 51 | time.Sleep(3 * time.Second) 52 | continue 53 | } 54 | 55 | var to int 56 | if numOfBlocksToProcess > stepSizeForBigLag { 57 | // quick mode 58 | to = blockNumToBeProcessedNext + stepSizeForBigLag - 1 59 | } else { 60 | // normal mode, 1block each time 61 | to = blockNumToBeProcessedNext 62 | } 63 | 64 | logs, err := rpc.GetLogs(uint64(blockNumToBeProcessedNext), uint64(to), contract, interestedTopics) 65 | if err != nil { 66 | return blockNumToBeProcessedNext - 1 67 | } 68 | 69 | for i := 0; i < len(logs); i++ { 70 | handler(structs.RemovableReceiptLog{ 71 | IReceiptLog: logs[i], 72 | }) 73 | } 74 | 75 | blockNumToBeProcessedNext = to + 1 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /receipt_log_handler_test.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "github.com/HydroProtocol/ethereum-watcher/rpc" 6 | "github.com/HydroProtocol/ethereum-watcher/structs" 7 | "github.com/sirupsen/logrus" 8 | "testing" 9 | ) 10 | 11 | func TestListenForReceiptLogTillExit(t *testing.T) { 12 | logrus.SetLevel(logrus.DebugLevel) 13 | 14 | ctx := context.Background() 15 | api := "https://kovan.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 16 | startBlock := 12220000 17 | contract := "0xAc34923B2b8De9d441570376e6c811D5aA5ed72f" 18 | interestedTopics := []string{ 19 | "0x23b872dd7302113369cda2901243429419bec145408fa8b352b3dd92b66c680b", 20 | "0x6bf96fcc2cec9e08b082506ebbc10114578a497ff1ea436628ba8996b750677c", 21 | "0x5a746ce5ce37fc996a6e682f4f84b6f90d3be79fd8ac9a8a11264345f3d29edd", 22 | "0x9c4e90320be51bb93d854d0ab9ba8aa249dabc21192529efcd76ae7c22c6bc0b", 23 | "0x0ce31a5f70780bb6770b52a6793403d856441ccb475715e8382a0525d35b0558", 24 | } 25 | 26 | handler := func(log structs.RemovableReceiptLog) { 27 | logrus.Infof("log from tx: %s", log.GetTransactionHash()) 28 | } 29 | 30 | stepsForBigLag := 100 31 | 32 | highestProcessed := ListenForReceiptLogTillExit(ctx, api, startBlock, contract, interestedTopics, handler, stepsForBigLag) 33 | logrus.Infof("highestProcessed: %d", highestProcessed) 34 | } 35 | 36 | func TestListenForReceiptLogTillExit2(t *testing.T) { 37 | logrus.SetLevel(logrus.DebugLevel) 38 | 39 | stepsForBigLag := 100 40 | 41 | ctx := context.Background() 42 | api := "https://kovan.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 43 | 44 | startBlock, _ := rpc.NewEthRPC(api).GetCurrentBlockNum() 45 | startBlock = startBlock - uint64(stepsForBigLag*10) 46 | 47 | contract := "0xAc34923B2b8De9d441570376e6c811D5aA5ed72f" 48 | interestedTopics := []string{ 49 | "0x23b872dd7302113369cda2901243429419bec145408fa8b352b3dd92b66c680b", 50 | } 51 | 52 | handler := func(log structs.RemovableReceiptLog) { 53 | logrus.Infof("log from tx: %s", log.GetTransactionHash()) 54 | } 55 | 56 | highestProcessed := ListenForReceiptLogTillExit(ctx, api, int(startBlock), contract, interestedTopics, handler, stepsForBigLag) 57 | logrus.Infof("highestProcessed: %d", highestProcessed) 58 | } 59 | -------------------------------------------------------------------------------- /rpc/eth_rpc_with_retry.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 5 | "time" 6 | ) 7 | 8 | type EthBlockChainRPCWithRetry struct { 9 | *EthBlockChainRPC 10 | maxRetryTimes int 11 | } 12 | 13 | func NewEthRPCWithRetry(api string, maxRetryCount int) *EthBlockChainRPCWithRetry { 14 | rpc := NewEthRPC(api) 15 | 16 | return &EthBlockChainRPCWithRetry{rpc, maxRetryCount} 17 | } 18 | 19 | func (rpc EthBlockChainRPCWithRetry) GetBlockByNum(num uint64) (rst blockchain.Block, err error) { 20 | for i := 0; i <= rpc.maxRetryTimes; i++ { 21 | rst, err = rpc.EthBlockChainRPC.GetBlockByNum(num) 22 | if err == nil { 23 | break 24 | } else { 25 | time.Sleep(time.Duration(500*(i+1)) * time.Millisecond) 26 | } 27 | } 28 | 29 | return 30 | } 31 | 32 | func (rpc EthBlockChainRPCWithRetry) GetLiteBlockByNum(num uint64) (rst blockchain.Block, err error) { 33 | for i := 0; i <= rpc.maxRetryTimes; i++ { 34 | rst, err = rpc.EthBlockChainRPC.GetLiteBlockByNum(num) 35 | if err == nil { 36 | break 37 | } else { 38 | time.Sleep(time.Duration(500*(i+1)) * time.Millisecond) 39 | } 40 | } 41 | 42 | return 43 | } 44 | 45 | func (rpc EthBlockChainRPCWithRetry) GetTransactionReceipt(txHash string) (rst blockchain.TransactionReceipt, err error) { 46 | for i := 0; i <= rpc.maxRetryTimes; i++ { 47 | rst, err = rpc.EthBlockChainRPC.GetTransactionReceipt(txHash) 48 | if err == nil { 49 | break 50 | } else { 51 | time.Sleep(time.Duration(500*(i+1)) * time.Millisecond) 52 | } 53 | } 54 | 55 | return 56 | } 57 | 58 | func (rpc EthBlockChainRPCWithRetry) GetCurrentBlockNum() (rst uint64, err error) { 59 | for i := 0; i <= rpc.maxRetryTimes; i++ { 60 | rst, err = rpc.EthBlockChainRPC.GetCurrentBlockNum() 61 | if err == nil { 62 | break 63 | } else { 64 | time.Sleep(time.Duration(500*(i+1)) * time.Millisecond) 65 | } 66 | } 67 | 68 | return 69 | } 70 | func (rpc EthBlockChainRPCWithRetry) GetLogs( 71 | fromBlockNum, toBlockNum uint64, 72 | address string, 73 | topics []string, 74 | ) (rst []blockchain.IReceiptLog, err error) { 75 | for i := 0; i <= rpc.maxRetryTimes; i++ { 76 | rst, err = rpc.EthBlockChainRPC.GetLogs(fromBlockNum, toBlockNum, address, topics) 77 | if err == nil { 78 | break 79 | } else { 80 | time.Sleep(time.Duration(500*(i+1)) * time.Millisecond) 81 | } 82 | } 83 | 84 | return 85 | } 86 | -------------------------------------------------------------------------------- /rpc/eth_rpc.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "errors" 5 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 6 | "github.com/onrik/ethrpc" 7 | "github.com/sirupsen/logrus" 8 | "strconv" 9 | ) 10 | 11 | type EthBlockChainRPC struct { 12 | rpcImpl *ethrpc.EthRPC 13 | } 14 | 15 | func NewEthRPC(api string) *EthBlockChainRPC { 16 | rpc := ethrpc.New(api) 17 | 18 | return &EthBlockChainRPC{rpc} 19 | } 20 | 21 | func (rpc EthBlockChainRPC) GetBlockByNum(num uint64) (blockchain.Block, error) { 22 | return rpc.getBlockByNum(num, true) 23 | } 24 | 25 | func (rpc EthBlockChainRPC) GetLiteBlockByNum(num uint64) (blockchain.Block, error) { 26 | return rpc.getBlockByNum(num, false) 27 | } 28 | 29 | func (rpc EthBlockChainRPC) getBlockByNum(num uint64, withTx bool) (blockchain.Block, error) { 30 | b, err := rpc.rpcImpl.EthGetBlockByNumber(int(num), withTx) 31 | if err != nil { 32 | return nil, err 33 | } 34 | if b == nil { 35 | return nil, errors.New("nil block") 36 | } 37 | 38 | return &blockchain.EthereumBlock{b}, err 39 | } 40 | 41 | func (rpc EthBlockChainRPC) GetTransactionReceipt(txHash string) (blockchain.TransactionReceipt, error) { 42 | receipt, err := rpc.rpcImpl.EthGetTransactionReceipt(txHash) 43 | if err != nil { 44 | return nil, err 45 | } 46 | if receipt == nil { 47 | return nil, errors.New("nil receipt") 48 | } 49 | 50 | return &blockchain.EthereumTransactionReceipt{receipt}, err 51 | } 52 | 53 | func (rpc EthBlockChainRPC) GetCurrentBlockNum() (uint64, error) { 54 | num, err := rpc.rpcImpl.EthBlockNumber() 55 | return uint64(num), err 56 | } 57 | 58 | func (rpc EthBlockChainRPC) GetLogs( 59 | fromBlockNum, toBlockNum uint64, 60 | address string, 61 | topics []string, 62 | ) ([]blockchain.IReceiptLog, error) { 63 | 64 | filterParam := ethrpc.FilterParams{ 65 | FromBlock: "0x" + strconv.FormatUint(fromBlockNum, 16), 66 | ToBlock: "0x" + strconv.FormatUint(toBlockNum, 16), 67 | Address: []string{address}, 68 | Topics: [][]string{topics}, 69 | } 70 | 71 | logs, err := rpc.rpcImpl.EthGetLogs(filterParam) 72 | if err != nil { 73 | logrus.Warnf("EthGetLogs err: %s, params: %+v", err, filterParam) 74 | return nil, err 75 | } 76 | 77 | logrus.Debugf("EthGetLogs logs count at block(%d - %d): %d", fromBlockNum, toBlockNum, len(logs)) 78 | 79 | var result []blockchain.IReceiptLog 80 | for i := 0; i < len(logs); i++ { 81 | l := logs[i] 82 | 83 | logrus.Debugf("EthGetLogs receipt log: %+v", l) 84 | 85 | result = append(result, blockchain.ReceiptLog{Log: &l}) 86 | } 87 | 88 | return result, err 89 | } 90 | -------------------------------------------------------------------------------- /plugin/tx_receipt_plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 5 | "github.com/HydroProtocol/ethereum-watcher/structs" 6 | "github.com/shopspring/decimal" 7 | "math/big" 8 | ) 9 | 10 | type ITxReceiptPlugin interface { 11 | Accept(tx *structs.RemovableTxAndReceipt) 12 | } 13 | 14 | type TxReceiptPluginWithFilter struct { 15 | ITxReceiptPlugin 16 | filterFunc func(transaction blockchain.Transaction) bool 17 | } 18 | 19 | func (p TxReceiptPluginWithFilter) NeedReceipt(tx blockchain.Transaction) bool { 20 | return p.filterFunc(tx) 21 | } 22 | 23 | func NewTxReceiptPluginWithFilter( 24 | callback func(tx *structs.RemovableTxAndReceipt), 25 | filterFunc func(transaction blockchain.Transaction) bool) *TxReceiptPluginWithFilter { 26 | 27 | p := NewTxReceiptPlugin(callback) 28 | return &TxReceiptPluginWithFilter{p, filterFunc} 29 | } 30 | 31 | type TxReceiptPlugin struct { 32 | callback func(tx *structs.RemovableTxAndReceipt) 33 | } 34 | 35 | func NewTxReceiptPlugin(callback func(tx *structs.RemovableTxAndReceipt)) *TxReceiptPlugin { 36 | return &TxReceiptPlugin{callback} 37 | } 38 | 39 | func (p TxReceiptPlugin) Accept(tx *structs.RemovableTxAndReceipt) { 40 | if p.callback != nil { 41 | p.callback(tx) 42 | } 43 | } 44 | 45 | type ERC20TransferPlugin struct { 46 | callback func(tokenAddress, from, to string, amount decimal.Decimal, isRemoved bool) 47 | } 48 | 49 | func NewERC20TransferPlugin(callback func(tokenAddress, from, to string, amount decimal.Decimal, isRemoved bool)) *ERC20TransferPlugin { 50 | return &ERC20TransferPlugin{callback} 51 | } 52 | 53 | func (p *ERC20TransferPlugin) Accept(tx *structs.RemovableTxAndReceipt) { 54 | if p.callback != nil { 55 | events := extractERC20TransfersIfExist(tx) 56 | 57 | for _, e := range events { 58 | p.callback(e.token, e.from, e.to, e.value, tx.IsRemoved) 59 | } 60 | } 61 | } 62 | 63 | type TransferEvent struct { 64 | token string 65 | from string 66 | to string 67 | value decimal.Decimal 68 | } 69 | 70 | func extractERC20TransfersIfExist(r *structs.RemovableTxAndReceipt) (rst []TransferEvent) { 71 | transferEventSig := "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" 72 | 73 | // todo a little weird 74 | if receipt, ok := r.Receipt.(*blockchain.EthereumTransactionReceipt); ok { 75 | for _, log := range receipt.Logs { 76 | if len(log.Topics) != 3 || log.Topics[0] != transferEventSig { 77 | continue 78 | } 79 | 80 | from := log.Topics[1] 81 | to := log.Topics[2] 82 | 83 | if amount, ok := HexToDecimal(log.Data); ok { 84 | rst = append(rst, TransferEvent{log.Address, from, to, amount}) 85 | } 86 | } 87 | } 88 | 89 | return 90 | } 91 | 92 | func HexToDecimal(hex string) (decimal.Decimal, bool) { 93 | if hex[0:2] == "0x" || hex[0:2] == "0X" { 94 | hex = hex[2:] 95 | } 96 | 97 | b := new(big.Int) 98 | b, ok := b.SetString(hex, 16) 99 | if !ok { 100 | return decimal.Zero, false 101 | } 102 | 103 | return decimal.NewFromBigInt(b, 0), true 104 | } 105 | -------------------------------------------------------------------------------- /watcher_with_tx_receipt_test.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 7 | "github.com/HydroProtocol/ethereum-watcher/plugin" 8 | "github.com/HydroProtocol/ethereum-watcher/structs" 9 | "github.com/labstack/gommon/log" 10 | "github.com/shopspring/decimal" 11 | "github.com/sirupsen/logrus" 12 | "testing" 13 | ) 14 | 15 | // todo why some tx index in block is zero? 16 | func TestTxReceiptPlugin(t *testing.T) { 17 | log.SetLevel(log.DEBUG) 18 | 19 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 20 | w := NewHttpBasedEthWatcher(context.Background(), api) 21 | 22 | w.RegisterTxReceiptPlugin(plugin.NewTxReceiptPlugin(func(txAndReceipt *structs.RemovableTxAndReceipt) { 23 | if txAndReceipt.IsRemoved { 24 | fmt.Println("Removed >>", txAndReceipt.Tx.GetHash(), txAndReceipt.Receipt.GetTxIndex()) 25 | } else { 26 | fmt.Println("Adding >>", txAndReceipt.Tx.GetHash(), txAndReceipt.Receipt.GetTxIndex()) 27 | } 28 | })) 29 | 30 | w.RunTillExit() 31 | } 32 | 33 | func TestErc20TransferPlugin(t *testing.T) { 34 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 35 | w := NewHttpBasedEthWatcher(context.Background(), api) 36 | 37 | w.RegisterTxReceiptPlugin(plugin.NewERC20TransferPlugin( 38 | func(token, from, to string, amount decimal.Decimal, isRemove bool) { 39 | 40 | logrus.Infof("New ERC20 Transfer >> token(%s), %s -> %s, amount: %s, isRemoved: %t", 41 | token, from, to, amount, isRemove) 42 | 43 | }, 44 | )) 45 | 46 | w.RunTillExit() 47 | } 48 | 49 | func TestFilterPlugin(t *testing.T) { 50 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 51 | w := NewHttpBasedEthWatcher(context.Background(), api) 52 | 53 | callback := func(txAndReceipt *structs.RemovableTxAndReceipt) { 54 | fmt.Println("tx:", txAndReceipt.Tx.GetHash()) 55 | } 56 | 57 | // only accept txs which end with: f 58 | filterFunc := func(tx blockchain.Transaction) bool { 59 | txHash := tx.GetHash() 60 | 61 | return txHash[len(txHash)-1:] == "f" 62 | } 63 | 64 | w.RegisterTxReceiptPlugin(plugin.NewTxReceiptPluginWithFilter(callback, filterFunc)) 65 | 66 | err := w.RunTillExitFromBlock(7840000) 67 | if err != nil { 68 | fmt.Println("RunTillExit with err:", err) 69 | } 70 | } 71 | 72 | func TestFilterPluginForDyDxApprove(t *testing.T) { 73 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 74 | w := NewHttpBasedEthWatcher(context.Background(), api) 75 | 76 | callback := func(txAndReceipt *structs.RemovableTxAndReceipt) { 77 | receipt := txAndReceipt.Receipt 78 | 79 | for _, log := range receipt.GetLogs() { 80 | topics := log.GetTopics() 81 | if len(topics) == 3 && 82 | topics[0] == "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925" && 83 | topics[2] == "0x0000000000000000000000001e0447b19bb6ecfdae1e4ae1694b0c3659614e4e" { 84 | fmt.Printf(">> approving to dydx, tx: %s\n", txAndReceipt.Tx.GetHash()) 85 | } 86 | } 87 | } 88 | 89 | // only accept txs which send to DAI 90 | filterFunc := func(tx blockchain.Transaction) bool { 91 | return tx.GetTo() == "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359" 92 | } 93 | 94 | w.RegisterTxReceiptPlugin(plugin.NewTxReceiptPluginWithFilter(callback, filterFunc)) 95 | 96 | err := w.RunTillExitFromBlock(7844853) 97 | if err != nil { 98 | fmt.Println("RunTillExit with err:", err) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher" 7 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 8 | "github.com/HydroProtocol/ethereum-watcher/plugin" 9 | "github.com/HydroProtocol/ethereum-watcher/rpc" 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | "os" 13 | "os/signal" 14 | ) 15 | 16 | const ( 17 | api = "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 18 | ) 19 | 20 | var contractAdx string 21 | var eventSigs []string 22 | var blockBackoff int 23 | 24 | func main() { 25 | rootCMD.AddCommand(blockNumCMD) 26 | rootCMD.AddCommand(usdtTransferCMD) 27 | 28 | contractEventListenerCMD.Flags().StringVarP(&contractAdx, "contract", "c", "", "contract address listen to") 29 | contractEventListenerCMD.MarkFlagRequired("contract") 30 | contractEventListenerCMD.Flags().StringArrayVarP(&eventSigs, "events", "e", []string{}, "signatures of events we are interested in") 31 | contractEventListenerCMD.MarkFlagRequired("events") 32 | contractEventListenerCMD.Flags().IntVar(&blockBackoff, "block-backoff", 0, "how many blocks we go back") 33 | rootCMD.AddCommand(contractEventListenerCMD) 34 | 35 | if err := rootCMD.Execute(); err != nil { 36 | fmt.Println(err) 37 | os.Exit(1) 38 | } 39 | } 40 | 41 | var rootCMD = &cobra.Command{ 42 | Use: "ethereum-watcher", 43 | Short: "ethereum-watcher makes getting updates from Ethereum easier", 44 | } 45 | 46 | var blockNumCMD = &cobra.Command{ 47 | Use: "new-block-number", 48 | Short: "Print number of new block", 49 | Run: func(cmd *cobra.Command, args []string) { 50 | ctx, cancel := context.WithCancel(context.Background()) 51 | 52 | c := make(chan os.Signal, 1) 53 | signal.Notify(c, os.Interrupt) 54 | 55 | w := ethereum_watcher.NewHttpBasedEthWatcher(ctx, api) 56 | 57 | logrus.Println("waiting for new block...") 58 | w.RegisterBlockPlugin(plugin.NewBlockNumPlugin(func(i uint64, b bool) { 59 | logrus.Printf(">> found new block: %d, is removed: %t", i, b) 60 | })) 61 | 62 | go func() { 63 | <-c 64 | cancel() 65 | }() 66 | 67 | err := w.RunTillExit() 68 | if err != nil { 69 | logrus.Printf("exit with err: %s", err) 70 | } else { 71 | logrus.Infoln("exit") 72 | } 73 | }, 74 | } 75 | 76 | var usdtTransferCMD = &cobra.Command{ 77 | Use: "usdt-transfer", 78 | Short: "Show Transfer Event of USDT", 79 | Run: func(cmd *cobra.Command, args []string) { 80 | usdtContractAdx := "0xdac17f958d2ee523a2206206994597c13d831ec7" 81 | 82 | // Transfer 83 | topicsInterestedIn := []string{"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"} 84 | 85 | handler := func(from, to int, receiptLogs []blockchain.IReceiptLog, isUpToHighestBlock bool) error { 86 | 87 | if from != to { 88 | logrus.Infof("See new USDT Transfer at blockRange: %d -> %d, count: %2d", from, to, len(receiptLogs)) 89 | } else { 90 | logrus.Infof("See new USDT Transfer at block: %d, count: %2d", from, len(receiptLogs)) 91 | } 92 | 93 | for _, log := range receiptLogs { 94 | logrus.Infof(" >> tx: https://etherscan.io/tx/%s", log.GetTransactionHash()) 95 | } 96 | 97 | fmt.Println(" ") 98 | 99 | return nil 100 | } 101 | 102 | receiptLogWatcher := ethereum_watcher.NewReceiptLogWatcher( 103 | context.TODO(), 104 | api, 105 | -1, 106 | usdtContractAdx, 107 | topicsInterestedIn, 108 | handler, 109 | ethereum_watcher.ReceiptLogWatcherConfig{ 110 | StepSizeForBigLag: 5, 111 | IntervalForPollingNewBlockInSec: 5, 112 | RPCMaxRetry: 3, 113 | ReturnForBlockWithNoReceiptLog: true, 114 | }, 115 | ) 116 | 117 | receiptLogWatcher.Run() 118 | }, 119 | } 120 | 121 | var contractEventListenerCMD = &cobra.Command{ 122 | Use: "contract-event-listener", 123 | Short: "listen and print events from contract", 124 | Example: ` 125 | listen to Transfer & Approve events from Multi-Collateral-DAI 126 | 127 | /bin/ethereum-watcher contract-event-listener \ 128 | --block-backoff 100 129 | --contract 0x6b175474e89094c44da98b954eedeac495271d0f \ 130 | --events 0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef`, 131 | Run: func(cmd *cobra.Command, args []string) { 132 | 133 | handler := func(from, to int, receiptLogs []blockchain.IReceiptLog, isUpToHighestBlock bool) error { 134 | 135 | if from != to { 136 | logrus.Infof("# of interested events at block(%d->%d): %d", from, to, len(receiptLogs)) 137 | } else { 138 | logrus.Infof("# of interested events at block(%d): %d", from, len(receiptLogs)) 139 | } 140 | 141 | for _, log := range receiptLogs { 142 | logrus.Infof(" >> tx: https://etherscan.io/tx/%s", log.GetTransactionHash()) 143 | } 144 | 145 | fmt.Println(" ") 146 | 147 | return nil 148 | } 149 | 150 | startBlockNum := -1 151 | if blockBackoff > 0 { 152 | rpc := rpc.NewEthRPCWithRetry(api, 3) 153 | curBlockNum, err := rpc.GetCurrentBlockNum() 154 | if err == nil { 155 | startBlockNum = int(curBlockNum) - blockBackoff 156 | 157 | if startBlockNum > 0 { 158 | logrus.Infof("--block-backoff activated, we start from block: %d (= %d - %d)", 159 | startBlockNum, curBlockNum, blockBackoff) 160 | } 161 | } 162 | } 163 | 164 | receiptLogWatcher := ethereum_watcher.NewReceiptLogWatcher( 165 | context.TODO(), 166 | api, 167 | startBlockNum, 168 | contractAdx, 169 | eventSigs, 170 | handler, 171 | ethereum_watcher.ReceiptLogWatcherConfig{ 172 | StepSizeForBigLag: 5, 173 | IntervalForPollingNewBlockInSec: 5, 174 | RPCMaxRetry: 3, 175 | ReturnForBlockWithNoReceiptLog: true, 176 | }, 177 | ) 178 | 179 | receiptLogWatcher.Run() 180 | }, 181 | } 182 | -------------------------------------------------------------------------------- /receipt_log_watcher.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 7 | "github.com/HydroProtocol/ethereum-watcher/rpc" 8 | "github.com/sirupsen/logrus" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type ReceiptLogWatcher struct { 14 | ctx context.Context 15 | api string 16 | startBlockNum int 17 | contract string 18 | interestedTopics []string 19 | handler func(from, to int, receiptLogs []blockchain.IReceiptLog, isUpToHighestBlock bool) error 20 | config ReceiptLogWatcherConfig 21 | highestSyncedBlockNum int 22 | highestSyncedLogIndex int 23 | } 24 | 25 | func NewReceiptLogWatcher( 26 | ctx context.Context, 27 | api string, 28 | startBlockNum int, 29 | contract string, 30 | interestedTopics []string, 31 | handler func(from, to int, receiptLogs []blockchain.IReceiptLog, isUpToHighestBlock bool) error, 32 | configs ...ReceiptLogWatcherConfig, 33 | ) *ReceiptLogWatcher { 34 | 35 | config := decideConfig(configs...) 36 | 37 | pseudoSyncedLogIndex := config.StartSyncAfterLogIndex - 1 38 | 39 | return &ReceiptLogWatcher{ 40 | ctx: ctx, 41 | api: api, 42 | startBlockNum: startBlockNum, 43 | contract: contract, 44 | interestedTopics: interestedTopics, 45 | handler: handler, 46 | config: config, 47 | highestSyncedBlockNum: startBlockNum, 48 | highestSyncedLogIndex: pseudoSyncedLogIndex, 49 | } 50 | } 51 | 52 | func decideConfig(configs ...ReceiptLogWatcherConfig) ReceiptLogWatcherConfig { 53 | var config ReceiptLogWatcherConfig 54 | if len(configs) == 0 { 55 | config = defaultConfig 56 | } else { 57 | config = configs[0] 58 | 59 | if config.IntervalForPollingNewBlockInSec <= 0 { 60 | config.IntervalForPollingNewBlockInSec = defaultConfig.IntervalForPollingNewBlockInSec 61 | } 62 | 63 | if config.StepSizeForBigLag <= 0 { 64 | config.StepSizeForBigLag = defaultConfig.StepSizeForBigLag 65 | } 66 | 67 | if config.RPCMaxRetry <= 0 { 68 | config.RPCMaxRetry = defaultConfig.RPCMaxRetry 69 | } 70 | } 71 | 72 | return config 73 | } 74 | 75 | type ReceiptLogWatcherConfig struct { 76 | StepSizeForBigLag int 77 | ReturnForBlockWithNoReceiptLog bool 78 | IntervalForPollingNewBlockInSec int 79 | RPCMaxRetry int 80 | LagToHighestBlock int 81 | StartSyncAfterLogIndex int 82 | } 83 | 84 | var defaultConfig = ReceiptLogWatcherConfig{ 85 | StepSizeForBigLag: 50, 86 | ReturnForBlockWithNoReceiptLog: false, 87 | IntervalForPollingNewBlockInSec: 15, 88 | RPCMaxRetry: 5, 89 | LagToHighestBlock: 0, 90 | StartSyncAfterLogIndex: 0, 91 | } 92 | 93 | func (w *ReceiptLogWatcher) Run() error { 94 | 95 | var blockNumToBeProcessedNext = w.startBlockNum 96 | 97 | rpc := rpc.NewEthRPCWithRetry(w.api, w.config.RPCMaxRetry) 98 | 99 | for { 100 | select { 101 | case <-w.ctx.Done(): 102 | return nil 103 | default: 104 | highestBlock, err := rpc.GetCurrentBlockNum() 105 | if err != nil { 106 | return err 107 | } 108 | 109 | if blockNumToBeProcessedNext < 0 { 110 | blockNumToBeProcessedNext = int(highestBlock) 111 | } 112 | 113 | // [blockNumToBeProcessedNext...highestBlockCanProcess..[Lag]..CurrentHighestBlock] 114 | highestBlockCanProcess := int(highestBlock) - w.config.LagToHighestBlock 115 | numOfBlocksToProcess := highestBlockCanProcess - blockNumToBeProcessedNext + 1 116 | 117 | if numOfBlocksToProcess <= 0 { 118 | sleepSec := w.config.IntervalForPollingNewBlockInSec 119 | 120 | logrus.Debugf("no ready block after %d(lag: %d), sleep %d seconds", highestBlockCanProcess, w.config.LagToHighestBlock, sleepSec) 121 | 122 | select { 123 | case <-time.After(time.Duration(sleepSec) * time.Second): 124 | continue 125 | case <-w.ctx.Done(): 126 | return nil 127 | } 128 | } 129 | 130 | var to int 131 | if numOfBlocksToProcess > w.config.StepSizeForBigLag { 132 | // quick mode 133 | to = blockNumToBeProcessedNext + w.config.StepSizeForBigLag - 1 134 | } else { 135 | // normal mode, up to cur highest block num can process 136 | to = highestBlockCanProcess 137 | } 138 | 139 | logs, err := rpc.GetLogs(uint64(blockNumToBeProcessedNext), uint64(to), w.contract, w.interestedTopics) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | isUpToHighestBlock := to == int(highestBlock) 145 | 146 | if len(logs) == 0 { 147 | if w.config.ReturnForBlockWithNoReceiptLog { 148 | err := w.handler(blockNumToBeProcessedNext, to, nil, isUpToHighestBlock) 149 | if err != nil { 150 | logrus.Infof("err when handling nil receipt log, block range: %d - %d", blockNumToBeProcessedNext, to) 151 | return fmt.Errorf("ethereum_watcher handler(nil) returns error: %s", err) 152 | } 153 | } 154 | } else { 155 | 156 | err := w.handler(blockNumToBeProcessedNext, to, logs, isUpToHighestBlock) 157 | if err != nil { 158 | logrus.Infof("err when handling receipt log, block range: %d - %d, receipt logs: %+v", 159 | blockNumToBeProcessedNext, to, logs, 160 | ) 161 | 162 | return fmt.Errorf("ethereum_watcher handler returns error: %s", err) 163 | } 164 | } 165 | 166 | // todo rm 2nd param 167 | w.updateHighestSyncedBlockNumAndLogIndex(to, -1) 168 | 169 | blockNumToBeProcessedNext = to + 1 170 | } 171 | } 172 | } 173 | 174 | var progressLock = sync.Mutex{} 175 | 176 | func (w *ReceiptLogWatcher) updateHighestSyncedBlockNumAndLogIndex(block int, logIndex int) { 177 | progressLock.Lock() 178 | defer progressLock.Unlock() 179 | 180 | w.highestSyncedBlockNum = block 181 | w.highestSyncedLogIndex = logIndex 182 | } 183 | 184 | func (w *ReceiptLogWatcher) GetHighestSyncedBlockNum() int { 185 | return w.highestSyncedBlockNum 186 | } 187 | 188 | func (w *ReceiptLogWatcher) GetHighestSyncedBlockNumAndLogIndex() (int, int) { 189 | progressLock.Lock() 190 | defer progressLock.Unlock() 191 | 192 | return w.highestSyncedBlockNum, w.highestSyncedLogIndex 193 | } 194 | -------------------------------------------------------------------------------- /blockchain/blockchain.go: -------------------------------------------------------------------------------- 1 | package blockchain 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/HydroProtocol/ethereum-watcher/utils" 7 | "github.com/labstack/gommon/log" 8 | "github.com/onrik/ethrpc" 9 | "github.com/shopspring/decimal" 10 | "math/big" 11 | "strconv" 12 | ) 13 | 14 | type BlockChain interface { 15 | GetTokenBalance(tokenAddress, address string) decimal.Decimal 16 | GetTokenAllowance(tokenAddress, proxyAddress, address string) decimal.Decimal 17 | 18 | GetBlockNumber() (uint64, error) 19 | GetBlockByNumber(blockNumber uint64) (Block, error) 20 | 21 | GetTransaction(ID string) (Transaction, error) 22 | GetTransactionReceipt(ID string) (TransactionReceipt, error) 23 | GetTransactionAndReceipt(ID string) (Transaction, TransactionReceipt, error) 24 | } 25 | 26 | type Block interface { 27 | Number() uint64 28 | Timestamp() uint64 29 | GetTransactions() []Transaction 30 | 31 | Hash() string 32 | ParentHash() string 33 | } 34 | 35 | type Transaction interface { 36 | GetBlockHash() string 37 | GetBlockNumber() uint64 38 | GetFrom() string 39 | GetGas() int 40 | GetGasPrice() big.Int 41 | GetHash() string 42 | GetTo() string 43 | GetValue() big.Int 44 | } 45 | 46 | type TransactionReceipt interface { 47 | GetResult() bool 48 | GetBlockNumber() uint64 49 | 50 | GetBlockHash() string 51 | GetTxHash() string 52 | GetTxIndex() int 53 | 54 | GetLogs() []IReceiptLog 55 | } 56 | 57 | type IReceiptLog interface { 58 | GetRemoved() bool 59 | GetLogIndex() int 60 | GetTransactionIndex() int 61 | GetTransactionHash() string 62 | GetBlockNum() int 63 | GetBlockHash() string 64 | GetAddress() string 65 | GetData() string 66 | GetTopics() []string 67 | } 68 | 69 | // compile time interface check 70 | var _ BlockChain = &Ethereum{} 71 | 72 | type EthereumBlock struct { 73 | *ethrpc.Block 74 | } 75 | 76 | func (block *EthereumBlock) Hash() string { 77 | return block.Block.Hash 78 | } 79 | 80 | func (block *EthereumBlock) ParentHash() string { 81 | return block.Block.ParentHash 82 | } 83 | 84 | func (block *EthereumBlock) GetTransactions() []Transaction { 85 | txs := make([]Transaction, 0, 20) 86 | 87 | for i := range block.Block.Transactions { 88 | tx := block.Block.Transactions[i] 89 | txs = append(txs, &EthereumTransaction{&tx}) 90 | } 91 | 92 | return txs 93 | } 94 | 95 | func (block *EthereumBlock) Number() uint64 { 96 | return uint64(block.Block.Number) 97 | } 98 | 99 | func (block *EthereumBlock) Timestamp() uint64 { 100 | return uint64(block.Block.Timestamp) 101 | } 102 | 103 | type EthereumTransaction struct { 104 | *ethrpc.Transaction 105 | } 106 | 107 | func (t *EthereumTransaction) GetBlockHash() string { 108 | return t.BlockHash 109 | } 110 | 111 | func (t *EthereumTransaction) GetFrom() string { 112 | return t.From 113 | } 114 | 115 | func (t *EthereumTransaction) GetGas() int { 116 | return t.Gas 117 | } 118 | 119 | func (t *EthereumTransaction) GetGasPrice() big.Int { 120 | return t.GasPrice 121 | } 122 | 123 | func (t *EthereumTransaction) GetValue() big.Int { 124 | return t.Value 125 | } 126 | 127 | func (t *EthereumTransaction) GetTo() string { 128 | return t.To 129 | } 130 | 131 | func (t *EthereumTransaction) GetHash() string { 132 | return t.Hash 133 | } 134 | func (t *EthereumTransaction) GetBlockNumber() uint64 { 135 | return uint64(*t.BlockNumber) 136 | } 137 | 138 | type EthereumTransactionReceipt struct { 139 | *ethrpc.TransactionReceipt 140 | } 141 | 142 | func (r *EthereumTransactionReceipt) GetLogs() (rst []IReceiptLog) { 143 | for i := range r.Logs { 144 | l := ReceiptLog{&r.Logs[i]} 145 | rst = append(rst, l) 146 | } 147 | 148 | return 149 | } 150 | 151 | func (r *EthereumTransactionReceipt) GetResult() bool { 152 | res, err := strconv.ParseInt(r.Status, 0, 64) 153 | 154 | if err != nil { 155 | panic(err) 156 | } 157 | 158 | return res == 1 159 | } 160 | 161 | func (r *EthereumTransactionReceipt) GetBlockNumber() uint64 { 162 | return uint64(r.BlockNumber) 163 | } 164 | 165 | func (r *EthereumTransactionReceipt) GetBlockHash() string { 166 | return r.BlockHash 167 | } 168 | func (r *EthereumTransactionReceipt) GetTxHash() string { 169 | return r.TransactionHash 170 | } 171 | func (r *EthereumTransactionReceipt) GetTxIndex() int { 172 | return r.TransactionIndex 173 | } 174 | 175 | type ReceiptLog struct { 176 | *ethrpc.Log 177 | } 178 | 179 | func (log ReceiptLog) GetRemoved() bool { 180 | return log.Removed 181 | } 182 | 183 | func (log ReceiptLog) GetLogIndex() int { 184 | return log.LogIndex 185 | } 186 | 187 | func (log ReceiptLog) GetTransactionIndex() int { 188 | return log.TransactionIndex 189 | } 190 | 191 | func (log ReceiptLog) GetTransactionHash() string { 192 | return log.TransactionHash 193 | } 194 | 195 | func (log ReceiptLog) GetBlockNum() int { 196 | return log.BlockNumber 197 | } 198 | 199 | func (log ReceiptLog) GetBlockHash() string { 200 | return log.BlockHash 201 | } 202 | 203 | func (log ReceiptLog) GetAddress() string { 204 | return log.Address 205 | } 206 | 207 | func (log ReceiptLog) GetData() string { 208 | return log.Data 209 | } 210 | 211 | func (log ReceiptLog) GetTopics() []string { 212 | return log.Topics 213 | } 214 | 215 | type Ethereum struct { 216 | client *ethrpc.EthRPC 217 | hybridExAddr string 218 | } 219 | 220 | func (e *Ethereum) EnableDebug(b bool) { 221 | e.client.Debug = b 222 | } 223 | 224 | func (e *Ethereum) GetBlockByNumber(number uint64) (Block, error) { 225 | 226 | block, err := e.client.EthGetBlockByNumber(int(number), true) 227 | 228 | if err != nil { 229 | log.Errorf("get Block by Number failed %+v", err) 230 | return nil, err 231 | } 232 | 233 | if block == nil { 234 | log.Errorf("get Block by Number returns nil block for num: %d", number) 235 | return nil, errors.New("get Block by Number returns nil block for num: " + strconv.Itoa(int(number))) 236 | } 237 | 238 | return &EthereumBlock{block}, nil 239 | } 240 | 241 | func (e *Ethereum) GetBlockNumber() (uint64, error) { 242 | number, err := e.client.EthBlockNumber() 243 | 244 | if err != nil { 245 | log.Errorf("GetBlockNumber failed, %v", err) 246 | return 0, err 247 | } 248 | 249 | return uint64(number), nil 250 | } 251 | 252 | func (e *Ethereum) GetTransaction(ID string) (Transaction, error) { 253 | tx, err := e.client.EthGetTransactionByHash(ID) 254 | 255 | if err != nil { 256 | log.Errorf("GetTransaction failed, %v", err) 257 | return nil, err 258 | } 259 | 260 | return &EthereumTransaction{tx}, nil 261 | } 262 | 263 | func (e *Ethereum) GetTransactionReceipt(ID string) (TransactionReceipt, error) { 264 | txReceipt, err := e.client.EthGetTransactionReceipt(ID) 265 | 266 | if err != nil { 267 | log.Errorf("GetTransactionReceipt failed, %v", err) 268 | return nil, err 269 | } 270 | 271 | return &EthereumTransactionReceipt{txReceipt}, nil 272 | } 273 | 274 | func (e *Ethereum) GetTransactionAndReceipt(ID string) (Transaction, TransactionReceipt, error) { 275 | txReceiptChannel := make(chan TransactionReceipt) 276 | 277 | go func() { 278 | rec, _ := e.GetTransactionReceipt(ID) 279 | txReceiptChannel <- rec 280 | }() 281 | 282 | txInfoChannel := make(chan Transaction) 283 | go func() { 284 | tx, _ := e.GetTransaction(ID) 285 | txInfoChannel <- tx 286 | }() 287 | 288 | return <-txInfoChannel, <-txReceiptChannel, nil 289 | } 290 | 291 | func (e *Ethereum) GetTokenBalance(tokenAddress, address string) decimal.Decimal { 292 | res, err := e.client.EthCall(ethrpc.T{ 293 | To: tokenAddress, 294 | From: address, 295 | Data: fmt.Sprintf("0x70a08231000000000000000000000000%s", without0xPrefix(address)), 296 | }, "latest") 297 | 298 | if err != nil { 299 | panic(err) 300 | } 301 | 302 | return utils.StringToDecimal(res) 303 | } 304 | 305 | func without0xPrefix(address string) string { 306 | if address[:2] == "0x" { 307 | address = address[2:] 308 | } 309 | 310 | return address 311 | } 312 | 313 | func (e *Ethereum) GetTokenAllowance(tokenAddress, proxyAddress, address string) decimal.Decimal { 314 | res, err := e.client.EthCall(ethrpc.T{ 315 | To: tokenAddress, 316 | From: address, 317 | Data: fmt.Sprintf("0xdd62ed3e000000000000000000000000%s000000000000000000000000%s", without0xPrefix(address), without0xPrefix(proxyAddress)), 318 | }, "latest") 319 | 320 | if err != nil { 321 | panic(err) 322 | } 323 | 324 | return utils.StringToDecimal(res) 325 | } 326 | 327 | func (e *Ethereum) GetTransactionCount(address string) (int, error) { 328 | return e.client.EthGetTransactionCount(address, "latest") 329 | } 330 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= 3 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 4 | github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d h1:xG8Pj6Y6J760xwETNmMzmlt38QSwz0BLp1cZ09g27uw= 5 | github.com/btcsuite/btcd v0.0.0-20190115013929-ed77733ec07d/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0= 6 | github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= 7 | github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= 8 | github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= 9 | github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= 10 | github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= 11 | github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= 12 | github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= 13 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 14 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 15 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 16 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 17 | github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 21 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 22 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 23 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 24 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 25 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 26 | github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= 27 | github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= 28 | github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 29 | github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= 30 | github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= 31 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 32 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 33 | github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0= 34 | github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= 35 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 36 | github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= 37 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 38 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 39 | github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= 40 | github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= 41 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 42 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 43 | github.com/onrik/ethrpc v0.0.0-20190305112807-6b8e9c0e9a8f h1:rbdYasawEKAS83d58peaGvf3rs8Okxag/DTQckZyJT0= 44 | github.com/onrik/ethrpc v0.0.0-20190305112807-6b8e9c0e9a8f/go.mod h1:RoqOlDiBBs1qYamkcYhxMgkPijxu5R8t55mgUiy4le8= 45 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 46 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 47 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 48 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 49 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 50 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 51 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 52 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= 53 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= 54 | github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= 55 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 56 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 57 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 58 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 59 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 60 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 61 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 62 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 63 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 64 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 65 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 66 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 67 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 68 | github.com/tidwall/gjson v1.3.5 h1:2oW9FBNu8qt9jy5URgrzsVx/T/KSn3qn/smJQ0crlDQ= 69 | github.com/tidwall/gjson v1.3.5/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= 70 | github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= 71 | github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= 72 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= 73 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 74 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 75 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 76 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 77 | github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= 78 | github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 79 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 80 | golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 81 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 82 | golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= 83 | golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= 84 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 85 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 86 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 87 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 88 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 89 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 90 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 91 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 92 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM= 93 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 94 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 95 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 96 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 97 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 98 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 99 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 100 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # ethereum-watcher 2 | 3 | ![](https://github.com/HydroProtocol/ethereum-watcher/workflows/Go/badge.svg) 4 | 5 | ethereum-watcher is an event listener for the [Ethereum Blockchain](https://ethereum.org/) written in Golang. With ethereum-watcher you can monitor and track current or historic events that occur on the Ethereum Blockchain. 6 | 7 | ## Background 8 | 9 | Many applications that interact with the Ethereum Blockchain need to know when specific actions occur on the chain, but cannot directly access the on-chain data. ethereum-watcher acts as an interface between application and chain: gathering specified data from the blockchain so that applications can more seamlessly interact with on-chain events. 10 | 11 | 12 | ## Features 13 | 14 | 1. Plug-in friendly. You can easily add a plugin to ethereum-watcher to listen to any type of on-chain event. 15 | 2. Fork Tolerance. If a [fork](https://en.wikipedia.org/wiki/Fork_(blockchain)) occurs, a revert message is sent to the subscriber. 16 | 17 | ## Example Use Cases 18 | 19 | - [DefiWatch](https://defiwatch.io/) monitors the state of Ethereum addresses on several DeFi platforms including DDEX, Compound, DyDx, and Maker. It tracks things like loan ROI, current borrows, liquidations, etc. To track all of this on multiple platforms, DefiWatch has to continuously receive updates from their associated smart contracts. This is done using ethereum-watcher instead of spending time dealing with serialization/deserialization messages from the Ethereum Node, so they can focus on their core logic. 20 | - Profit & Loss calculations on [DDEX](https://ddex.io). DDEX provides their margin trading users with estimated Profit and Loss (P&L) calculations for their margin positions. To update the P&L as timely and accurately as possible, DDEX uses ethereum-watcher to listen to updates from the Ethereum Blockchain. These updates include: onchain price updates, trading actions from users, and more. 21 | - DDEX also uses an "Eth-Transaction-Watcher" to monitor the on-chain status of trading transactions. DDEX needs to know the latest states of these transactions once they are included in newly mined blocks, so that the platform properly updates trading balances and histories. This is done using the `TxReceiptPlugin` of ethereum-watcher. 22 | 23 | 24 | # Installation 25 | 26 | Run `go get github.com/HydroProtocol/ethereum-watcher` 27 | 28 | ## Sample Commands 29 | 30 | This project is primarily designed as a library to build upon. However, to help others easily understand how ethereum-watcher works and what it is capable of, we prepared some sample commands for you to try out. 31 | 32 | **display basic help info** 33 | 34 | ```shell 35 | docker run hydroprotocolio/ethereum-watcher:master /bin/ethereum-watcher help 36 | 37 | ethereum-watcher makes getting updates from Ethereum easier 38 | 39 | Usage: 40 | ethereum-watcher [command] 41 | 42 | Available Commands: 43 | contract-event-listener listen and print events from contract 44 | help Help about any command 45 | new-block-number Print number of new block 46 | usdt-transfer Show Transfer Event of USDT 47 | 48 | Flags: 49 | -h, --help help for ethereum-watcher 50 | 51 | Use "ethereum-watcher [command] --help" for more information about a command. 52 | ``` 53 | 54 | 55 | 56 | **print new block numbers** 57 | 58 | ```shell 59 | docker run hydroprotocolio/ethereum-watcher:master /bin/ethereum-watcher new-block-number 60 | 61 | time="2020-01-07T07:33:17Z" level=info msg="waiting for new block..." 62 | time="2020-01-07T07:33:19Z" level=info msg=">> found new block: 9232152, is removed: false" 63 | time="2020-01-07T07:33:44Z" level=info msg=">> found new block: 9232153, is removed: false" 64 | time="2020-01-07T07:34:03Z" level=info msg=">> found new block: 9232154, is removed: false" 65 | time="2020-01-07T07:34:04Z" level=info msg=">> found new block: 9232155, is removed: false" 66 | time="2020-01-07T07:34:05Z" level=info msg=">> found new block: 9232156, is removed: false" 67 | ... 68 | ``` 69 | 70 | 71 | 72 | **see USDT transfer events** 73 | 74 | ```shell 75 | docker run hydroprotocolio/ethereum-watcher:master /bin/ethereum-watcher usdt-transfer 76 | 77 | time="2020-01-07T07:34:32Z" level=info msg="See new USDT Transfer at block: 9232158, count: 9" 78 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0x1072efee913229a5e9a3013af6b580099f03ac9c75bfc60013cfa7efac726067" 79 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0xae191608df7688ad6d83ebe8151fc5519ff0e29515b557e15ed3fbcbfadca698" 80 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0xc3bcfcffa4d3318c27b923eccbd6bbac179f9f982d0282bed164c8e5216130cc" 81 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0xe15f2b029c248d1560b10f1879af46b2c8fb2362db9287a8a257042ec2dbb46c" 82 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0x860156ebe4c1f00cc479d4f75b7665e2c6ef66d3b085ba1630e803fa44ce2836" 83 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0x5b2f4b07332096f55c6ca69e3b349fc19e2c6f54f3a7062f4365697ffb6c5d51" 84 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0xb9325afae978b509feee8bc092a090e7a4df750e9e94d10a2bce29e543406c26" 85 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0xe379b9e776cf0920a5cc284e9a0f7d141894078827b28f43905dd45457f886e3" 86 | time="2020-01-07T07:34:32Z" level=info msg=" >> tx: https://etherscan.io/tx/0x0f89d867117a6107de44da523b1ad8fbd630d42d9f809c16b82c6f582a8c97da" 87 | 88 | time="2020-01-07T07:34:50Z" level=info msg="See new USDT Transfer at block: 9232159, count: 18" 89 | time="2020-01-07T07:34:50Z" level=info msg=" >> tx: https://etherscan.io/tx/0x506ff84b205f0802f037eefe988c9c431d5cae40d3c2c9354616f99f1751bad9" 90 | time="2020-01-07T07:34:50Z" level=info msg=" >> tx: https://etherscan.io/tx/0xd55a1175801eb9af3be65db23d3a9a32f4494f31473ad92dbd95e1d9f9b57fc3" 91 | time="2020-01-07T07:34:50Z" level=info msg=" >> tx: https://etherscan.io/tx/0x93bba744986c75645590b2c930b43e34e9ab219feb38875b87b327854c37e7eb" 92 | time="2020-01-07T07:34:50Z" level=info msg=" >> tx: https://etherscan.io/tx/0xae22eaa9aef7079ff9bb23fa25f314fcf22e9688ef7b585462d3a2afe33628ef" 93 | ... 94 | ``` 95 | **see specific events that occur within a smart contract. The example shows Transfer & Approve events from Multi-Collateral-DAI** 96 | 97 | ```shell 98 | docker run hydroprotocolio/ethereum-watcher:master /bin/ethereum-watcher contract-event-listener \ 99 | --block-backoff 100 \ 100 | --contract 0x6b175474e89094c44da98b954eedeac495271d0f \ 101 | --events 0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef 102 | 103 | INFO[2020-01-07T18:05:26+08:00] --block-backoff activated, we start from block: 9232741 (= 9232841 - 100) 104 | 105 | INFO[2020-01-07T18:05:27+08:00] # of interested events at block(9232741->9232745): 1 106 | INFO[2020-01-07T18:05:27+08:00] >> tx: https://etherscan.io/tx/0x4e05d731ca1b8bf0e2a85d825751312c722bfa3c9210a8b574c06e1e6c992a75 107 | 108 | INFO[2020-01-07T18:05:28+08:00] # of interested events at block(9232746->9232750): 3 109 | INFO[2020-01-07T18:05:28+08:00] >> tx: https://etherscan.io/tx/0x8645784c881a01d326469ddbf45a151f9eab8bedbfc02f6cbe1ab2f03c03c022 110 | INFO[2020-01-07T18:05:28+08:00] >> tx: https://etherscan.io/tx/0xac5e9fa166a2e7f13aa104812b9278cc65247beff68a4aca7b382e9547d84ec5 111 | INFO[2020-01-07T18:05:28+08:00] >> tx: https://etherscan.io/tx/0xac5e9fa166a2e7f13aa104812b9278cc65247beff68a4aca7b382e9547d84ec5 112 | 113 | INFO[2020-01-07T18:05:29+08:00] # of interested events at block(9232751->9232755): 2 114 | INFO[2020-01-07T18:05:29+08:00] >> tx: https://etherscan.io/tx/0x280eb9556f6286bf707e859fc6c08bdd1e2ed8f2bdd360883c207f0da0122b01 115 | INFO[2020-01-07T18:05:29+08:00] >> tx: https://etherscan.io/tx/0xcc5ee10d5ac8f55f51f74b23186052c042ebc5dda61578c1a1038d0b30b6fd91 116 | ... 117 | ``` 118 | Here the flag `--block-backoff` signals for ethereum-watcher to use historic tracking from 100 blocks ago. 119 | 120 | # Usage 121 | 122 | To effectively use ethereum-watcher, you will be interacting with two primary structs: 123 | 124 | - Watcher 125 | - ReceiptLogWatcher 126 | 127 | ## Watcher 128 | 129 | `Watcher` is an HTTP client which continuously polls newly mined blocks on the Ethereum Blockchain. We can incorporate various kinds of "plugins" into `Watcher`, which will poll for specific types of events and data, such as: 130 | 131 | - BlockPlugin 132 | - TransactionPlugin 133 | - TransactionReceiptPlugin 134 | - ReceiptLogPlugin 135 | 136 | Once the `Watcher` sees a new block, it will parse the info and feed the data into the registered plugins. You can see some of the code examples [below](#watcher-examples). 137 | 138 | The plugins are designed to be easily modifiable so that you can create your own plugin based on the provided ones. For example, the code shown [below](#listen-for-new-erc20-transfer-events) registers an `ERC20TransferPlugin` to show new ERC20 Transfer Events. This plugin simply parses some receipt info from a different `TransactionReceiptPlugin`. So if you want to show more info than what the `ERC20TransferPlugin` shows, like the gas used in the transaction, you can easily create a `BetterERC20TransferPlugin` showing that. 139 | 140 | ### Watcher Examples 141 | 142 | #### Print number of newly mined blocks 143 | 144 | ```go 145 | package main 146 | 147 | import ( 148 | "context" 149 | "fmt" 150 | "github.com/HydroProtocol/ethereum-watcher/plugin" 151 | "github.com/HydroProtocol/ethereum-watcher/structs" 152 | ) 153 | 154 | func main() { 155 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 156 | w := NewHttpBasedEthWatcher(context.Background(), api) 157 | 158 | // we use BlockPlugin here 159 | w.RegisterBlockPlugin(plugin.NewBlockNumPlugin(func(i uint64, b bool) { 160 | fmt.Println(">>", i, b) 161 | })) 162 | 163 | w.RunTillExit() 164 | } 165 | ``` 166 | 167 | #### Listen for new ERC20 Transfer Events 168 | 169 | ```go 170 | package main 171 | 172 | import ( 173 | "context" 174 | "fmt" 175 | "github.com/HydroProtocol/ethereum-watcher/plugin" 176 | "github.com/HydroProtocol/ethereum-watcher/structs" 177 | "github.com/sirupsen/logrus" 178 | ) 179 | 180 | func main() { 181 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 182 | w := NewHttpBasedEthWatcher(context.Background(), api) 183 | 184 | // we use TxReceiptPlugin here 185 | w.RegisterTxReceiptPlugin(plugin.NewERC20TransferPlugin( 186 | func(token, from, to string, amount decimal.Decimal, isRemove bool) { 187 | 188 | logrus.Infof("New ERC20 Transfer >> token(%s), %s -> %s, amount: %s, isRemoved: %t", 189 | token, from, to, amount, isRemove) 190 | 191 | }, 192 | )) 193 | 194 | w.RunTillExit() 195 | } 196 | ``` 197 | 198 | ## ReceiptLogWatcher 199 | 200 | `Watcher` is polling for blocks one by one, so what if we want to query certain events from the latest 10000 blocks? `Watcher` can do that but fetching blocks one by one can be slow. `ReceiptLogWatcher` to the rescue! 201 | 202 | `ReceiptLogWatcher` makes use of the `eth_getLogs` to query for logs in a batch. Check out the code [below](#example-of-receiptlogwatcher) to see how to use it. 203 | 204 | 205 | ### Example of ReceiptLogWatcher 206 | 207 | ```go 208 | package main 209 | 210 | import ( 211 | "context" 212 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 213 | "github.com/sirupsen/logrus" 214 | ) 215 | 216 | func main() { 217 | api := "https://mainnet.infura.io/v3/19d753b2600445e292d54b1ef58d4df4" 218 | usdtContractAdx := "0xdac17f958d2ee523a2206206994597c13d831ec7" 219 | 220 | // ERC20 Transfer Event 221 | topicsInterestedIn := []string{"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"} 222 | 223 | handler := func(from, to int, receiptLogs []blockchain.IReceiptLog, isUpToHighestBlock bool) error { 224 | logrus.Infof("USDT Transfer count: %d, %d -> %d", len(receiptLogs), from, to) 225 | return nil 226 | } 227 | 228 | // query for USDT Transfer Events 229 | receiptLogWatcher := NewReceiptLogWatcher( 230 | context.TODO(), 231 | api, 232 | -1, 233 | usdtContractAdx, 234 | topicsInterestedIn, 235 | handler, 236 | ReceiptLogWatcherConfig{ 237 | StepSizeForBigLag: 5, 238 | IntervalForPollingNewBlockInSec: 5, 239 | RPCMaxRetry: 3, 240 | ReturnForBlockWithNoReceiptLog: true, 241 | }, 242 | ) 243 | 244 | receiptLogWatcher.Run() 245 | } 246 | ``` 247 | 248 | 249 | 250 | # License 251 | 252 | [Apache 2.0 License](LICENSE) 253 | -------------------------------------------------------------------------------- /watcher.go: -------------------------------------------------------------------------------- 1 | package ethereum_watcher 2 | 3 | import ( 4 | "container/list" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "github.com/HydroProtocol/ethereum-watcher/blockchain" 9 | "github.com/HydroProtocol/ethereum-watcher/plugin" 10 | "github.com/HydroProtocol/ethereum-watcher/rpc" 11 | "github.com/HydroProtocol/ethereum-watcher/structs" 12 | "github.com/sirupsen/logrus" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | type AbstractWatcher struct { 18 | rpc rpc.IBlockChainRPC 19 | 20 | Ctx context.Context 21 | lock sync.RWMutex 22 | 23 | NewBlockChan chan *structs.RemovableBlock 24 | NewTxAndReceiptChan chan *structs.RemovableTxAndReceipt 25 | NewReceiptLogChan chan *structs.RemovableReceiptLog 26 | 27 | SyncedBlocks *list.List 28 | SyncedTxAndReceipts *list.List 29 | MaxSyncedBlockToKeep int 30 | 31 | BlockPlugins []plugin.IBlockPlugin 32 | TxPlugins []plugin.ITxPlugin 33 | TxReceiptPlugins []plugin.ITxReceiptPlugin 34 | ReceiptLogPlugins []plugin.IReceiptLogPlugin 35 | 36 | ReceiptCatchUpFromBlock uint64 37 | 38 | sleepSecondsForNewBlock int 39 | wg sync.WaitGroup 40 | } 41 | 42 | func NewHttpBasedEthWatcher(ctx context.Context, api string) *AbstractWatcher { 43 | rpc := rpc.NewEthRPCWithRetry(api, 5) 44 | 45 | return &AbstractWatcher{ 46 | Ctx: ctx, 47 | rpc: rpc, 48 | NewBlockChan: make(chan *structs.RemovableBlock, 32), 49 | NewTxAndReceiptChan: make(chan *structs.RemovableTxAndReceipt, 518), 50 | NewReceiptLogChan: make(chan *structs.RemovableReceiptLog, 518), 51 | SyncedBlocks: list.New(), 52 | SyncedTxAndReceipts: list.New(), 53 | MaxSyncedBlockToKeep: 64, 54 | sleepSecondsForNewBlock: 5, 55 | wg: sync.WaitGroup{}, 56 | } 57 | } 58 | 59 | func (watcher *AbstractWatcher) RegisterBlockPlugin(plugin plugin.IBlockPlugin) { 60 | watcher.BlockPlugins = append(watcher.BlockPlugins, plugin) 61 | } 62 | 63 | func (watcher *AbstractWatcher) RegisterTxPlugin(plugin plugin.ITxPlugin) { 64 | watcher.TxPlugins = append(watcher.TxPlugins, plugin) 65 | } 66 | 67 | func (watcher *AbstractWatcher) RegisterTxReceiptPlugin(plugin plugin.ITxReceiptPlugin) { 68 | watcher.TxReceiptPlugins = append(watcher.TxReceiptPlugins, plugin) 69 | } 70 | 71 | func (watcher *AbstractWatcher) RegisterReceiptLogPlugin(plugin plugin.IReceiptLogPlugin) { 72 | watcher.ReceiptLogPlugins = append(watcher.ReceiptLogPlugins, plugin) 73 | } 74 | 75 | // start sync from latest block 76 | func (watcher *AbstractWatcher) RunTillExit() error { 77 | return watcher.RunTillExitFromBlock(0) 78 | } 79 | 80 | // start sync from given block 81 | // 0 means start from latest block 82 | func (watcher *AbstractWatcher) RunTillExitFromBlock(startBlockNum uint64) error { 83 | 84 | watcher.wg.Add(1) 85 | go func() { 86 | for block := range watcher.NewBlockChan { 87 | // run thru block plugins 88 | for i := 0; i < len(watcher.BlockPlugins); i++ { 89 | blockPlugin := watcher.BlockPlugins[i] 90 | 91 | blockPlugin.AcceptBlock(block) 92 | } 93 | 94 | // run thru tx plugins 95 | txPlugins := watcher.TxPlugins 96 | for i := 0; i < len(txPlugins); i++ { 97 | txPlugin := txPlugins[i] 98 | 99 | for j := 0; j < len(block.GetTransactions()); j++ { 100 | tx := structs.NewRemovableTx(block.GetTransactions()[j], false) 101 | txPlugin.AcceptTx(tx) 102 | } 103 | } 104 | } 105 | 106 | watcher.wg.Done() 107 | }() 108 | 109 | watcher.wg.Add(1) 110 | go func() { 111 | for removableTxAndReceipt := range watcher.NewTxAndReceiptChan { 112 | 113 | txReceiptPlugins := watcher.TxReceiptPlugins 114 | for i := 0; i < len(txReceiptPlugins); i++ { 115 | txReceiptPlugin := txReceiptPlugins[i] 116 | 117 | if p, ok := txReceiptPlugin.(*plugin.TxReceiptPluginWithFilter); ok { 118 | // for filter plugin, only feed receipt it wants 119 | if p.NeedReceipt(removableTxAndReceipt.Tx) { 120 | txReceiptPlugin.Accept(removableTxAndReceipt) 121 | } 122 | } else { 123 | txReceiptPlugin.Accept(removableTxAndReceipt) 124 | } 125 | } 126 | } 127 | 128 | watcher.wg.Done() 129 | }() 130 | 131 | watcher.wg.Add(1) 132 | go func() { 133 | for removableReceiptLog := range watcher.NewReceiptLogChan { 134 | logrus.Debugf("get receipt log from chan: %+v, txHash: %s", removableReceiptLog, removableReceiptLog.IReceiptLog.GetTransactionHash()) 135 | 136 | receiptLogsPlugins := watcher.ReceiptLogPlugins 137 | for i := 0; i < len(receiptLogsPlugins); i++ { 138 | p := receiptLogsPlugins[i] 139 | 140 | if p.NeedReceiptLog(removableReceiptLog) { 141 | logrus.Debugln("receipt log accepted") 142 | p.Accept(removableReceiptLog) 143 | } else { 144 | logrus.Debugln("receipt log not accepted") 145 | } 146 | } 147 | } 148 | 149 | watcher.wg.Done() 150 | }() 151 | 152 | for { 153 | latestBlockNum, err := watcher.rpc.GetCurrentBlockNum() 154 | if err != nil { 155 | return err 156 | } 157 | 158 | if startBlockNum <= 0 { 159 | startBlockNum = latestBlockNum 160 | } 161 | 162 | noNewBlockForSync := watcher.LatestSyncedBlockNum() >= latestBlockNum 163 | logrus.Debugln("watcher.LatestSyncedBlockNum()", watcher.LatestSyncedBlockNum()) 164 | 165 | if noNewBlockForSync { 166 | logrus.Debugf("no new block to sync, sleep for %d secs", watcher.sleepSecondsForNewBlock) 167 | 168 | // sleep for 3 secs 169 | select { 170 | case <-watcher.Ctx.Done(): 171 | closeWatcher(watcher) 172 | return nil 173 | case <-time.After(time.Duration(watcher.sleepSecondsForNewBlock) * time.Second): 174 | continue 175 | } 176 | } 177 | 178 | for watcher.LatestSyncedBlockNum() < latestBlockNum { 179 | select { 180 | case <-watcher.Ctx.Done(): 181 | logrus.Info("watcher context down, closing channels to exit...") 182 | closeWatcher(watcher) 183 | logrus.Info("watcher done!") 184 | 185 | return nil 186 | default: 187 | var newBlockNumToSync uint64 188 | if watcher.LatestSyncedBlockNum() <= 0 { 189 | newBlockNumToSync = startBlockNum 190 | } else { 191 | newBlockNumToSync = watcher.LatestSyncedBlockNum() + 1 192 | } 193 | 194 | logrus.Debugln("newBlockNumToSync:", newBlockNumToSync) 195 | 196 | newBlock, err := watcher.rpc.GetBlockByNum(newBlockNumToSync) 197 | if err != nil { 198 | return err 199 | } 200 | 201 | if newBlock == nil { 202 | msg := fmt.Sprintf("GetBlockByNum(%d) returns nil block", newBlockNumToSync) 203 | return errors.New(msg) 204 | } 205 | 206 | if watcher.FoundFork(newBlock) { 207 | logrus.Infoln("found fork, popping") 208 | err = watcher.popBlocksUntilReachMainChain() 209 | } else { 210 | logrus.Debugln("adding new block:", newBlock.Number()) 211 | err = watcher.addNewBlock(structs.NewRemovableBlock(newBlock, false), latestBlockNum) 212 | } 213 | 214 | if err != nil { 215 | return err 216 | } 217 | } 218 | } 219 | } 220 | } 221 | 222 | func closeWatcher(w *AbstractWatcher) { 223 | close(w.NewBlockChan) 224 | close(w.NewTxAndReceiptChan) 225 | close(w.NewReceiptLogChan) 226 | 227 | w.wg.Wait() 228 | } 229 | 230 | func (watcher *AbstractWatcher) SetSleepSecondsForNewBlock(sec int) { 231 | watcher.sleepSecondsForNewBlock = sec 232 | } 233 | 234 | func (watcher *AbstractWatcher) LatestSyncedBlockNum() uint64 { 235 | watcher.lock.RLock() 236 | defer watcher.lock.RUnlock() 237 | 238 | if watcher.SyncedBlocks.Len() <= 0 { 239 | return 0 240 | } 241 | 242 | b := watcher.SyncedBlocks.Back().Value.(blockchain.Block) 243 | 244 | return b.Number() 245 | } 246 | 247 | // go thru plugins to check if this watcher need fetch receipt for tx 248 | // network load for fetching receipts per tx is heavy, 249 | // we use this method to make sure we only do the work we need 250 | func (watcher *AbstractWatcher) needReceipt(tx blockchain.Transaction) bool { 251 | plugins := watcher.TxReceiptPlugins 252 | 253 | for _, p := range plugins { 254 | if filterPlugin, ok := p.(plugin.TxReceiptPluginWithFilter); ok { 255 | if filterPlugin.NeedReceipt(tx) { 256 | return true 257 | } 258 | } else { 259 | // exist global tx-receipt listener 260 | return true 261 | } 262 | } 263 | 264 | return false 265 | } 266 | 267 | // return query map: contractAddress -> interested 1stTopics 268 | func (watcher *AbstractWatcher) getReceiptLogQueryMap() (queryMap map[string][]string) { 269 | queryMap = make(map[string][]string, 16) 270 | 271 | for _, p := range watcher.ReceiptLogPlugins { 272 | key := p.FromContract() 273 | 274 | if v, exist := queryMap[key]; exist { 275 | queryMap[key] = append(v, p.InterestedTopics()...) 276 | } else { 277 | queryMap[key] = p.InterestedTopics() 278 | } 279 | } 280 | 281 | return 282 | } 283 | 284 | func (watcher *AbstractWatcher) addNewBlock(block *structs.RemovableBlock, curHighestBlockNum uint64) error { 285 | watcher.lock.Lock() 286 | defer watcher.lock.Unlock() 287 | 288 | // get tx receipts in block, which is time consuming 289 | signals := make([]*SyncSignal, 0, len(block.GetTransactions())) 290 | for i := 0; i < len(block.GetTransactions()); i++ { 291 | tx := block.GetTransactions()[i] 292 | 293 | if !watcher.needReceipt(tx) { 294 | //logrus.Debugf("no need to get receipt of tx(%s), skipped", tx.GetHash()) 295 | continue 296 | } else { 297 | logrus.Debugf("needReceipt of tx: %s in block: %d", tx.GetHash(), block.Number()) 298 | } 299 | 300 | syncSigName := fmt.Sprintf("B:%d T:%s", block.Number(), tx.GetHash()) 301 | 302 | sig := newSyncSignal(syncSigName) 303 | signals = append(signals, sig) 304 | 305 | go func() { 306 | txReceipt, err := watcher.rpc.GetTransactionReceipt(tx.GetHash()) 307 | 308 | if err != nil { 309 | fmt.Printf("GetTransactionReceipt fail, err: %s", err) 310 | sig.err = err 311 | 312 | // one fails all 313 | return 314 | } 315 | 316 | sig.WaitPermission() 317 | 318 | sig.rst = structs.NewRemovableTxAndReceipt(tx, txReceipt, false, block.Timestamp()) 319 | 320 | sig.Done() 321 | }() 322 | } 323 | 324 | for i := 0; i < len(signals); i++ { 325 | sig := signals[i] 326 | sig.Permit() 327 | sig.WaitDone() 328 | 329 | if sig.err != nil { 330 | return sig.err 331 | } 332 | } 333 | 334 | for i := 0; i < len(signals); i++ { 335 | watcher.SyncedTxAndReceipts.PushBack(signals[i].rst.TxAndReceipt) 336 | watcher.NewTxAndReceiptChan <- signals[i].rst 337 | } 338 | 339 | queryMap := watcher.getReceiptLogQueryMap() 340 | logrus.Debugln("getReceiptLogQueryMap:", queryMap) 341 | 342 | bigStep := uint64(50) 343 | if curHighestBlockNum-block.Number() > bigStep { 344 | // only do request with bigStep 345 | if watcher.ReceiptCatchUpFromBlock == 0 { 346 | // init 347 | logrus.Debugf("bigStep, init to %d", block.Number()) 348 | watcher.ReceiptCatchUpFromBlock = block.Number() 349 | } else { 350 | // check if we need do requests 351 | if (block.Number() - watcher.ReceiptCatchUpFromBlock + 1) == bigStep { 352 | fromBlock := watcher.ReceiptCatchUpFromBlock 353 | toBlock := block.Number() 354 | 355 | logrus.Debugf("bigStep, doing request, range: %d -> %d (minus: %d)", fromBlock, toBlock, block.Number()-watcher.ReceiptCatchUpFromBlock) 356 | 357 | for k, v := range queryMap { 358 | err := watcher.fetchReceiptLogs(false, block, fromBlock, toBlock, k, v) 359 | if err != nil { 360 | return err 361 | } 362 | } 363 | 364 | // update catch up block 365 | watcher.ReceiptCatchUpFromBlock = block.Number() + 1 366 | } else { 367 | logrus.Debugf("bigStep, holding %d blocks: %d -> %d", block.Number()-watcher.ReceiptCatchUpFromBlock+1, watcher.ReceiptCatchUpFromBlock, block.Number()) 368 | } 369 | } 370 | } else { 371 | // reset 372 | if watcher.ReceiptCatchUpFromBlock != 0 { 373 | logrus.Debugf("exit bigStep mode, ReceiptCatchUpFromBlock: %d, curBlock: %d, gap: %d", watcher.ReceiptCatchUpFromBlock, block.Number(), curHighestBlockNum-block.Number()) 374 | watcher.ReceiptCatchUpFromBlock = 0 375 | } 376 | 377 | for k, v := range queryMap { 378 | err := watcher.fetchReceiptLogs(block.IsRemoved, block, block.Number(), block.Number(), k, v) 379 | if err != nil { 380 | return err 381 | } 382 | } 383 | } 384 | 385 | // clean synced data 386 | for watcher.SyncedBlocks.Len() >= watcher.MaxSyncedBlockToKeep { 387 | // clean block 388 | b := watcher.SyncedBlocks.Remove(watcher.SyncedBlocks.Front()).(blockchain.Block) 389 | 390 | // clean txAndReceipt 391 | for watcher.SyncedTxAndReceipts.Front() != nil { 392 | head := watcher.SyncedTxAndReceipts.Front() 393 | 394 | if head.Value.(*structs.TxAndReceipt).Tx.GetBlockNumber() <= b.Number() { 395 | watcher.SyncedTxAndReceipts.Remove(head) 396 | } else { 397 | break 398 | } 399 | } 400 | } 401 | 402 | // block 403 | watcher.SyncedBlocks.PushBack(block.Block) 404 | watcher.NewBlockChan <- block 405 | 406 | return nil 407 | } 408 | 409 | func (watcher *AbstractWatcher) fetchReceiptLogs(isRemoved bool, block blockchain.Block, from, to uint64, address string, topics []string) error { 410 | 411 | receiptLogs, err := watcher.rpc.GetLogs(from, to, address, topics) 412 | if err != nil { 413 | return err 414 | } 415 | 416 | for i := 0; i < len(receiptLogs); i++ { 417 | log := receiptLogs[i] 418 | logrus.Debugln("insert into chan: ", log.GetTransactionHash()) 419 | 420 | watcher.NewReceiptLogChan <- &structs.RemovableReceiptLog{ 421 | IReceiptLog: log, 422 | IsRemoved: isRemoved, 423 | } 424 | } 425 | 426 | return nil 427 | } 428 | 429 | type SyncSignal struct { 430 | name string 431 | permission chan bool 432 | jobDone chan bool 433 | rst *structs.RemovableTxAndReceipt 434 | err error 435 | } 436 | 437 | func newSyncSignal(name string) *SyncSignal { 438 | return &SyncSignal{ 439 | name: name, 440 | permission: make(chan bool, 1), 441 | jobDone: make(chan bool, 1), 442 | } 443 | } 444 | 445 | func (s *SyncSignal) Permit() { 446 | s.permission <- true 447 | } 448 | 449 | func (s *SyncSignal) WaitPermission() { 450 | <-s.permission 451 | } 452 | 453 | func (s *SyncSignal) Done() { 454 | s.jobDone <- true 455 | } 456 | 457 | func (s *SyncSignal) WaitDone() { 458 | <-s.jobDone 459 | } 460 | 461 | func (watcher *AbstractWatcher) popBlocksUntilReachMainChain() error { 462 | watcher.lock.Lock() 463 | defer watcher.lock.Unlock() 464 | 465 | for { 466 | if watcher.SyncedBlocks.Back() == nil { 467 | return nil 468 | } 469 | 470 | // NOTE: instead of watcher.LatestSyncedBlockNum() cuz it has lock 471 | lastSyncedBlock := watcher.SyncedBlocks.Back().Value.(blockchain.Block) 472 | block, err := watcher.rpc.GetBlockByNum(lastSyncedBlock.Number()) 473 | if err != nil { 474 | return err 475 | } 476 | 477 | if block.Hash() != lastSyncedBlock.Hash() { 478 | fmt.Println("removing tail block:", watcher.SyncedBlocks.Back()) 479 | removedBlock := watcher.SyncedBlocks.Remove(watcher.SyncedBlocks.Back()).(blockchain.Block) 480 | 481 | for watcher.SyncedTxAndReceipts.Back() != nil { 482 | 483 | tail := watcher.SyncedTxAndReceipts.Back() 484 | 485 | if tail.Value.(*structs.TxAndReceipt).Tx.GetBlockNumber() >= removedBlock.Number() { 486 | fmt.Printf("removing tail txAndReceipt: %+v", tail.Value) 487 | tuple := watcher.SyncedTxAndReceipts.Remove(tail).(*structs.TxAndReceipt) 488 | 489 | watcher.NewTxAndReceiptChan <- structs.NewRemovableTxAndReceipt(tuple.Tx, tuple.Receipt, true, block.Timestamp()) 490 | } else { 491 | fmt.Printf("all txAndReceipts removed for block: %+v", removedBlock) 492 | break 493 | } 494 | } 495 | 496 | watcher.NewBlockChan <- structs.NewRemovableBlock(removedBlock, true) 497 | } else { 498 | return nil 499 | } 500 | } 501 | } 502 | 503 | func (watcher *AbstractWatcher) FoundFork(newBlock blockchain.Block) bool { 504 | for e := watcher.SyncedBlocks.Back(); e != nil; e = e.Prev() { 505 | syncedBlock := e.Value.(blockchain.Block) 506 | 507 | //if syncedBlock == nil { 508 | // logrus.Warnln("error, syncedBlock is nil") 509 | //} 510 | //logrus.Debugf("syncedBlock: %+v", syncedBlock) 511 | 512 | //if newBlock == nil { 513 | // logrus.Warnln("error, newBlock is nil") 514 | //} 515 | //logrus.Debugf("newBlock: %+v", newBlock) 516 | 517 | if syncedBlock.Number()+1 == newBlock.Number() { 518 | notMatch := (syncedBlock).Hash() != newBlock.ParentHash() 519 | 520 | if notMatch { 521 | fmt.Printf("found fork, new block(%d): %s, new block's parent: %s, parent we synced: %s", 522 | newBlock.Number(), newBlock.Hash(), newBlock.ParentHash(), syncedBlock.Hash()) 523 | 524 | return true 525 | } 526 | } 527 | } 528 | 529 | return false 530 | } 531 | --------------------------------------------------------------------------------