├── userful.sh ├── README.md ├── lib └── common.go ├── craw.py ├── summary.py ├── synBlockTime.go ├── getTxCountGroupByAccount.go ├── getBalance.go ├── getBalanceMulti.go ├── getAccount.go ├── getTxTimelineGroupByAccount.go └── getTxByTime.go /userful.sh: -------------------------------------------------------------------------------- 1 | curl -X POST --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x51cd215bB9Aa24484870b91a5E61B8F6aB693A0f","latest"],"id":1}' -H "Content-type: application/json;charset=UTF-8" localhost:8545 2 | awk 'BEGIN{FS=" "}{keycount[$1]++;valsum[$1]+=$2}END{for(i in keycount){printf("%s %d\n", i,valsum[i]);}}' accountTxCnt 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 基于geth对以太坊区块链的解析 2 | * 本地[下载安装geth](https://github.com/ethereum/go-ethereum) 3 | * 本地设置环境变量$GOPATH，可以指定为任意目录 4 | * 下载依赖的开源库，依次执行以下命令 5 | * `go get github.com/ethereum/go-ethereum` 6 | * `go get github.com/deckarep/golang-set` 7 | * 执行`geth --rpc`启动geth的http服务器 8 | * 可选操作：执行`go run synBlockTime.go`，将本地所有block的编号和时间写入blocktime文件。当我们使用后面功能时，脚本会从blocktime中读取出生成于[timeFrom]到[timeTo]间的所有block编号，再根据block编号获取block详细信息。因此，如果不执行该操作，blocktime文件可能未记录最新block，使用后面功能时就无法获取最新block的详细信息 9 | * 功能 10 | * 获取一段时间内所有交易账户：`go run getAccount.go [timeFrom] [timeTo]`, 比如`go run getAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00`。结果会存到accounts目录下，文件名为[timeFrom]-[timeTo] 11 | * 获取一段时间内所有交易账户及其余额：`go run getBalance.go [timeFrom] [timeTo]`, 比如`go run getBalance.go 2018-01-01-00-00-00 2018-02-01-00-00-00`。结果会存到balance目录下，文件名为[timeFrom]-[timeTo]。注意：执行此命令前需要先执行`go run getAccount.go [timeFrom] [timeTo]` 12 | * 获取一段时间内所有交易明细：`go run getTxByTime.go [timeFrom] [timeTo]`, 比如`go run getTxByTime.go 2018-01-01-00-00-00 2018-02-01-00-00-00`。结果会存到tx目录下，文件名为[timeFrom]-[timeTo]。同时还会产生另外两个文件[timeFrom]-[timeTo]-from-sort和[timeFrom]-[timeTo]-to-sort，分别存放以交易卖出账户和买进账户排序后的结果 13 | * 获取一段时间内每个交易账户的交易明细：`go run getTxTimelineGroupByAccount.go [timeFrom] [timeTo]`, 比如`go run getTxTimelineGroupByAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00`。结果会存到tx目录下，文件名为[timeFrom]-[timeTo]-timeline。注意：执行此命令前需要先执行`go run getTxByTime.go [timeFrom] [timeTo]` 14 | * 获取一段时间内每个交易账户的交易总数（含进出总数和进出交易量）：`go run getTxCountGroupByAccount.go [timeFrom] [timeTo]`, 比如`go run getTxCountGroupByAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00`。结果会存到tx目录下，文件名为[timeFrom]-[timeTo]-count。注意：执行此命令前需要先执行`go run getTxByTime.go [timeFrom] [timeTo]` 15 | 16 | 17 | # wiki 18 | https://github.com/star2478/go-ethereum-blockchain-parser/wiki/%E4%BB%A5%E5%A4%AA%E5%9D%8AGo-Ethereum(Geth)%E8%83%BD%E5%81%9A%E4%BB%80%E4%B9%88 19 | -------------------------------------------------------------------------------- /lib/common.go: -------------------------------------------------------------------------------- 1 | package lib 2 | import ( 3 | "log" 4 | "os" 5 | "os/exec" 6 | "io" 7 | "strings" 8 | "strconv" 9 | "bufio" 10 | ) 11 | 12 | func GetBlockNumberByTime(timeBegin string, timeEnd string) (int,int) { 13 | log.Print("begin GetBlockNumberByTime, timeBegin=", timeBegin, " timeEnd=", timeEnd) 14 | fileName := "blocktime" 15 | fd, err := os.Open(fileName) 16 | if err != nil { 17 | log.Fatal("file=",fileName," open fail!!", err); 18 | } 19 | defer fd.Close() 20 | blockNumberBegin := -1 21 | blockNumberEnd := -1 22 | lastScanBlockNumber := -1 23 | blockTime := "" 24 | readBuf := bufio.NewReader(fd) 25 | for { 26 | line, err := readBuf.ReadString('\n') 27 | if err == io.EOF { 28 | break 29 | } 30 | line = strings.TrimSpace(line) 31 | arrLine := strings.Split(line, " ") 32 | blockTime = arrLine[0] 33 | blockNumber,_ := strconv.Atoi(arrLine[1]) 34 | if blockNumberBegin == -1 && timeBegin <= blockTime { 35 | blockNumberBegin = blockNumber 36 | } 37 | if timeEnd < blockTime { 38 | blockNumberEnd = lastScanBlockNumber 39 | break 40 | } 41 | lastScanBlockNumber = blockNumber 42 | } 43 | if blockNumberBegin != -1 && timeEnd >= blockTime { 44 | blockNumberEnd = lastScanBlockNumber 45 | } 46 | if blockNumberBegin == -1 || blockNumberEnd == -1 { 47 | log.Fatal("Cannot find blockNumber!!! timeBegin=", timeBegin, " timeEnd=", timeEnd, ", blockNumberBegin=", blockNumberBegin, " blockNumberEnd=", blockNumberEnd, ". You can try to run 'go run synBlockTime.go' to get current block time before this"); 48 | } 49 | log.Print("finish GetBlockNumberByTime, blockNumberBegin=", blockNumberBegin, " blockNumberEnd=",blockNumberEnd) 50 | return blockNumberBegin, blockNumberEnd 51 | } 52 | 53 | func PathExists(path string) (bool) { 54 | _, err := os.Stat(path) 55 | if err == nil { 56 | return true 57 | } 58 | return false 59 | } 60 | func GetAndCheckDir(dirname string) string { 61 | curDir, _ := os.Getwd() // 62 | newDir := curDir +"/" + dirname 63 | if PathExists(newDir) == false { 64 | err := os.Mkdir(newDir, os.ModePerm) // 65 | if err != nil { 66 | log.Fatal(newDir, " create fail! ", err) 67 | //return nil 68 | } 69 | } 70 | return newDir 71 | } 72 | func ExecCmd(command string, isFatalExit bool) { 73 | 74 | log.Print("begin to run command=" + command); 75 | cmd := exec.Command("/bin/bash", "-c", command) 76 | err := cmd.Run() 77 | if err != nil { 78 | if isFatalExit == true { 79 | log.Fatal("Fail!!!cmd="+command+" run fail!", err) 80 | } else { 81 | log.Print("Fail!!!cmd="+command+" run fail!", err) 82 | 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /craw.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.7 2 | 3 | from bs4 import BeautifulSoup 4 | import urllib2 5 | import MySQLdb 6 | import time 7 | 8 | def spider(url): 9 | request = urllib2.Request(url) 10 | request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36') 11 | opener = urllib2.build_opener() 12 | #opener.addheaders.append(('Cookie','')) 13 | opener.addheaders.append(('Cookie','__cfduid=dff3d50d7a7fba6a53dd29b711e7d93521516259067; ASP.NET_SessionId=gde0u3rf5lxk53qfjmokhfjb; _ga=GA1.2.1085595957.1516259070; _gid=GA1.2.33895899.1516616011; __cflb=4215917907; cf_clearance=c1c1598e3452b4f9f37df8cd49e0e9cdac7b4f17-1516671681-10800')) 14 | f = opener.open(request) 15 | soup = BeautifulSoup(f.read().decode('utf-8'), "html5lib") 16 | return soup 17 | 18 | def getAccount(db, cursor, addr): 19 | sql = "SELECT * FROM Accounts WHERE Address='%s'" % addr 20 | cursor.execute(sql) 21 | results = cursor.fetchall() 22 | if len(results)==0: 23 | fromSoup = spider("https://etherscan.io/token/EOS?a=%s" % addr) 24 | div = fromSoup.find(id='ContentPlaceHolder1_tr_tokenValue') 25 | if div==None: 26 | return 1 27 | td = div.findAll('td') 28 | if td==None or len(td)!=2: 29 | return 1 30 | value = td[1].text 31 | if value==None or value.strip()=='': 32 | return 1 33 | value = value.split(' ')[0] 34 | value = value.replace('$', '') 35 | value = value.replace(',', '') 36 | value = float(value) 37 | print addr, value 38 | 39 | name = fromSoup.find(id='ContentPlaceHolder1_divSummary') 40 | if name!=None: 41 | name = name.find('font') 42 | if name!=None: 43 | name = name.text 44 | if name!=None and name.strip()!='': 45 | name = name.replace('Filtered By Individual Token Holder', '') 46 | name = name.strip() 47 | if name==None or name.strip()=='': 48 | name = '' 49 | isql = "INSERT INTO Accounts(`Address`, `Name`, `Balance`) VALUE('%s', '%s', '%.2f')" % (addr, name, value) 50 | cursor.execute(isql) 51 | db.commit() 52 | time.sleep(0.2) 53 | return 0 54 | 55 | 56 | db = MySQLdb.connect("127.0.0.1","eth","eth","eth" ) 57 | 58 | while True: 59 | try: 60 | soup = spider('https://etherscan.io/token/generic-tokentxns2?contractAddress=0x86fa049857e0209aa7d9e616f7eb3b3b78ecfdb0&a=&mode=') 61 | table = soup.find(attrs={'class':'table'}) 62 | db.ping(True) 63 | cursor = db.cursor() 64 | for row in table.findAll("tr"): 65 | cells = row.findAll("td") 66 | if len(cells) == 7: 67 | txhash = cells[0].find('a').text 68 | if txhash==None or txhash.strip()=='': 69 | continue 70 | age = cells[1].find('span').get('title') 71 | if age==None or age.strip()=='': 72 | continue 73 | fromc = cells[2].find('a').text 74 | if fromc==None or fromc.strip()=='': 75 | continue 76 | toc = cells[4].find('a').text 77 | if toc==None or toc.strip()=='': 78 | continue 79 | quantity = cells[5].text 80 | if quantity==None or quantity.strip()=='': 81 | continue 82 | quantity = quantity.replace(',', '') 83 | quantity = float(quantity) 84 | sql = "INSERT INTO Transactions(`TxHash`, `Age`, `FromAcc`, `ToAcc`, `Quantity`) VALUE('%s', '%s', '%s', '%s', '%.2f')" % (txhash, age, fromc, toc, quantity) 85 | db.ping(True) 86 | cursor = db.cursor() 87 | try: 88 | cursor.execute(sql) 89 | except: 90 | print 'already exists' 91 | db.commit() 92 | time.sleep(0.2) 93 | getAccount(db, cursor, fromc) 94 | getAccount(db, cursor, toc) 95 | except Exception, e: 96 | print e 97 | db.close() 98 | 99 | -------------------------------------------------------------------------------- /summary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2.7 2 | 3 | from bs4 import BeautifulSoup 4 | import urllib2 5 | import MySQLdb 6 | import time 7 | import datetime 8 | 9 | def spider(url): 10 | request = urllib2.Request(url) 11 | request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36') 12 | opener = urllib2.build_opener() 13 | opener.addheaders.append(('Cookie','__cfduid=dff3d50d7a7fba6a53dd29b711e7d93521516259067; ASP.NET_SessionId=gde0u3rf5lxk53qfjmokhfjb; _ga=GA1.2.1085595957.1516259070; _gid=GA1.2.33895899.1516616011; __cflb=4215917907; cf_clearance=c1c1598e3452b4f9f37df8cd49e0e9cdac7b4f17-1516671681-10800')) 14 | f = opener.open(request) 15 | soup = BeautifulSoup(f.read().decode('utf-8'), "html5lib") 16 | return soup 17 | 18 | 19 | def getTransaction(addr): 20 | summary = {} 21 | transfer = {} 22 | 23 | flag = True 24 | page = 0 25 | while (flag): 26 | page = page + 1 27 | print page 28 | url = "https://etherscan.io/token/generic-tokentxns2?contractAddress=0x86fa049857e0209aa7d9e616f7eb3b3b78ecfdb0&mode=&a=%s&p=%d" % (addr, page) 29 | fromSoup = spider(url) 30 | div = fromSoup.find(id='maindiv') 31 | if div==None: 32 | return 1 33 | tb = div.find('table') 34 | tb = tb.find('tbody') 35 | tr = tb.findAll('tr') 36 | if tr==None or len(tr)==0: 37 | return 1 38 | for row in tr: 39 | cells = row.findAll('td') 40 | if len(cells)==7: 41 | age = cells[1].find('span').get('title') 42 | if age==None or age.strip()=='': 43 | continue 44 | age = age.split(' ')[0] 45 | date_time = datetime.datetime.strptime(age,'%b-%d-%Y') 46 | day = date_time.strftime('%Y-%m-%d') 47 | if day<'2018-01-01': 48 | flag = False 49 | break 50 | 51 | type = cells[3].find('span').text 52 | if type==None or type.strip()=='': 53 | continue 54 | 55 | targetCell = cells[2] 56 | if "IN" in type: 57 | type = "in" 58 | else: 59 | type = "out" 60 | targetCell = cells[4] 61 | target = targetCell.find('a').text 62 | if target==None or target.strip()=='': 63 | continue 64 | quantity = cells[5].text 65 | if quantity==None or quantity.strip()=='': 66 | continue 67 | quantity = quantity.replace(',', '') 68 | quantity = float(quantity) 69 | 70 | if not summary.has_key(day): 71 | summary[day] = {} 72 | if not transfer.has_key(type): 73 | transfer[type] = {} 74 | 75 | if summary[day].has_key(type): 76 | summary[day][type] = summary[day][type] + quantity 77 | else: 78 | summary[day][type] = quantity 79 | 80 | if transfer[type].has_key(target): 81 | transfer[type][target] = transfer[type][target] + quantity 82 | else: 83 | transfer[type][target] = quantity 84 | 85 | time.sleep(0.5) 86 | 87 | db = MySQLdb.connect("127.0.0.1","eth","eth","eth" ) 88 | cursor = db.cursor() 89 | for d in summary: 90 | inAmount = 0 91 | outAmount = 0 92 | if "in" in summary[d]: 93 | inAmount = summary[d]["in"] 94 | if "out" in summary[d]: 95 | outAmount = summary[d]["out"] 96 | sql = "INSERT INTO Summary(`Account`,`Day`,`InAmount`,`OutAmount`) VALUE('%s','%s','%.2f','%.2f')" %(addr, d, inAmount, outAmount) 97 | print sql 98 | cursor.execute(sql) 99 | db.commit() 100 | 101 | print "" 102 | 103 | for type in transfer: 104 | for target in transfer[type]: 105 | sql = "INSERT INTO Transfer(`Account`,`Type`,`Target`,`Number`) VALUE('%s','%s','%s','%.2f')" %(addr, type, target, transfer[type][target]) 106 | print sql 107 | cursor.execute(sql) 108 | db.commit() 109 | 110 | sql = "UPDATE Accounts set flag='y' where Address='%s'" %addr 111 | cursor.execute(sql) 112 | db.commit() 113 | db.close() 114 | return 0 115 | 116 | 117 | while True: 118 | db = MySQLdb.connect("127.0.0.1","eth","eth","eth" ) 119 | cursor = db.cursor() 120 | sql = "SELECT * FROM Accounts WHERE flag='n' order by Balance desc limit 1" 121 | cursor.execute(sql) 122 | results = cursor.fetchall() 123 | if len(results)!=0: 124 | acc = results[0][0] 125 | print "Begin to stat account %s:" % acc 126 | getTransaction(acc) 127 | print "" 128 | 129 | db.close() 130 | -------------------------------------------------------------------------------- /synBlockTime.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | //"sort" 6 | "fmt" 7 | "log" 8 | "strconv" 9 | "strings" 10 | "time" 11 | "runtime" 12 | "encoding/json" 13 | "os" 14 | "io" 15 | "math" 16 | "os/exec" 17 | "bufio" 18 | "github.com/ethereum/go-ethereum/common" 19 | "github.com/ethereum/go-ethereum/common/hexutil" 20 | ) 21 | 22 | var MAX_NUM_SHARE = 10 23 | var MAX_NUM_RETRY = 3 24 | var APPEND_FILE_ACCOUNT_MAX_NUM = 100 25 | var APPEND_FILE_LOOP_MAX_NUM = 1000 26 | var APPEND_FILE_ACCOUNT_TX_MAX_NUM = 10000 27 | var txMap = map[string]string{} 28 | var tenToAny map[int]string = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29:"t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62:"M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"} 29 | 30 | type ResultOfBlockNumber struct { 31 | Jsonrpc string 32 | Id int 33 | Result string 34 | } 35 | type ResultOfGetBlockByNumber struct { 36 | Jsonrpc string 37 | Id int 38 | Result Block 39 | } 40 | type RPCTransaction struct { 41 | BlockHash common.Hash `json:"blockHash"` 42 | BlockNumber *hexutil.Big `json:"blockNumber"` 43 | From *common.Address `json:"from"` 44 | Gas hexutil.Uint64 `json:"gas"` 45 | GasPrice *hexutil.Big `json:"gasPrice"` 46 | Hash common.Hash `json:"hash"` 47 | Input hexutil.Bytes `json:"input"` 48 | Nonce hexutil.Uint64 `json:"nonce"` 49 | To *common.Address `json:"to"` 50 | TransactionIndex hexutil.Uint `json:"transactionIndex"` 51 | Value *hexutil.Big `json:"value"` 52 | V *hexutil.Big `json:"v"` 53 | R *hexutil.Big `json:"r"` 54 | S *hexutil.Big `json:"s"` 55 | } 56 | 57 | type Block struct { 58 | UncleHashes []common.Hash `json:"uncles"` 59 | Hash common.Hash `json:"hash"` 60 | Timestamp string `json:"timestamp"` 61 | //Transactions []RPCTransaction `json:"transactions"` 62 | } 63 | 64 | func decimalToAny(num, n int) string { 65 | new_num_str := "" 66 | var remainder int 67 | var remainder_string string 68 | for num != 0 { 69 | remainder = num % n 70 | if 76 > remainder && remainder > 9 { 71 | remainder_string = tenToAny[remainder] 72 | } else { 73 | remainder_string = strconv.Itoa(remainder) 74 | } 75 | new_num_str = remainder_string + new_num_str 76 | num = num / n 77 | } 78 | return new_num_str 79 | } 80 | func findkey(in string) int { 81 | result := -1 82 | for k, v := range tenToAny { 83 | if in == v { 84 | result = k 85 | } 86 | } 87 | return result 88 | } 89 | func anyToDecimal(num string, n int) float64 { 90 | var new_num float64 91 | new_num = 0.0 92 | nNum := len(strings.Split(num, "")) - 1 93 | for _, value := range strings.Split(num, "") { 94 | tmp := float64(findkey(value)) 95 | if tmp != -1 { 96 | new_num = new_num + tmp*math.Pow(float64(n), float64(nNum)) 97 | nNum = nNum - 1 98 | } else { 99 | break 100 | } 101 | } 102 | return float64(new_num) 103 | } 104 | func getCurrentBlockNumber() int { 105 | command := "curl -X POST --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\": 84}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545" 106 | cmd := exec.Command("/bin/bash", "-c", command) 107 | var out bytes.Buffer 108 | cmd.Stdout = &out 109 | err := cmd.Run() 110 | if err != nil { 111 | log.Print("Fatal!!!cmd="+command+" run fail!", err) 112 | } 113 | stb := &ResultOfBlockNumber{} 114 | err = json.Unmarshal([]byte(out.String()), &stb) 115 | 116 | if err != nil { 117 | log.Fatal("umarshal fail!!! cmd=", command, " input=", out.String()," ", err) 118 | } 119 | hexBlockNumber := stb.Result 120 | hexBlockNumber = hexBlockNumber[2 : len(hexBlockNumber)] // get rid of 0x prefix 121 | return int(anyToDecimal(hexBlockNumber, 16)) 122 | } 123 | 124 | func getBlockTime(blockNumber int) (string,int) { 125 | blockNumberHex := "0x" + decimalToAny(blockNumber, 16); 126 | command := "curl -X POST --data '{\"jsonrpc\":\"3.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"" + blockNumberHex + "\", false],\"id\":1}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545"; 127 | 128 | cmd := exec.Command("/bin/bash", "-c", command) 129 | var out bytes.Buffer 130 | cmd.Stdout = &out 131 | err := cmd.Run() 132 | if err != nil { 133 | log.Print("Fatal!!!cmd="+command+" run fail!", err) 134 | } 135 | stb := &ResultOfGetBlockByNumber{} 136 | err = json.Unmarshal([]byte(out.String()), &stb) 137 | 138 | if err != nil { 139 | log.Fatal("umarshal fail!!! block=",blockNumber," cmd=", command, " input=", out.String()," ", err) 140 | } 141 | //fmt.Printf("----result=%s:%d\n", stb.Jsonrpc, stb.Id) 142 | blockInfo := stb.Result 143 | if len(blockInfo.Timestamp) <= 0 { 144 | return "0", blockNumber 145 | } 146 | timeLayout := "2006-01-02-15-04-05" 147 | timestamp := blockInfo.Timestamp 148 | timestamp = timestamp[2 : len(timestamp)] // get rid of 0x prefix 149 | txTime := time.Unix(int64(anyToDecimal(timestamp, 16)), 0).Format(timeLayout) 150 | return txTime, blockNumber 151 | } 152 | 153 | func main() { 154 | fmt.Println("synBlockTime begin=================="); 155 | timeBegin := time.Now().Unix() 156 | MULTICORE := runtime.NumCPU() 157 | runtime.GOMAXPROCS(MULTICORE) 158 | fileName := "blocktime" 159 | fd,err := os.OpenFile(fileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 160 | if err != nil { 161 | log.Fatal("file=", fileName," open fail!!", err); 162 | } 163 | defer fd.Close() 164 | lastBlockInFile := "" 165 | readBuf := bufio.NewReader(fd) 166 | for { 167 | line, errRead := readBuf.ReadString('\n') 168 | if errRead == io.EOF { 169 | break 170 | } 171 | lastBlockInFile = line 172 | } 173 | lastBlockInFile = strings.TrimSpace(lastBlockInFile) 174 | firstToFindBlockNumber := -1 175 | if len(lastBlockInFile) > 0 { 176 | arrBlockInfo := strings.Split(lastBlockInFile, " ") 177 | firstToFindBlockNumber,_ = strconv.Atoi(arrBlockInfo[1]) 178 | } 179 | 180 | curMaxBlockNumber := getCurrentBlockNumber() 181 | log.Print("begin to write result file, curMaxBlockNumber=", curMaxBlockNumber, " firstToFindBlockNumber=", firstToFindBlockNumber+1) 182 | loopCount := 0 183 | fileContent := bytes.Buffer{} 184 | for i := firstToFindBlockNumber+1; i <= curMaxBlockNumber; i++ { 185 | txTime, blockNumber := getBlockTime(i) 186 | 187 | fileContent.WriteString(txTime + " " + strconv.Itoa(blockNumber) + "\n") 188 | if loopCount % APPEND_FILE_LOOP_MAX_NUM == 0 { 189 | wBuf:=[]byte(fileContent.String()) 190 | fd.Write(wBuf) 191 | log.Print("write block time into file=", fileName, " loopCount=", loopCount) 192 | //log.Print("write block time into =", txTime, blockNumber) 193 | fileContent = bytes.Buffer{} 194 | } 195 | loopCount++ 196 | } 197 | if len(fileContent.String()) > 0 { 198 | wBuf:=[]byte(fileContent.String()) 199 | fd.Write(wBuf) 200 | log.Print("write block time into file=", fileName, " loopCount=", loopCount) 201 | } 202 | 203 | 204 | timeEnd := time.Now().Unix() 205 | log.Print("synBlockTime finish, cost=", (timeEnd - timeBegin), "s") 206 | } 207 | -------------------------------------------------------------------------------- /getTxCountGroupByAccount.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | //"sort" 6 | "fmt" 7 | "log" 8 | "strconv" 9 | "strings" 10 | "time" 11 | "runtime" 12 | //"encoding/json" 13 | "os" 14 | "io" 15 | "bufio" 16 | "./lib" 17 | //"./golang-set" 18 | // "github.com/ethereum/go-ethereum/common" 19 | // "github.com/ethereum/go-ethereum/common/hexutil" 20 | // "github.com/deckarep/golang-set" 21 | ) 22 | 23 | var MAX_NUM_SHARE = 10 24 | var MAX_NUM_RETRY = 3 25 | var APPEND_FILE_ACCOUNT_MAX_NUM = 100 26 | var APPEND_FILE_LOOP_MAX_NUM = 10000 27 | var APPEND_FILE_ACCOUNT_TX_MAX_NUM = 10000 28 | var txMap = map[string]string{} 29 | var txMapFrom = map[string]*Count{} 30 | var txMapTo = map[string]*Count{} 31 | 32 | type Count struct { 33 | TxCount int 34 | TxAmtTotal float64 35 | } 36 | func initMap(key string) { 37 | 38 | if _, ok := txMap[key]; !ok { 39 | txMap[key] = "" 40 | } 41 | if _, ok := txMapFrom[key]; !ok { 42 | txMapFrom[key] = &Count{0, 0} 43 | } 44 | if _, ok := txMapTo[key]; !ok { 45 | txMapTo[key] = &Count{0, 0} 46 | } 47 | } 48 | func incTxMap(tx map[string]*Count, key string, incCount int, incAmt float64) { 49 | initMap(key) 50 | // log.Print(key, " :", incCount," ", incAmt) 51 | tx[key].TxCount += incCount 52 | tx[key].TxAmtTotal += incAmt 53 | // log.Print(key, " ", tx[key].TxCount, " ", tx[key].TxAmtTotal, " ", incCount, " ", incAmt) 54 | } 55 | 56 | func main() { 57 | if len(os.Args) < 3 { 58 | log.Fatal("Param Invalid!!! go run getTxCountGroupByAccount.go [timeFrom] [timeTo], eg. go run getTxCountGroupByAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00") 59 | } 60 | fmt.Println("getTxCountGroupByAccount begin=================="); 61 | timeBegin := time.Now().Unix() 62 | MULTICORE := runtime.NumCPU() 63 | runtime.GOMAXPROCS(MULTICORE) 64 | timeFrom := os.Args[1] 65 | timeTo := os.Args[2] 66 | if timeFrom >= timeTo { 67 | log.Fatal("timeFrom=", timeFrom, " >= timeTo=", timeTo) 68 | } 69 | dir := lib.GetAndCheckDir("tx") 70 | fromFileName := dir + "/" + timeFrom + "-" + timeTo + "-from-sort" 71 | toFileName := dir + "/" + timeFrom + "-" + timeTo + "-to-sort" 72 | desFileName := dir + "/" + timeFrom + "-" + timeTo + "-count" 73 | fd1, err1 := os.Open(fromFileName) 74 | if err1 != nil { 75 | log.Fatal("file=",fromFileName," open fail!! First of all, you must run 'go run getTxByTime.go ", timeFrom, " ", timeTo, "' to get transactions! err=" , err1); 76 | } 77 | fd2, err2 := os.Open(toFileName) 78 | if err2 != nil { 79 | log.Fatal("file=",toFileName," open fail!! First of all, you must run 'go run getTxByTime.go ", timeFrom, " ", timeTo, "' to get transactions! err=" , err1); 80 | } 81 | desFd,err3 := os.OpenFile(desFileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 82 | if err3 != nil { 83 | log.Fatal("file=",desFileName," open fail!!", err3); 84 | } 85 | fromLineNum := 0 86 | toLineNum := 0 87 | readBufFrom := bufio.NewReader(fd1) 88 | readBufTo := bufio.NewReader(fd2) 89 | 90 | from := "" 91 | to := "" 92 | //txTime := "" 93 | //txType := "" 94 | //timelineString := "" 95 | var arrFrom [] string 96 | var arrTo [] string 97 | loopCount := 0 98 | 99 | lineFrom, errReadFrom := readBufFrom.ReadString('\n') 100 | if errReadFrom != io.EOF { 101 | fromLineNum++ 102 | lineFrom = strings.TrimSpace(lineFrom) 103 | arrFrom = strings.Split(lineFrom, "\t") 104 | from = arrFrom[2] 105 | } 106 | lineTo, errReadTo := readBufTo.ReadString('\n') 107 | if errReadTo != io.EOF { 108 | toLineNum++ 109 | lineTo = strings.TrimSpace(lineTo) 110 | arrTo = strings.Split(lineTo, "\t") 111 | to = arrTo[3] 112 | } 113 | 114 | 115 | for { 116 | if errReadFrom == io.EOF || errReadTo == io.EOF { 117 | break 118 | } 119 | if from == to { 120 | tmp := from 121 | for { 122 | amtFrom,_ := strconv.ParseFloat(arrFrom[4],64) 123 | incTxMap(txMapFrom, from, 1, amtFrom) 124 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 125 | fromLineNum++ 126 | if errReadFrom == io.EOF { 127 | break 128 | } 129 | lineFrom = strings.TrimSpace(lineFrom) 130 | arrFrom = strings.Split(lineFrom, "\t") 131 | from = arrFrom[2] 132 | if from > tmp { 133 | break 134 | } 135 | } 136 | for { 137 | amtTo,_ := strconv.ParseFloat(arrTo[4],64) 138 | incTxMap(txMapTo, to, 1, float64(amtTo)) 139 | lineTo, errReadTo = readBufTo.ReadString('\n') 140 | toLineNum++ 141 | if errReadTo == io.EOF { 142 | break 143 | } 144 | lineTo = strings.TrimSpace(lineTo) 145 | arrTo = strings.Split(lineTo, "\t") 146 | to = arrTo[3] 147 | if to > tmp { 148 | break 149 | } 150 | } 151 | } else if from < to { 152 | amtFrom,_ := strconv.ParseFloat(arrFrom[4],64) 153 | incTxMap(txMapFrom, from, 1, amtFrom) 154 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 155 | fromLineNum++ 156 | if errReadFrom == io.EOF { 157 | break 158 | } 159 | lineFrom = strings.TrimSpace(lineFrom) 160 | arrFrom = strings.Split(lineFrom, "\t") 161 | from = arrFrom[2] 162 | } else { 163 | amtTo,_ := strconv.ParseFloat(arrTo[4],64) 164 | incTxMap(txMapTo, to, 1, amtTo) 165 | lineTo, errReadTo = readBufTo.ReadString('\n') 166 | toLineNum++ 167 | if errReadTo == io.EOF { 168 | break 169 | } 170 | lineTo = strings.TrimSpace(lineTo) 171 | arrTo = strings.Split(lineTo, "\t") 172 | to = arrTo[3] 173 | } 174 | loopCount++ 175 | if loopCount % APPEND_FILE_LOOP_MAX_NUM == 0 { 176 | log.Print("loopCount=", loopCount) 177 | } 178 | } 179 | log.Print("==================finish loop================fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 180 | 181 | if errReadFrom != io.EOF && errReadTo == io.EOF { 182 | for { 183 | amtFrom,_ := strconv.ParseFloat(arrFrom[4],64) 184 | incTxMap(txMapFrom, from, 1, amtFrom) 185 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 186 | fromLineNum++ 187 | if errReadFrom == io.EOF { 188 | break 189 | } 190 | lineFrom = strings.TrimSpace(lineFrom) 191 | arrFrom = strings.Split(lineFrom, "\t") 192 | from = arrFrom[2] 193 | } 194 | } 195 | if errReadFrom == io.EOF && errReadTo != io.EOF { 196 | for { 197 | amtTo,_ := strconv.ParseFloat(arrTo[4],64) 198 | incTxMap(txMapTo, to, 1, amtTo) 199 | lineTo, errReadTo = readBufTo.ReadString('\n') 200 | toLineNum++ 201 | if errReadTo == io.EOF { 202 | break 203 | } 204 | lineTo = strings.TrimSpace(lineTo) 205 | arrTo = strings.Split(lineTo, "\t") 206 | to = arrTo[3] 207 | } 208 | } 209 | 210 | log.Print("begin to write result file") 211 | loopCount = 0 212 | fileContent := bytes.Buffer{} 213 | for account,_ := range txMap { 214 | countFrom := txMapFrom[account] 215 | countTo := txMapTo[account] 216 | countTotal := countFrom.TxCount + countTo.TxCount 217 | amtTotal := countFrom.TxAmtTotal + countTo.TxAmtTotal 218 | fileContent.WriteString(account) 219 | fileContent.WriteString(" " + strconv.Itoa(countTotal) + " " + strconv.Itoa(countFrom.TxCount) + " " + strconv.Itoa(countTo.TxCount)) 220 | fileContent.WriteString(" " + strconv.FormatFloat(amtTotal, 'f', -1, 64) + " " + strconv.FormatFloat(countFrom.TxAmtTotal, 'f', -1, 64) + " " + strconv.FormatFloat(countTo.TxAmtTotal, 'f', -1, 64) + "\n") 221 | if loopCount % APPEND_FILE_LOOP_MAX_NUM == 0 { 222 | wBuf:=[]byte(fileContent.String()) 223 | desFd.Write(wBuf) 224 | log.Print("write account's tx into file=", desFileName, " loopCount=", loopCount) 225 | fileContent = bytes.Buffer{} 226 | } 227 | loopCount++ 228 | } 229 | if len(fileContent.String()) > 0 { 230 | wBuf:=[]byte(fileContent.String()) 231 | desFd.Write(wBuf) 232 | log.Print("write account's tx into file=", desFileName, " loopCount=", loopCount) 233 | } 234 | 235 | fd1.Close() 236 | fd2.Close() 237 | desFd.Close() 238 | 239 | timeEnd := time.Now().Unix() 240 | log.Print("getTxCountGroupByAccount finish, cost=", (timeEnd - timeBegin), "s") 241 | } 242 | -------------------------------------------------------------------------------- /getBalance.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | "os/exec" 10 | "math" 11 | "time" 12 | "runtime" 13 | "encoding/json" 14 | "os" 15 | "io" 16 | "bufio" 17 | "./lib" 18 | // "./go-ethereum/common" 19 | // "./go-ethereum/common/hexutil" 20 | //"./golang-set" 21 | "github.com/ethereum/go-ethereum/common" 22 | "github.com/ethereum/go-ethereum/common/hexutil" 23 | // "github.com/deckarep/golang-set" 24 | ) 25 | 26 | var MAX_NUM_SHARE = 10 27 | var MAX_NUM_RETRY = 3 28 | var APPEND_FILE_NUM = 100 29 | var c = make(chan string, 100) 30 | 31 | type ResultOfGetBalance struct { 32 | Jsonrpc string 33 | Id int 34 | Result string//hexutil.Uint64 35 | } 36 | type RPCTransaction struct { 37 | BlockHash common.Hash `json:"blockHash"` 38 | BlockNumber *hexutil.Big `json:"blockNumber"` 39 | From *common.Address `json:"from"` 40 | Gas hexutil.Uint64 `json:"gas"` 41 | GasPrice *hexutil.Big `json:"gasPrice"` 42 | Hash common.Hash `json:"hash"` 43 | Input hexutil.Bytes `json:"input"` 44 | Nonce hexutil.Uint64 `json:"nonce"` 45 | To *common.Address `json:"to"` 46 | TransactionIndex hexutil.Uint `json:"transactionIndex"` 47 | Value *hexutil.Big `json:"value"` 48 | V *hexutil.Big `json:"v"` 49 | R *hexutil.Big `json:"r"` 50 | S *hexutil.Big `json:"s"` 51 | } 52 | 53 | type Block struct { 54 | UncleHashes []common.Hash `json:"uncles"` 55 | Hash common.Hash `json:"hash"` 56 | Transactions []RPCTransaction `json:"transactions"` 57 | } 58 | 59 | var tenToAny map[int]string = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29: "t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62: "M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"} 60 | 61 | 62 | func decimalToAny(num, n int) string { 63 | new_num_str := "" 64 | var remainder int 65 | var remainder_string string 66 | for num != 0 { 67 | remainder = num % n 68 | if 76 > remainder && remainder > 9 { 69 | remainder_string = tenToAny[remainder] 70 | } else { 71 | remainder_string = strconv.Itoa(remainder) 72 | } 73 | new_num_str = remainder_string + new_num_str 74 | num = num / n 75 | } 76 | return new_num_str 77 | } 78 | 79 | func findkey(in string) int { 80 | result := -1 81 | for k, v := range tenToAny { 82 | if in == v { 83 | result = k 84 | } 85 | } 86 | return result 87 | } 88 | //187b2445cfb31e42b6a 89 | func anyToDecimal(num string, n int) float64 { 90 | var new_num float64 91 | new_num = 0.0 92 | nNum := len(strings.Split(num, "")) - 1 93 | for _, value := range strings.Split(num, "") { 94 | 95 | tmp := float64(findkey(value)) 96 | if tmp != -1 { 97 | new_num = new_num + tmp*math.Pow(float64(n), float64(nNum)) 98 | nNum = nNum - 1 99 | } else { 100 | break 101 | } 102 | } 103 | return float64(new_num) 104 | } 105 | 106 | func exec_shell(s string, accountNo string, retry int, fileContent *string) { 107 | cmd := exec.Command("/bin/bash", "-c", s) 108 | var out bytes.Buffer 109 | 110 | cmd.Stdout = &out 111 | err := cmd.Run() 112 | if err != nil { 113 | //log.Fatal("cmd="+s+" run fail!", err) 114 | retry-- 115 | log.Print("Fatal!!!retry=",retry," cmd="+s+" run fail!", err) 116 | if(retry > 0) { 117 | exec_shell(s, accountNo, retry, fileContent) 118 | } 119 | } 120 | 121 | stb := &ResultOfGetBalance{} 122 | //fmt.Printf("-----cmd=%s, result=%s\n", s, out.String()) 123 | err = json.Unmarshal([]byte(out.String()), &stb) 124 | 125 | if err != nil { 126 | retry--; 127 | log.Print("umarshal fail!!!retry=",retry," accountNo=",accountNo," cmd=", s, " input=", out.String()," ", err) 128 | if(retry > 0) { 129 | exec_shell(s, accountNo, retry, fileContent) 130 | } 131 | } 132 | //fmt.Printf("----result=%s:%d\n", stb.Jsonrpc, stb.Id) 133 | balance := stb.Result 134 | balance = balance[2 : len(balance)] // get rid of 0x prefix 135 | price := anyToDecimal(balance, 16)/1000000000000000000 136 | *fileContent += accountNo + " " + strconv.FormatFloat(price, 'f', -1, 64) + "\n"; 137 | //if balance <= 0 { 138 | //log.Print("block=", blockNumber, " balance len <= 0") 139 | // return 140 | // } 141 | 142 | //fmt.Printf("----cmd=%s balance=%s\n", s, balance) 143 | } 144 | 145 | func pathExists(path string) (bool) { 146 | _, err := os.Stat(path) 147 | if err == nil { 148 | return true 149 | } 150 | return false 151 | } 152 | 153 | func getBalance(accountNo string, fileContent *string) { 154 | command := "curl -X POST --data '{\"jsonrpc\":\"3.0\",\"method\":\"eth_getBalance\",\"params\":[\""+accountNo+"\",\"latest\"],\"id\":1}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545"; 155 | exec_shell(command, accountNo, MAX_NUM_RETRY, fileContent) 156 | } 157 | 158 | func main() { 159 | if len(os.Args) < 3 { 160 | log.Fatal("Param Invalid!!! go run getBalance.go [timeFrom] [timeTo], eg. go run getBalance.go 2018-01-01-00-00-00 2018-02-01-00-00-00") 161 | } 162 | //log.Print("187b2445cfb31e42b6a"); 163 | fmt.Println("getBalance begin=================="); 164 | timeBegin := time.Now().Unix() 165 | MULTICORE := runtime.NumCPU() 166 | runtime.GOMAXPROCS(MULTICORE) 167 | timeFrom := os.Args[1] 168 | timeTo := os.Args[2] 169 | if timeFrom >= timeTo { 170 | log.Fatal("timeFrom=", timeFrom, " >= timeTo=", timeTo) 171 | } 172 | accountsDir := lib.GetAndCheckDir("accounts") 173 | balanceDir := lib.GetAndCheckDir("balance") 174 | srcFileName := accountsDir + "/" + timeFrom + "-" + timeTo 175 | desFileName := balanceDir + "/" + timeFrom + "-" + timeTo 176 | lib.ExecCmd("rm " + desFileName, false) 177 | fd1, err1 := os.Open(srcFileName) 178 | if err1 != nil { 179 | log.Fatal("file=",srcFileName," open fail!! First of all, you must run 'go run getAccount.go ", timeFrom, " ", timeTo, "' to get accounts! err=" , err1); 180 | } 181 | fd2,err2 := os.OpenFile(desFileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 182 | if err2 != nil { 183 | log.Fatal("file=",desFileName," open fail!!", err2); 184 | } 185 | fileContent := "" 186 | i := 0 187 | readBuf := bufio.NewReader(fd1) 188 | for { 189 | line, err := readBuf.ReadString('\n') 190 | if err != nil { 191 | if err == io.EOF { 192 | break 193 | } 194 | return 195 | } 196 | line = strings.TrimSpace(line) 197 | getBalance(line, &fileContent) 198 | if i%APPEND_FILE_NUM == 0 { 199 | if len(fileContent) > 0 { 200 | buf:=[]byte(fileContent) 201 | fd2.Write(buf) 202 | fileContent = "" 203 | log.Print("write balance into file=", desFileName, " i=",i) 204 | } else { 205 | fileContent = "" 206 | log.Print("fileContent len <= 0, not need to write balance into file=", desFileName, " i=",i) 207 | } 208 | } 209 | i++ 210 | //break////////////////////////////////////////////// 211 | } 212 | if len(fileContent) > 0 { 213 | buf:=[]byte(fileContent) 214 | fd2.Write(buf) 215 | log.Print("write balance into file=", desFileName, " i=",i) 216 | } 217 | fd1.Close() 218 | fd2.Close() 219 | 220 | timeEnd := time.Now().Unix() 221 | log.Print("getBalance finish, cost=", (timeEnd - timeBegin), "s") 222 | } 223 | -------------------------------------------------------------------------------- /getBalanceMulti.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | "os/exec" 10 | "math" 11 | "time" 12 | "runtime" 13 | "encoding/json" 14 | "os" 15 | "io" 16 | "bufio" 17 | "./lib" 18 | // "./go-ethereum/common" 19 | // "./go-ethereum/common/hexutil" 20 | //"./golang-set" 21 | "github.com/ethereum/go-ethereum/common" 22 | "github.com/ethereum/go-ethereum/common/hexutil" 23 | // "github.com/deckarep/golang-set" 24 | ) 25 | 26 | var MAX_NUM_SHARE = 10 27 | var MAX_NUM_RETRY = 3 28 | var APPEND_FILE_NUM = 100 29 | var c = make(chan string, 100) 30 | 31 | type ResultOfGetBalance struct { 32 | Jsonrpc string 33 | Id int 34 | Result string//hexutil.Uint64 35 | } 36 | type RPCTransaction struct { 37 | BlockHash common.Hash `json:"blockHash"` 38 | BlockNumber *hexutil.Big `json:"blockNumber"` 39 | From *common.Address `json:"from"` 40 | Gas hexutil.Uint64 `json:"gas"` 41 | GasPrice *hexutil.Big `json:"gasPrice"` 42 | Hash common.Hash `json:"hash"` 43 | Input hexutil.Bytes `json:"input"` 44 | Nonce hexutil.Uint64 `json:"nonce"` 45 | To *common.Address `json:"to"` 46 | TransactionIndex hexutil.Uint `json:"transactionIndex"` 47 | Value *hexutil.Big `json:"value"` 48 | V *hexutil.Big `json:"v"` 49 | R *hexutil.Big `json:"r"` 50 | S *hexutil.Big `json:"s"` 51 | } 52 | 53 | type Block struct { 54 | UncleHashes []common.Hash `json:"uncles"` 55 | Hash common.Hash `json:"hash"` 56 | Transactions []RPCTransaction `json:"transactions"` 57 | } 58 | 59 | var tenToAny map[int]string = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29: "t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62: "M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"} 60 | 61 | 62 | func decimalToAny(num, n int) string { 63 | new_num_str := "" 64 | var remainder int 65 | var remainder_string string 66 | for num != 0 { 67 | remainder = num % n 68 | if 76 > remainder && remainder > 9 { 69 | remainder_string = tenToAny[remainder] 70 | } else { 71 | remainder_string = strconv.Itoa(remainder) 72 | } 73 | new_num_str = remainder_string + new_num_str 74 | num = num / n 75 | } 76 | return new_num_str 77 | } 78 | 79 | func findkey(in string) int { 80 | result := -1 81 | for k, v := range tenToAny { 82 | if in == v { 83 | result = k 84 | } 85 | } 86 | return result 87 | } 88 | //187b2445cfb31e42b6a 89 | func anyToDecimal(num string, n int) float64 { 90 | var new_num float64 91 | new_num = 0.0 92 | nNum := len(strings.Split(num, "")) - 1 93 | for _, value := range strings.Split(num, "") { 94 | 95 | tmp := float64(findkey(value)) 96 | if tmp != -1 { 97 | new_num = new_num + tmp*math.Pow(float64(n), float64(nNum)) 98 | nNum = nNum - 1 99 | } else { 100 | break 101 | } 102 | } 103 | return float64(new_num) 104 | } 105 | 106 | func exec_shell(s string, accountNo string, retry int, fileContent *string) { 107 | cmd := exec.Command("/bin/bash", "-c", s) 108 | var out bytes.Buffer 109 | 110 | cmd.Stdout = &out 111 | err := cmd.Run() 112 | if err != nil { 113 | //log.Fatal("cmd="+s+" run fail!", err) 114 | retry-- 115 | log.Print("Fatal!!!retry=",retry," cmd="+s+" run fail!", err) 116 | if(retry > 0) { 117 | exec_shell(s, accountNo, retry, fileContent) 118 | } 119 | } 120 | 121 | stb := &ResultOfGetBalance{} 122 | //fmt.Printf("-----cmd=%s, result=%s\n", s, out.String()) 123 | err = json.Unmarshal([]byte(out.String()), &stb) 124 | 125 | if err != nil { 126 | retry--; 127 | log.Print("umarshal fail!!!retry=",retry," accountNo=",accountNo," cmd=", s, " input=", out.String()," ", err) 128 | if(retry > 0) { 129 | exec_shell(s, accountNo, retry, fileContent) 130 | } 131 | } 132 | //fmt.Printf("----result=%s:%d\n", stb.Jsonrpc, stb.Id) 133 | balance := stb.Result 134 | balance = balance[2 : len(balance)] // get rid of 0x prefix 135 | price := anyToDecimal(balance, 16)/1000000000000000000 136 | *fileContent += accountNo + " " + strconv.FormatFloat(price, 'f', -1, 64) + "\n"; 137 | //if balance <= 0 { 138 | //log.Print("block=", blockNumber, " balance len <= 0") 139 | // return 140 | // } 141 | 142 | //fmt.Printf("----cmd=%s balance=%s\n", s, balance) 143 | } 144 | 145 | func pathExists(path string) (bool) { 146 | _, err := os.Stat(path) 147 | if err == nil { 148 | return true 149 | } 150 | return false 151 | } 152 | 153 | func getBalance(srcFileNamePrefix string, desFileNamePrefix string, sufix int) { 154 | srcFileName := srcFileNamePrefix + strconv.Itoa(sufix) 155 | desFileName := desFileNamePrefix + strconv.Itoa(sufix) 156 | log.Print("taskId=", desFileName, " begin! srcFileName=", srcFileName) 157 | fd1, err1 := os.Open(srcFileName) 158 | if err1 != nil { 159 | log.Fatal("file=",srcFileName," open fail!!", err1); 160 | } 161 | fd2,err2 := os.OpenFile(desFileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 162 | if err2 != nil { 163 | log.Fatal("file=",desFileName," open fail!!", err2); 164 | } 165 | fileContent := "" 166 | i := 0 167 | readBuf := bufio.NewReader(fd1) 168 | for { 169 | accountNo, err := readBuf.ReadString('\n') 170 | if err != nil { 171 | if err == io.EOF { 172 | break 173 | } 174 | return 175 | } 176 | accountNo = strings.TrimSpace(accountNo) 177 | command := "curl -X POST --data '{\"jsonrpc\":\"3.0\",\"method\":\"eth_getBalance\",\"params\":[\""+accountNo+"\",\"latest\"],\"id\":1}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545"; 178 | exec_shell(command, accountNo, MAX_NUM_RETRY, &fileContent) 179 | if i%APPEND_FILE_NUM == 0 { 180 | if len(fileContent) > 0 { 181 | buf:=[]byte(fileContent) 182 | fd2.Write(buf) 183 | fileContent = "" 184 | log.Print("write balance into file=", desFileName, " i=",i) 185 | } else { 186 | fileContent = "" 187 | log.Print("fileContent len <= 0, not need to write balance into file=", desFileName, " i=",i) 188 | } 189 | } 190 | i++ 191 | //break////////////////////////////////////////////// 192 | } 193 | if len(fileContent) > 0 { 194 | buf:=[]byte(fileContent) 195 | fd2.Write(buf) 196 | log.Print("write balance into file=", desFileName, " i=",i) 197 | } 198 | fd1.Close() 199 | fd2.Close() 200 | c <- desFileName 201 | } 202 | 203 | func main() { 204 | log.Print(lib.GetAndCheckDir("haha")) 205 | if len(os.Args) < 4 { 206 | log.Fatal("Param Invalid!!! go run getBalanceMulti.go [srcFileNamePrefix] [desFileNamePrefix] [concurrentNum]") 207 | } 208 | fmt.Println("getBalance begin=================="); 209 | timeBegin := time.Now().Unix() 210 | MULTICORE := runtime.NumCPU() 211 | runtime.GOMAXPROCS(MULTICORE) 212 | srcFileNamePrefix := os.Args[1] 213 | desFileNamePrefix := os.Args[2] 214 | concurrentNum,err := strconv.Atoi(os.Args[3]) 215 | if err != nil { 216 | log.Fatal("Param fail!!! concurrentNum=", os.Args[3], " must be integer! ", err); 217 | } 218 | for sufix := 1; sufix <= concurrentNum; sufix++ { 219 | go getBalance(srcFileNamePrefix, desFileNamePrefix, sufix) 220 | } 221 | for i := 1; i <= concurrentNum; i++ { 222 | taskId := <- c 223 | log.Print(taskId, " finish"); 224 | } 225 | 226 | timeEnd := time.Now().Unix() 227 | log.Print("getBalance finish, cost=", (timeEnd - timeBegin), "s") 228 | } 229 | -------------------------------------------------------------------------------- /getAccount.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | // "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | "os/exec" 10 | "math" 11 | "time" 12 | "runtime" 13 | "encoding/json" 14 | "os" 15 | "./lib" 16 | // "./go-ethereum/common" 17 | // "./go-ethereum/common/hexutil" 18 | //"./golang-set" 19 | "github.com/ethereum/go-ethereum/common" 20 | "github.com/ethereum/go-ethereum/common/hexutil" 21 | // "github.com/deckarep/golang-set" 22 | ) 23 | 24 | var MAX_NUM_SHARE = 10 25 | var MAX_NUM_RETRY = 3 26 | var APPEND_FILE_NUM = 100 27 | var c = make(chan string, 100) 28 | 29 | type ResultOfGetBlockByNumber struct { 30 | Jsonrpc string 31 | Id int 32 | Result Block 33 | } 34 | type RPCTransaction struct { 35 | BlockHash common.Hash `json:"blockHash"` 36 | BlockNumber *hexutil.Big `json:"blockNumber"` 37 | From *common.Address `json:"from"` 38 | Gas hexutil.Uint64 `json:"gas"` 39 | GasPrice *hexutil.Big `json:"gasPrice"` 40 | Hash common.Hash `json:"hash"` 41 | Input hexutil.Bytes `json:"input"` 42 | Nonce hexutil.Uint64 `json:"nonce"` 43 | To *common.Address `json:"to"` 44 | TransactionIndex hexutil.Uint `json:"transactionIndex"` 45 | Value *hexutil.Big `json:"value"` 46 | V *hexutil.Big `json:"v"` 47 | R *hexutil.Big `json:"r"` 48 | S *hexutil.Big `json:"s"` 49 | } 50 | 51 | type Block struct { 52 | UncleHashes []common.Hash `json:"uncles"` 53 | Hash common.Hash `json:"hash"` 54 | Transactions []RPCTransaction `json:"transactions"` 55 | } 56 | 57 | var tenToAny map[int]string = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29: "t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62: "M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"} 58 | 59 | 60 | func decimalToAny(num, n int) string { 61 | new_num_str := "" 62 | var remainder int 63 | var remainder_string string 64 | for num != 0 { 65 | remainder = num % n 66 | if 76 > remainder && remainder > 9 { 67 | remainder_string = tenToAny[remainder] 68 | } else { 69 | remainder_string = strconv.Itoa(remainder) 70 | } 71 | new_num_str = remainder_string + new_num_str 72 | num = num / n 73 | } 74 | return new_num_str 75 | } 76 | 77 | func findkey(in string) int { 78 | result := -1 79 | for k, v := range tenToAny { 80 | if in == v { 81 | result = k 82 | } 83 | } 84 | return result 85 | } 86 | func anyToDecimal(num string, n int) int { 87 | var new_num float64 88 | new_num = 0.0 89 | nNum := len(strings.Split(num, "")) - 1 90 | for _, value := range strings.Split(num, "") { 91 | tmp := float64(findkey(value)) 92 | if tmp != -1 { 93 | new_num = new_num + tmp*math.Pow(float64(n), float64(nNum)) 94 | nNum = nNum - 1 95 | } else { 96 | break 97 | } 98 | } 99 | return int(new_num) 100 | } 101 | 102 | func exec_shell(s string, blockNumber int, retry int, fileContent *string) { 103 | cmd := exec.Command("/bin/bash", "-c", s) 104 | var out bytes.Buffer 105 | 106 | cmd.Stdout = &out 107 | err := cmd.Run() 108 | if err != nil { 109 | //log.Fatal("cmd="+s+" run fail!", err) 110 | retry-- 111 | log.Print("Fatal!!!retry=",retry," cmd="+s+" run fail!", err) 112 | if(retry > 0) { 113 | exec_shell(s, blockNumber, retry, fileContent) 114 | } 115 | } 116 | 117 | stb := &ResultOfGetBlockByNumber{} 118 | //fmt.Printf("%s\n", out.String()) 119 | err = json.Unmarshal([]byte(out.String()), &stb) 120 | 121 | if err != nil { 122 | retry--; 123 | log.Print("umarshal fail!!!retry=",retry," block=",blockNumber," cmd=", s, " input=", out.String()," ", err) 124 | if(retry > 0) { 125 | exec_shell(s, blockNumber, retry, fileContent) 126 | } 127 | } 128 | //fmt.Printf("----result=%s:%d\n", stb.Jsonrpc, stb.Id) 129 | blockInfo := stb.Result 130 | if len(blockInfo.Transactions) <= 0 { 131 | //log.Print("block=", blockNumber, " transaction len <= 0") 132 | return 133 | } 134 | //accountSet := mapset.NewSet() 135 | for _, value := range blockInfo.Transactions { 136 | //from := "" 137 | if value.From != nil { 138 | // from = value.From.String() 139 | // accountSet.Add(value.From.String()) 140 | *fileContent += value.From.String() + "\n"; 141 | } 142 | //to := "" 143 | if value.To != nil { 144 | // to = value.To.String() 145 | // accountSet.Add(value.To.String()) 146 | *fileContent += value.To.String() + "\n"; 147 | } 148 | //fmt.Printf("----block=%d i=%d fr=%s, to=%s\n", blockNumber, index, from, to) 149 | } 150 | //log.Print(accountSet) 151 | } 152 | 153 | 154 | func getAccount(fromBlockNumber int, toBlockNumber int, fileName string) { 155 | taskId := strconv.Itoa(fromBlockNumber) + "-" + strconv.Itoa(toBlockNumber) 156 | log.Print(taskId," begin: file=", fileName) 157 | fd,err := os.OpenFile(fileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 158 | if err != nil { 159 | log.Fatal(fileName, " open fail! ", err) 160 | return 161 | } 162 | fileContent := "" 163 | i := 0 164 | for blockNumber := fromBlockNumber; blockNumber <= toBlockNumber; blockNumber++ { 165 | blockNumberHex := "0x" + decimalToAny(blockNumber, 16); 166 | command := "curl -X POST --data '{\"jsonrpc\":\"3.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"" + blockNumberHex + "\", true],\"id\":1}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545"; 167 | 168 | exec_shell(command, blockNumber, MAX_NUM_RETRY, &fileContent) 169 | if i%APPEND_FILE_NUM == 0 { 170 | if len(fileContent) > 0 { 171 | buf:=[]byte(fileContent) 172 | fd.Write(buf) 173 | fileContent = "" 174 | log.Print("write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 175 | } else { 176 | fileContent = "" 177 | log.Print("fileContent len <= 0, not need to write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 178 | } 179 | } 180 | i++ 181 | } 182 | if len(fileContent) > 0 { 183 | buf:=[]byte(fileContent) 184 | fd.Write(buf) 185 | log.Print("write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 186 | } 187 | fd.Close() 188 | c <- taskId 189 | } 190 | 191 | func main() { 192 | if len(os.Args) < 3 { 193 | log.Fatal("Param Invalid!!! go run getAccount.go [timeFrom] [timeTo], eg. go run getAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00") 194 | } 195 | log.Print("====getAccount begin=================="); 196 | timeBegin := time.Now().Unix() 197 | MULTICORE := runtime.NumCPU() 198 | runtime.GOMAXPROCS(MULTICORE) 199 | //blockNumber := 4927600; 200 | timeFrom := os.Args[1] 201 | timeTo := os.Args[2] 202 | if timeFrom >= timeTo { 203 | log.Fatal("timeFrom=", timeFrom, " >= timeTo=", timeTo) 204 | } 205 | blockNumberBegin, blockNumberEnd := lib.GetBlockNumberByTime(timeFrom, timeTo) 206 | totalBlockNum := blockNumberEnd - blockNumberBegin + 1; 207 | share := totalBlockNum / MAX_NUM_SHARE + 1 208 | loopCount := 0 209 | dir := lib.GetAndCheckDir("accounts") 210 | files := dir + "/" + timeFrom + "-" + timeTo + "-*" 211 | resultFile := dir + "/" + timeFrom + "-" + timeTo 212 | resultFileTmp := dir + "/"+strconv.FormatInt(timeBegin, 10)+"tmp" 213 | lib.ExecCmd("rm " + files, false) 214 | lib.ExecCmd("rm " + resultFileTmp, false) 215 | for i := blockNumberBegin; i <= blockNumberEnd; i++ { 216 | from := i 217 | if ((share+i) <= blockNumberEnd) { 218 | i += share 219 | } else { 220 | i = blockNumberEnd 221 | } 222 | to := i 223 | loopCount++ 224 | fileName := dir + "/" + timeFrom + "-" + timeTo + "-" + strconv.Itoa(loopCount) 225 | go getAccount(from, to, fileName); 226 | } 227 | for i := 0; i < loopCount; i++ { 228 | taskId := <- c 229 | log.Print(taskId, " finish"); 230 | } 231 | 232 | lib.ExecCmd( "cat "+files + " >> " + resultFileTmp, true) 233 | lib.ExecCmd("rm " + files, true) 234 | lib.ExecCmd("sort " + resultFileTmp + "|uniq >" + resultFile, true) 235 | lib.ExecCmd("rm " + resultFileTmp, true) 236 | 237 | timeEnd := time.Now().Unix() 238 | log.Print("====getAccount finish, cost=", (timeEnd - timeBegin), "s") 239 | } 240 | -------------------------------------------------------------------------------- /getTxTimelineGroupByAccount.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "sort" 6 | "fmt" 7 | "log" 8 | "strconv" 9 | "strings" 10 | "time" 11 | "runtime" 12 | "encoding/json" 13 | "os" 14 | "io" 15 | "bufio" 16 | "./lib" 17 | //"./golang-set" 18 | // "github.com/ethereum/go-ethereum/common" 19 | // "github.com/ethereum/go-ethereum/common/hexutil" 20 | // "github.com/deckarep/golang-set" 21 | ) 22 | 23 | var MAX_NUM_SHARE = 10 24 | var MAX_NUM_RETRY = 3 25 | var APPEND_FILE_ACCOUNT_MAX_NUM = 100 26 | var APPEND_FILE_LOOP_MAX_NUM = 1000 27 | var APPEND_FILE_ACCOUNT_TX_MAX_NUM = 10000 28 | 29 | type Timeline struct { 30 | Txtime string 31 | TxAmt string 32 | TxType string 33 | } 34 | type TimelineSlice [] Timeline 35 | func (a TimelineSlice) Len() int { // Öд Len() ·½·¨ 36 | return len(a) 37 | } 38 | func (a TimelineSlice) Swap(i, j int){ // Öд Swap() ·½·¨ 39 | a[i], a[j] = a[j], a[i] 40 | } 41 | func (a TimelineSlice) Less(i, j int) bool { // Öд Less() ·½·¨£¬ ´Ӵó¡ÅÐ 42 | return a[j].Txtime < a[i].Txtime 43 | } 44 | func setTxMap(m3 map[string]string, key string, value string) { 45 | if _, ok := m3[key]; ok { 46 | m3[key] += "&" + value 47 | } else { 48 | m3[key] = value 49 | } 50 | } 51 | func getFileContent(txMap map[string]string) bytes.Buffer { 52 | fileContent := bytes.Buffer{} 53 | for account, v := range txMap { 54 | if len(v) <= 0 { 55 | continue; 56 | } 57 | timelineList := strings.Split(v, "&") 58 | timelineSlice := make([]Timeline, len(timelineList)) 59 | for i := range timelineList { 60 | timelineItemArr := strings.Split(timelineList[i], "|") 61 | timeline := &Timeline{ 62 | timelineItemArr[0], 63 | timelineItemArr[1], 64 | timelineItemArr[2], 65 | } 66 | timelineSlice[i] = *timeline 67 | } 68 | sort.Sort(TimelineSlice(timelineSlice)) 69 | jsonStr, err := json.Marshal(timelineSlice) 70 | if err != nil { 71 | log.Print("Marshal failed!!!", err) 72 | continue 73 | } 74 | fileContent.WriteString(account + "\t") 75 | fileContent.WriteString(strconv.Itoa(len(timelineList)) + "\t") 76 | fileContent.WriteString(string(jsonStr) + "\n") 77 | } 78 | return fileContent 79 | } 80 | 81 | func main() { 82 | if len(os.Args) < 3 { 83 | log.Fatal("Param Invalid!!! go run getTxTimelineGroupByAccount.go [timeFrom] [timeTo], eg. go run getTxTimelineGroupByAccount.go 2018-01-01-00-00-00 2018-02-01-00-00-00") 84 | } 85 | fmt.Println("getTxTimelineGroupByAccount begin=================="); 86 | timeBegin := time.Now().Unix() 87 | MULTICORE := runtime.NumCPU() 88 | runtime.GOMAXPROCS(MULTICORE) 89 | timeFrom := os.Args[1] 90 | timeTo := os.Args[2] 91 | if timeFrom >= timeTo { 92 | log.Fatal("timeFrom=", timeFrom, " >= timeTo=", timeTo) 93 | } 94 | dir := lib.GetAndCheckDir("tx") 95 | fromFileName := dir + "/" + timeFrom + "-" + timeTo + "-from-sort" 96 | toFileName := dir + "/" + timeFrom + "-" + timeTo + "-to-sort" 97 | desFileName := dir + "/" + timeFrom + "-" + timeTo + "-timeline" 98 | fd1, err1 := os.Open(fromFileName) 99 | if err1 != nil { 100 | log.Fatal("file=",fromFileName," open fail!! First of all, you must run 'go run getTxByTime.go ", timeFrom, " ", timeTo, "' to get transactions! err=" , err1); 101 | } 102 | fd2, err2 := os.Open(toFileName) 103 | if err2 != nil { 104 | log.Fatal("file=",toFileName," open fail!! First of all, you must run 'go run getTxByTime.go ", timeFrom, " ", timeTo, "' to get transactions! err=" , err1); 105 | } 106 | desFd,err3 := os.OpenFile(desFileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 107 | if err3 != nil { 108 | log.Fatal("file=",desFileName," open fail!!", err3); 109 | } 110 | fromLineNum := 0 111 | toLineNum := 0 112 | loopCount := 0 113 | readBufFrom := bufio.NewReader(fd1) 114 | readBufTo := bufio.NewReader(fd2) 115 | 116 | from := "" 117 | to := "" 118 | //txTime := "" 119 | //txType := "" 120 | //timelineString := "" 121 | var arrFrom [] string 122 | var arrTo [] string 123 | txMap := map[string]string{} 124 | 125 | lineFrom, errReadFrom := readBufFrom.ReadString('\n') 126 | if errReadFrom != io.EOF { 127 | fromLineNum++ 128 | lineFrom = strings.TrimSpace(lineFrom) 129 | arrFrom = strings.Split(lineFrom, "\t") 130 | from = arrFrom[2] 131 | } 132 | lineTo, errReadTo := readBufTo.ReadString('\n') 133 | if errReadTo != io.EOF { 134 | toLineNum++ 135 | lineTo = strings.TrimSpace(lineTo) 136 | arrTo = strings.Split(lineTo, "\t") 137 | to = arrTo[3] 138 | } 139 | 140 | 141 | for { 142 | if errReadFrom == io.EOF || errReadTo == io.EOF { 143 | break 144 | } 145 | if from == to { 146 | tmp := from 147 | accountTxNumFrom :=0 148 | accountTxNumTo :=0 149 | for { 150 | setTxMap(txMap, from, arrFrom[0] + "|" + arrFrom[4] + "|out") 151 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 152 | fromLineNum++ 153 | if errReadFrom == io.EOF { 154 | break 155 | } 156 | lineFrom = strings.TrimSpace(lineFrom) 157 | arrFrom = strings.Split(lineFrom, "\t") 158 | from = arrFrom[2] 159 | if from > tmp { 160 | break 161 | } 162 | accountTxNumFrom++ 163 | if accountTxNumFrom >= APPEND_FILE_ACCOUNT_TX_MAX_NUM { 164 | fileContent := getFileContent(txMap) 165 | wBuf:=[]byte(fileContent.String()) 166 | desFd.Write(wBuf) 167 | log.Print("write account's tx into file=", desFileName, " fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 168 | txMap = map[string]string{} 169 | accountTxNumFrom = 0 170 | } 171 | } 172 | for { 173 | setTxMap(txMap, to, arrTo[0] + "|" + arrTo[4] + "|in") 174 | lineTo, errReadTo = readBufTo.ReadString('\n') 175 | toLineNum++ 176 | if errReadTo == io.EOF { 177 | break 178 | } 179 | lineTo = strings.TrimSpace(lineTo) 180 | arrTo = strings.Split(lineTo, "\t") 181 | to = arrTo[3] 182 | if to > tmp { 183 | break 184 | } 185 | accountTxNumTo++ 186 | if accountTxNumTo >= APPEND_FILE_ACCOUNT_TX_MAX_NUM { 187 | fileContent := getFileContent(txMap) 188 | wBuf:=[]byte(fileContent.String()) 189 | desFd.Write(wBuf) 190 | log.Print("write account's tx into file=", desFileName, " fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 191 | txMap = map[string]string{} 192 | accountTxNumTo = 0 193 | } 194 | } 195 | } else if from < to { 196 | setTxMap(txMap, from, arrFrom[0] + "|" + arrFrom[4] + "|out") 197 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 198 | fromLineNum++ 199 | if errReadFrom == io.EOF { 200 | break 201 | } 202 | lineFrom = strings.TrimSpace(lineFrom) 203 | arrFrom = strings.Split(lineFrom, "\t") 204 | from = arrFrom[2] 205 | } else { 206 | setTxMap(txMap, to, arrTo[0] + "|" + arrTo[4] + "|in") 207 | lineTo, errReadTo = readBufTo.ReadString('\n') 208 | toLineNum++ 209 | if errReadTo == io.EOF { 210 | break 211 | } 212 | lineTo = strings.TrimSpace(lineTo) 213 | arrTo = strings.Split(lineTo, "\t") 214 | to = arrTo[3] 215 | } 216 | loopCount++ 217 | if len(txMap) >= APPEND_FILE_ACCOUNT_MAX_NUM || loopCount >= APPEND_FILE_LOOP_MAX_NUM { 218 | fileContent := getFileContent(txMap) 219 | wBuf:=[]byte(fileContent.String()) 220 | desFd.Write(wBuf) 221 | log.Print("write account's tx into file=", desFileName, " fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 222 | txMap = map[string]string{} 223 | loopCount = 0 224 | } 225 | } 226 | log.Print("==================out of loop================fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 227 | 228 | if len(txMap) >= APPEND_FILE_ACCOUNT_MAX_NUM || loopCount >= APPEND_FILE_LOOP_MAX_NUM { 229 | fileContent := getFileContent(txMap) 230 | wBuf:=[]byte(fileContent.String()) 231 | desFd.Write(wBuf) 232 | log.Print("write account's tx into file=", desFileName, " fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 233 | txMap = map[string]string{} 234 | } 235 | 236 | if errReadFrom != io.EOF && errReadTo == io.EOF { 237 | for { 238 | setTxMap(txMap, from, arrFrom[0] + "|" + arrFrom[4] + "|out") 239 | lineFrom, errReadFrom = readBufFrom.ReadString('\n') 240 | fromLineNum++ 241 | if errReadFrom == io.EOF { 242 | break 243 | } 244 | lineFrom = strings.TrimSpace(lineFrom) 245 | arrFrom = strings.Split(lineFrom, "\t") 246 | from = arrFrom[2] 247 | } 248 | } 249 | if errReadFrom == io.EOF && errReadTo != io.EOF { 250 | for { 251 | setTxMap(txMap, to, arrTo[0] + "|" + arrTo[4] + "|in") 252 | lineTo, errReadTo = readBufTo.ReadString('\n') 253 | toLineNum++ 254 | if errReadTo == io.EOF { 255 | break 256 | } 257 | lineTo = strings.TrimSpace(lineTo) 258 | arrTo = strings.Split(lineTo, "\t") 259 | to = arrTo[3] 260 | } 261 | } 262 | 263 | if len(txMap) > 0 { 264 | fileContent := getFileContent(txMap) 265 | wBuf:=[]byte(fileContent.String()) 266 | desFd.Write(wBuf) 267 | log.Print("write account's tx into file=", desFileName, " fromLineNum=",fromLineNum, " toLineNum=", toLineNum) 268 | } 269 | 270 | fd1.Close() 271 | fd2.Close() 272 | desFd.Close() 273 | 274 | timeEnd := time.Now().Unix() 275 | log.Print("getTxTimelineGroupByAccount finish, cost=", (timeEnd - timeBegin), "s") 276 | } 277 | -------------------------------------------------------------------------------- /getTxByTime.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | // "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | "os/exec" 10 | "math" 11 | "time" 12 | "runtime" 13 | "encoding/json" 14 | "os" 15 | "./lib" 16 | // "./go-ethereum/common" 17 | // "./go-ethereum/common/hexutil" 18 | //"./golang-set" 19 | "github.com/ethereum/go-ethereum/common" 20 | "github.com/ethereum/go-ethereum/common/hexutil" 21 | // "github.com/deckarep/golang-set" 22 | ) 23 | 24 | var MAX_NUM_SHARE = 10 25 | var MAX_NUM_RETRY = 3 26 | var APPEND_FILE_NUM = 100 27 | var c = make(chan string, 100) 28 | 29 | type ResultOfGetBlockByNumber struct { 30 | Jsonrpc string 31 | Id int 32 | Result Block 33 | } 34 | type RPCTransaction struct { 35 | BlockHash common.Hash `json:"blockHash"` 36 | BlockNumber *hexutil.Big `json:"blockNumber"` 37 | From *common.Address `json:"from"` 38 | Gas hexutil.Uint64 `json:"gas"` 39 | GasPrice *hexutil.Big `json:"gasPrice"` 40 | Hash common.Hash `json:"hash"` 41 | Input hexutil.Bytes `json:"input"` 42 | Nonce hexutil.Uint64 `json:"nonce"` 43 | To *common.Address `json:"to"` 44 | TransactionIndex hexutil.Uint `json:"transactionIndex"` 45 | Value *hexutil.Big `json:"value"` 46 | V *hexutil.Big `json:"v"` 47 | R *hexutil.Big `json:"r"` 48 | S *hexutil.Big `json:"s"` 49 | } 50 | 51 | type Block struct { 52 | UncleHashes []common.Hash `json:"uncles"` 53 | Hash common.Hash `json:"hash"` 54 | Timestamp string `json:"timestamp"` 55 | Transactions []RPCTransaction `json:"transactions"` 56 | } 57 | 58 | var tenToAny map[int]string = map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5", 6: "6", 7: "7", 8: "8", 9: "9", 10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "f", 16: "g", 17: "h", 18: "i", 19: "j", 20: "k", 21: "l", 22: "m", 23: "n", 24: "o", 25: "p", 26: "q", 27: "r", 28: "s", 29: "t", 30: "u", 31: "v", 32: "w", 33: "x", 34: "y", 35: "z", 36: ":", 37: ";", 38: "<", 39: "=", 40: ">", 41: "?", 42: "@", 43: "[", 44: "]", 45: "^", 46: "_", 47: "{", 48: "|", 49: "}", 50: "A", 51: "B", 52: "C", 53: "D", 54: "E", 55: "F", 56: "G", 57: "H", 58: "I", 59: "J", 60: "K", 61: "L", 62: "M", 63: "N", 64: "O", 65: "P", 66: "Q", 67: "R", 68: "S", 69: "T", 70: "U", 71: "V", 72: "W", 73: "X", 74: "Y", 75: "Z"} 59 | 60 | 61 | func decimalToAny(num, n int) string { 62 | new_num_str := "" 63 | var remainder int 64 | var remainder_string string 65 | for num != 0 { 66 | remainder = num % n 67 | if 76 > remainder && remainder > 9 { 68 | remainder_string = tenToAny[remainder] 69 | } else { 70 | remainder_string = strconv.Itoa(remainder) 71 | } 72 | new_num_str = remainder_string + new_num_str 73 | num = num / n 74 | } 75 | return new_num_str 76 | } 77 | 78 | func findkey(in string) int { 79 | result := -1 80 | for k, v := range tenToAny { 81 | if in == v { 82 | result = k 83 | } 84 | } 85 | return result 86 | } 87 | func anyToDecimal(num string, n int) float64 { 88 | var new_num float64 89 | new_num = 0.0 90 | nNum := len(strings.Split(num, "")) - 1 91 | for _, value := range strings.Split(num, "") { 92 | tmp := float64(findkey(value)) 93 | if tmp != -1 { 94 | new_num = new_num + tmp*math.Pow(float64(n), float64(nNum)) 95 | nNum = nNum - 1 96 | } else { 97 | break 98 | } 99 | } 100 | return float64(new_num) 101 | } 102 | 103 | func exec_shell(s string, blockNumber int, retry int, fileContent *bytes.Buffer) { 104 | cmd := exec.Command("/bin/bash", "-c", s) 105 | var out bytes.Buffer 106 | 107 | cmd.Stdout = &out 108 | err := cmd.Run() 109 | if err != nil { 110 | //log.Fatal("cmd="+s+" run fail!", err) 111 | retry-- 112 | log.Print("Fatal!!!retry=",retry," cmd="+s+" run fail!", err) 113 | if(retry > 0) { 114 | exec_shell(s, blockNumber, retry, fileContent) 115 | } 116 | } 117 | 118 | stb := &ResultOfGetBlockByNumber{} 119 | //fmt.Printf("%s\n", out.String()) 120 | err = json.Unmarshal([]byte(out.String()), &stb) 121 | 122 | if err != nil { 123 | retry--; 124 | log.Print("umarshal fail!!!retry=",retry," block=",blockNumber," cmd=", s, " input=", out.String()," ", err) 125 | if(retry > 0) { 126 | exec_shell(s, blockNumber, retry, fileContent) 127 | } 128 | } 129 | //fmt.Printf("----result=%s:%d\n", stb.Jsonrpc, stb.Id) 130 | blockInfo := stb.Result 131 | if len(blockInfo.Transactions) <= 0 { 132 | //log.Print("block=", blockNumber, " transaction len <= 0") 133 | return 134 | } 135 | timeLayout := "2006-01-02-15-04-05" 136 | timestamp := blockInfo.Timestamp 137 | timestamp = timestamp[2 : len(timestamp)] // get rid of 0x prefix 138 | txTime := time.Unix(int64(anyToDecimal(timestamp, 16)), 0).Format(timeLayout) 139 | for _, value := range blockInfo.Transactions { 140 | fileContent.WriteString(txTime + "\t") 141 | fileContent.WriteString(value.Hash.String() + "\t") 142 | if value.From != nil { 143 | fileContent.WriteString(value.From.String() + "\t") 144 | } else { 145 | fileContent.WriteString("null\t") 146 | } 147 | 148 | if value.To != nil { 149 | fileContent.WriteString(value.To.String() + "\t") 150 | } else { 151 | fileContent.WriteString("null\t") 152 | } 153 | txValue := value.Value.String() 154 | txValue = txValue[2 : len(txValue)] // get rid of 0x prefix 155 | price := anyToDecimal(txValue, 16)/1000000000000000000 156 | fileContent.WriteString(strconv.FormatFloat(price, 'f', -1, 64) + "\n") 157 | //fmt.Printf("----block=%d transactionIndex=%s fr=%s, to=%s\n", blockNumber, value.TransactionIndex.String(), value.From.String(), value.To.String()) 158 | } 159 | //log.Print(accountSet) 160 | } 161 | 162 | func pathExists(path string) (bool) { 163 | _, err := os.Stat(path) 164 | if err == nil { 165 | return true 166 | } 167 | return false 168 | } 169 | 170 | func getTxByBlock(fromBlockNumber int, toBlockNumber int, fileName string) { 171 | taskId := strconv.Itoa(fromBlockNumber) + "-" + strconv.Itoa(toBlockNumber) 172 | log.Print(taskId," begin: file=", fileName) 173 | fd,err := os.OpenFile(fileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644) 174 | if err != nil { 175 | log.Fatal(fileName, " open fail! ", err) 176 | return 177 | } 178 | fileContent := bytes.Buffer{} 179 | i := 0 180 | for blockNumber := fromBlockNumber; blockNumber <= toBlockNumber; blockNumber++ { 181 | blockNumberHex := "0x" + decimalToAny(blockNumber, 16); 182 | command := "curl -X POST --data '{\"jsonrpc\":\"3.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"" + blockNumberHex + "\", true],\"id\":1}' -H \"Content-type: application/json;charset=UTF-8\" localhost:8545"; 183 | 184 | exec_shell(command, blockNumber, MAX_NUM_RETRY, &fileContent) 185 | if i%APPEND_FILE_NUM == 0 { 186 | if len(fileContent.String()) > 0 { 187 | buf:=[]byte(fileContent.String()) 188 | fd.Write(buf) 189 | fileContent = bytes.Buffer{} 190 | log.Print("write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 191 | } else { 192 | fileContent = bytes.Buffer{} 193 | log.Print("fileContent len <= 0, not need to write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 194 | } 195 | } 196 | i++ 197 | } 198 | if len(fileContent.String()) > 0 { 199 | buf:=[]byte(fileContent.String()) 200 | fd.Write(buf) 201 | log.Print("write accounts into file=", fileName, " taskId=", taskId, ", i=",i) 202 | } 203 | fd.Close() 204 | c <- taskId 205 | } 206 | 207 | func main() { 208 | 209 | if len(os.Args) < 3 { 210 | log.Fatal("Param Invalid!!! go run getTxByTime.go [timeFrom] [timeTo], eg. go run getTxByTime.go 2018-01-01-00-00-00 2018-02-01-00-00-00") 211 | } 212 | log.Print("====getTxByTime begin=================="); 213 | timeBegin := time.Now().Unix() 214 | MULTICORE := runtime.NumCPU() 215 | runtime.GOMAXPROCS(MULTICORE) 216 | //blockNumber := 4927600; 217 | timeFrom := os.Args[1] 218 | timeTo := os.Args[2] 219 | if timeFrom >= timeTo { 220 | log.Fatal("timeFrom=", timeFrom, " >= timeTo=", timeTo) 221 | } 222 | blockNumberBegin, blockNumberEnd := lib.GetBlockNumberByTime(timeFrom, timeTo) 223 | totalBlockNum := blockNumberEnd - blockNumberBegin + 1; 224 | share := totalBlockNum / MAX_NUM_SHARE + 1 225 | loopCount := 0 226 | dir := lib.GetAndCheckDir("tx") 227 | filePrefix := timeFrom + "-" + timeTo 228 | files := dir + "/" + filePrefix + "-*" 229 | resultFile := dir + "/" + filePrefix 230 | resultFileTmp := dir + "/"+strconv.FormatInt(timeBegin, 10)+"tmp" 231 | lib.ExecCmd("rm " + files, false) 232 | lib.ExecCmd("rm " + resultFileTmp, false) 233 | for i := blockNumberBegin; i <= blockNumberEnd; i++ { 234 | from := i 235 | if ((share+i) <= blockNumberEnd) { 236 | i += share 237 | } else { 238 | i = blockNumberEnd 239 | } 240 | to := i 241 | loopCount++ 242 | fileName := dir + "/" + filePrefix + "-"+ strconv.Itoa(loopCount) 243 | go getTxByBlock(from, to, fileName); 244 | } 245 | for i := 0; i < loopCount; i++ { 246 | taskId := <- c 247 | log.Print(taskId, " finish"); 248 | } 249 | 250 | lib.ExecCmd( "cat "+files + " >> " + resultFileTmp, true) 251 | lib.ExecCmd("rm " + files, true) 252 | lib.ExecCmd("mv " + resultFileTmp + " " + resultFile, true) 253 | lib.ExecCmd("sort -k 3 " + resultFile + " > " + resultFile + "-from-sort", true) 254 | lib.ExecCmd("sort -k 4 " + resultFile + " > " + resultFile + "-to-sort", true) 255 | 256 | 257 | timeEnd := time.Now().Unix() 258 | log.Print("getTxByBlock finish, cost=", (timeEnd - timeBegin), "s") 259 | } 260 | --------------------------------------------------------------------------------