├── .gitignore ├── bc-data-analyzer ├── bc_data_analyzer │ ├── __init__.py │ ├── blockchains │ │ ├── __init__.py │ │ ├── tezos.py │ │ └── eos.py │ ├── data │ │ ├── eos-accounts.json │ │ └── eos-categories.json │ ├── settings.py │ ├── aggregator.py │ ├── data_reader.py │ ├── commands.py │ ├── blockchain.py │ ├── cli.py │ ├── base_factory.py │ └── plot_utils.py ├── bin │ └── bc-data-analyzer ├── setup.py ├── README.md └── .gitignore ├── core ├── fixtures │ ├── invalid.json │ ├── eos-blocks-120893532--120893631.jsonl.gz │ ├── xrp-ledgers-54387273--54387372.jsonl.gz │ ├── xrp-missing-block.jsonl │ ├── xrp-ledgers-simple-format-50287874--50287973.jsonl.gz │ ├── xrp-duplicated.jsonl │ └── tezos-blocks.jsonl ├── data_test.go ├── blockchain.go ├── utils.go ├── test_helpers.go └── data.go ├── Makefile ├── .circleci └── config.yml ├── go.mod ├── .github └── workflows │ ├── go.yml │ └── release-artifacts.yml ├── tezos ├── tezos_test.go └── tezos.go ├── eos ├── eos_test.go ├── transfer.go └── eos.go ├── xrp ├── xrp_test.go ├── xrp.go └── fetcher.go ├── config ├── xrp.json ├── eos.json └── tezos.json ├── processor ├── exporter.go ├── bulk.go ├── processor_test.go └── processor.go ├── fetcher └── http.go ├── go.sum ├── README.md └── cmd └── blockchain-analyzer └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.gz 2 | /blockchain-analyzer 3 | tmp/ 4 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | from bc_data_analyzer import blockchains 2 | -------------------------------------------------------------------------------- /core/fixtures/invalid.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danhper/blockchain-analyzer/HEAD/core/fixtures/invalid.json -------------------------------------------------------------------------------- /bc-data-analyzer/bin/bc-data-analyzer: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from bc_data_analyzer import cli 4 | 5 | cli.run() 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: deps build test 2 | 3 | deps: 4 | @go get ./... 5 | 6 | build: 7 | @go build -o . ./... 8 | 9 | test: 10 | @go test ./... 11 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/blockchains/__init__.py: -------------------------------------------------------------------------------- 1 | from bc_data_analyzer.blockchains.tezos import Tezos 2 | from bc_data_analyzer.blockchains.eos import EOS 3 | -------------------------------------------------------------------------------- /core/fixtures/eos-blocks-120893532--120893631.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danhper/blockchain-analyzer/HEAD/core/fixtures/eos-blocks-120893532--120893631.jsonl.gz -------------------------------------------------------------------------------- /core/fixtures/xrp-ledgers-54387273--54387372.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danhper/blockchain-analyzer/HEAD/core/fixtures/xrp-ledgers-54387273--54387372.jsonl.gz -------------------------------------------------------------------------------- /core/fixtures/xrp-missing-block.jsonl: -------------------------------------------------------------------------------- 1 | {"result": {"ledger": {}, "ledger_index": 123}} 2 | {"result": {"ledger": {}, "ledger_index": 125}} 3 | {"result": {"ledger": {}, "ledger_index": 126}} 4 | -------------------------------------------------------------------------------- /core/fixtures/xrp-ledgers-simple-format-50287874--50287973.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danhper/blockchain-analyzer/HEAD/core/fixtures/xrp-ledgers-simple-format-50287874--50287973.jsonl.gz -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/data/eos-accounts.json: -------------------------------------------------------------------------------- 1 | { 2 | "eosio.token": "EOS token", 3 | "betdicetasks": "Gambling game", 4 | "whaleextrust": "Decentralized exchange", 5 | "pornhashbaby": "Porn website" 6 | } 7 | -------------------------------------------------------------------------------- /core/fixtures/xrp-duplicated.jsonl: -------------------------------------------------------------------------------- 1 | {"result": {"ledger": {}, "ledger_index": 123}} 2 | {"result": {"ledger": {}, "ledger_index": 124}} 3 | {"result": {"ledger": {}, "ledger_index": 124}} 4 | {"result": {"ledger": {}, "ledger_index": 125}} 5 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/settings.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | 4 | PACKAGE_NAME = "bc_data_analyzer" 5 | 6 | START_DATE = dt.datetime(2019, 10, 1, tzinfo=dt.timezone.utc) 7 | END_DATE = dt.datetime(2020, 4, 30, tzinfo=dt.timezone.utc) 8 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: circleci/golang:1.14 6 | 7 | working_directory: /go/src/github.com/danhper/blockchain-data-fetcher 8 | steps: 9 | - checkout 10 | 11 | - run: 12 | name: fetch dependencies 13 | command: go get -v -t -d ./... 14 | - run: 15 | name: run tests 16 | command: go test -v ./... 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/danhper/blockchain-analyzer 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/danhper/structomap v0.6.2 7 | github.com/fatih/structs v1.1.0 // indirect 8 | github.com/gorilla/websocket v1.4.2 9 | github.com/huandu/xstrings v1.3.1 // indirect 10 | github.com/json-iterator/go v1.1.9 11 | github.com/stretchr/testify v1.5.1 12 | github.com/ugorji/go/codec v1.1.7 13 | github.com/urfave/cli/v2 v2.2.0 14 | ) 15 | -------------------------------------------------------------------------------- /bc-data-analyzer/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name="bc-data-analyzer", 6 | packages=["bc_data_analyzer"], 7 | scripts=["bin/bc-data-analyzer"], 8 | include_package_data=True, 9 | install_requires=[ 10 | "numpy", 11 | "matplotlib", 12 | ], 13 | extras_require={ 14 | "dev": [ 15 | "pylint", 16 | "ipython", 17 | ] 18 | } 19 | ) 20 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/aggregator.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import datetime as dt 3 | from typing import List, Tuple, Dict 4 | 5 | 6 | def count_actions_over_time(actions: List[Tuple[dt.datetime, dict]]) -> Dict[str, int]: 7 | result = defaultdict(int) 8 | for _, actions_count in actions: 9 | for action in actions_count["Actions"]: 10 | result[action["Name"]] += action["Count"] 11 | return result 12 | -------------------------------------------------------------------------------- /core/data_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetActionProperty(t *testing.T) { 10 | prop, err := GetActionProperty("name") 11 | assert.Nil(t, err) 12 | assert.Equal(t, ActionName, prop) 13 | prop, err = GetActionProperty("sender") 14 | assert.Nil(t, err) 15 | assert.Equal(t, ActionSender, prop) 16 | prop, err = GetActionProperty("other") 17 | assert.NotNil(t, err) 18 | } 19 | -------------------------------------------------------------------------------- /core/blockchain.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Blockchain interface { 8 | FetchData(filepath string, start, end uint64) error 9 | ParseBlock(rawLine []byte) (Block, error) 10 | EmptyBlock() Block 11 | } 12 | 13 | type Block interface { 14 | Number() uint64 15 | TransactionsCount() int 16 | Time() time.Time 17 | ListActions() []Action 18 | } 19 | 20 | type Action interface { 21 | Sender() string 22 | Receiver() string 23 | Name() string 24 | } 25 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/data_reader.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | import json 3 | 4 | 5 | def read_actions_over_time(filename: str): 6 | with open(filename) as f: 7 | data = json.load(f) 8 | if "Results" in data and "GroupedActionsOverTime" in data["Results"]: 9 | data = data["Results"]["GroupedActionsOverTime"] 10 | actions_over_time = [] 11 | for key, value in data["Actions"].items(): 12 | parsed_time = dt.datetime.fromisoformat(key.rstrip("Z")) 13 | actions_over_time.append((parsed_time, value)) 14 | return sorted(actions_over_time, key=lambda a: a[0]) 15 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | 29 | - name: Build 30 | run: go build -v ./... 31 | 32 | - name: Test 33 | run: go test -v ./... 34 | -------------------------------------------------------------------------------- /tezos/tezos_test.go: -------------------------------------------------------------------------------- 1 | package tezos 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/danhper/blockchain-analyzer/core" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestParseBlock(t *testing.T) { 12 | rawBlock := core.ReadAllBlocks("tezos")[0] 13 | block, err := New().ParseBlock(rawBlock) 14 | 15 | assert.Nil(t, err) 16 | assert.Equal(t, uint64(10000), block.Number()) 17 | assert.Equal(t, 8, block.TransactionsCount()) 18 | 19 | expectedTime := time.Date(2018, 7, 7, 17, 06, 27, 0, time.UTC) 20 | assert.Equal(t, expectedTime, block.Time()) 21 | } 22 | 23 | func TestListActions(t *testing.T) { 24 | rawBlock := core.ReadAllBlocks("tezos")[1] 25 | block, _ := New().ParseBlock(rawBlock) 26 | actions := block.ListActions() 27 | assert.Len(t, actions, 9) 28 | } 29 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/commands.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from bc_data_analyzer.blockchain import Blockchain 4 | from bc_data_analyzer import data_reader 5 | from bc_data_analyzer import plot_utils 6 | 7 | 8 | def plot_actions_over_time(args): 9 | actions_over_time = data_reader.read_actions_over_time(args["input"]) 10 | blockchain = Blockchain.create(args["blockchain"]) 11 | labels, dates, ys, colors = blockchain.transform_actions_over_time(actions_over_time) 12 | plot_utils.plot_chart_area(labels, dates, *ys, colors=colors, filename=args["output"]) 13 | 14 | 15 | def generate_table(args): 16 | with open(args["input"]) as f: 17 | data = json.load(f) 18 | blockchain: Blockchain = Blockchain.create(args["blockchain"]) 19 | table = blockchain.generate_table(args["name"], data) 20 | print(table) 21 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/blockchain.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | import datetime as dt 3 | from typing import List, Tuple 4 | 5 | import numpy as np 6 | 7 | from bc_data_analyzer.base_factory import BaseFactory 8 | 9 | 10 | class Blockchain(BaseFactory, ABC): 11 | @abstractmethod 12 | def transform_actions_over_time(self, actions: List[Tuple[dt.datetime, dict]]) \ 13 | -> Tuple[List[dt.datetime], List[str], List[np.ndarray]]: 14 | pass 15 | 16 | def generate_table(self, table_name: str, data: dict) -> str: 17 | if table_name not in self.available_tables: 18 | raise ValueError( 19 | "unknown table type {0}, available: {1}".format( 20 | table_name, ", ".join(self.available_tables))) 21 | return self._generate_table(table_name, data) 22 | 23 | @abstractmethod 24 | def _generate_table(self, table_name: str, data: dict) -> str: 25 | pass 26 | 27 | @property 28 | @abstractmethod 29 | def available_tables(self) -> List[str]: 30 | pass 31 | -------------------------------------------------------------------------------- /eos/eos_test.go: -------------------------------------------------------------------------------- 1 | package eos 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/danhper/blockchain-analyzer/core" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestParseBlock(t *testing.T) { 12 | rawBlock := core.ReadAllBlocks("eos")[0] 13 | block, err := New().ParseBlock(rawBlock) 14 | 15 | assert.Nil(t, err) 16 | assert.Equal(t, uint64(120893628), block.Number()) 17 | assert.Equal(t, 8, block.TransactionsCount()) 18 | expectedTime := time.Date(2020, time.Month(5), 16, 0, 10, 43, 0, time.UTC) 19 | assert.Equal(t, expectedTime, block.Time()) 20 | } 21 | 22 | func TestParseBlockWithoutTrx(t *testing.T) { 23 | rawBlock := core.ReadAllBlocks("eos")[3] 24 | block, err := New().ParseBlock(rawBlock) 25 | 26 | assert.Nil(t, err) 27 | assert.Equal(t, uint64(120893629), block.Number()) 28 | assert.Equal(t, 10, block.TransactionsCount()) 29 | } 30 | 31 | func TestListActions(t *testing.T) { 32 | rawBlock := core.ReadAllBlocks("eos")[0] 33 | block, _ := New().ParseBlock(rawBlock) 34 | actions := block.ListActions() 35 | assert.Len(t, actions, 176) 36 | } 37 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from bc_data_analyzer import commands 4 | 5 | 6 | parser = argparse.ArgumentParser(prog="data-analyzer") 7 | parser.add_argument("-b", "--blockchain", required=True, 8 | help="Which blockchain to use") 9 | 10 | subparsers = parser.add_subparsers(dest="command") 11 | 12 | plot_action_over_time = subparsers.add_parser("plot-actions-over-time", help="Plot actions over time") 13 | plot_action_over_time.add_argument("input", help="Input file containing actions over time") 14 | plot_action_over_time.add_argument("-o", "--output", help="Output file") 15 | 16 | generate_table = subparsers.add_parser("generate-table", help="Generate a table from the data") 17 | generate_table.add_argument("input", help="Input file containing results") 18 | generate_table.add_argument("-n", "--name", help="Name of the table to generate", required=True) 19 | 20 | 21 | def run(): 22 | args = vars(parser.parse_args()) 23 | if not args["command"]: 24 | parser.error("no command given") 25 | 26 | func = getattr(commands, args["command"].replace("-", "_")) 27 | func(args) 28 | -------------------------------------------------------------------------------- /xrp/xrp_test.go: -------------------------------------------------------------------------------- 1 | package xrp 2 | 3 | import ( 4 | "bufio" 5 | "testing" 6 | "time" 7 | 8 | "github.com/danhper/blockchain-analyzer/core" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParseRawLedger(t *testing.T) { 13 | rawLedger := core.ReadAllBlocks("xrp")[0] 14 | ledger, err := ParseRawLedger(rawLedger) 15 | 16 | assert.Nil(t, err) 17 | assert.Equal(t, uint64(54387329), ledger.Number()) 18 | assert.Equal(t, 33, ledger.TransactionsCount()) 19 | expectedTime := time.Date(2020, 3, 27, 20, 52, 50, 0, time.UTC) 20 | assert.Equal(t, expectedTime, ledger.Time()) 21 | } 22 | 23 | func TestParseRawLedgerSimpleFormat(t *testing.T) { 24 | reader := core.GetFixtureReader(core.XRPSimpleValidLedgersFilename) 25 | defer reader.Close() 26 | rawLedger, err := bufio.NewReader(reader).ReadBytes('\n') 27 | assert.Nil(t, err) 28 | ledger, err := ParseRawLedger(rawLedger) 29 | assert.Nil(t, err) 30 | assert.Equal(t, uint64(50387844), ledger.Number()) 31 | } 32 | 33 | func TestListActions(t *testing.T) { 34 | rawLedger := core.ReadAllBlocks("xrp")[0] 35 | ledger, _ := ParseRawLedger(rawLedger) 36 | actions := ledger.ListActions() 37 | assert.Len(t, actions, 33) 38 | } 39 | -------------------------------------------------------------------------------- /bc-data-analyzer/README.md: -------------------------------------------------------------------------------- 1 | # bc-data-analyzer 2 | 3 | This is a set of Python script to analyze the data produced by `blockchain-analyzer`. 4 | 5 | ## Installation 6 | 7 | This repository can be installed directly from GitHub using the following command: 8 | 9 | ``` 10 | pip install 'git+https://github.com/danhper/blockchain-analyzer#subdirector 11 | y=bc-data-analyzer' 12 | ``` 13 | 14 | or after a git clone, by running 15 | 16 | ``` 17 | cd bc-data-analyzer 18 | pip install . 19 | ``` 20 | 21 | ## Usage 22 | 23 | The entrypoint is the CLI command `bc-data-analyzer` 24 | It takes as input a file outputted by the `bulk-process` command of `blockchain-analyzer` 25 | and can be use to plot or produce LaTeX tables. 26 | 27 | For example, to plot the chart area of the distribution of actions, the following 28 | command can be used: 29 | 30 | ``` 31 | bc-data-analyzer -b eos plot-actions-over-time /path/to/eos-results.json 32 | ``` 33 | 34 | or to generate a table with the top senders: 35 | 36 | ``` 37 | bc-data-analyzer -b tezos generate-table -n top-senders /path/to/tezos-results.json 38 | ``` 39 | 40 | More information on the different options can be obtained by using the help command 41 | 42 | ``` 43 | bc-data-analyzer -h 44 | ``` 45 | -------------------------------------------------------------------------------- /config/xrp.json: -------------------------------------------------------------------------------- 1 | { 2 | "Pattern": "/mnt/quantum/dp4318/research-data/xrp/xrp-ledgers-*.jsonl.gz", 3 | "StartBlock": 50399027, 4 | "EndBlock": 55152991, 5 | "Processors": [ 6 | { 7 | "Name": "TransactionsCount", 8 | "Type": "count-transactions" 9 | }, 10 | { 11 | "Name": "TransactionsCountOverTime", 12 | "Type": "count-transactions-over-time", 13 | "Params": { 14 | "Duration": "6h" 15 | } 16 | }, 17 | { 18 | "Name": "GroupedActionsOverTime", 19 | "Type": "group-actions-over-time", 20 | "Params": { 21 | "By": "name", 22 | "Duration": "6h" 23 | } 24 | }, 25 | { 26 | "Name": "ActionsByName", 27 | "Type": "group-actions", 28 | "Params": { 29 | "By": "name", 30 | "Detailed": false 31 | } 32 | }, 33 | { 34 | "Name": "ActionsBySender", 35 | "Type": "group-actions", 36 | "Params": { 37 | "By": "sender", 38 | "Detailed": true 39 | } 40 | }, 41 | { 42 | "Name": "ActionsByReceiver", 43 | "Type": "group-actions", 44 | "Params": { 45 | "By": "receiver", 46 | "Detailed": true 47 | } 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /config/eos.json: -------------------------------------------------------------------------------- 1 | { 2 | "Pattern": "/mnt/quantum/dp4318/research-data/eos/eos_blocks-*.jsonl.gz", 3 | "StartBlock": 82152667, 4 | "EndBlock": 118286375, 5 | "Processors": [ 6 | { 7 | "Name": "TransactionsCount", 8 | "Type": "count-transactions" 9 | }, 10 | { 11 | "Name": "TransactionsCountOverTime", 12 | "Type": "count-transactions-over-time", 13 | "Params": { 14 | "Duration": "6h" 15 | } 16 | }, 17 | { 18 | "Name": "GroupedActionsOverTime", 19 | "Type": "group-actions-over-time", 20 | "Params": { 21 | "By": "receiver", 22 | "Duration": "6h" 23 | } 24 | }, 25 | { 26 | "Name": "ActionsByName", 27 | "Type": "group-actions", 28 | "Params": { 29 | "By": "name", 30 | "Detailed": false 31 | } 32 | }, 33 | { 34 | "Name": "ActionsBySender", 35 | "Type": "group-actions", 36 | "Params": { 37 | "By": "sender", 38 | "Detailed": true 39 | } 40 | }, 41 | { 42 | "Name": "ActionsByReceiver", 43 | "Type": "group-actions", 44 | "Params": { 45 | "By": "receiver", 46 | "Detailed": true 47 | } 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /config/tezos.json: -------------------------------------------------------------------------------- 1 | { 2 | "Pattern": "/mnt/data/daniel/research-data/transactional-throughput/tezos/*.jsonl.gz", 3 | "StartBlock": 630709, 4 | "EndBlock": 932530, 5 | "Processors": [ 6 | { 7 | "Name": "TransactionsCount", 8 | "Type": "count-transactions" 9 | }, 10 | { 11 | "Name": "TransactionsCountOverTime", 12 | "Type": "count-transactions-over-time", 13 | "Params": { 14 | "Duration": "6h" 15 | } 16 | }, 17 | { 18 | "Name": "GroupedActionsOverTime", 19 | "Type": "group-actions-over-time", 20 | "Params": { 21 | "By": "name", 22 | "Duration": "6h" 23 | } 24 | }, 25 | { 26 | "Name": "ActionsByName", 27 | "Type": "group-actions", 28 | "Params": { 29 | "By": "name", 30 | "Detailed": false 31 | } 32 | }, 33 | { 34 | "Name": "ActionsBySender", 35 | "Type": "group-actions", 36 | "Params": { 37 | "By": "sender", 38 | "Detailed": true 39 | } 40 | }, 41 | { 42 | "Name": "ActionsByReceiver", 43 | "Type": "group-actions", 44 | "Params": { 45 | "By": "receiver", 46 | "Detailed": true 47 | } 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /core/utils.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "sort" 10 | "strings" 11 | ) 12 | 13 | const ( 14 | BatchSize uint64 = 100000 15 | ) 16 | 17 | func MakeFilename(filePath string, first, last uint64) string { 18 | splitted := strings.SplitN(filePath, ".", 2) 19 | return fmt.Sprintf("%s-%d--%d.%s", splitted[0], first, last, splitted[1]) 20 | } 21 | 22 | func MakeErrFilename(filePath string, first, last uint64) string { 23 | splitted := strings.SplitN(filePath, ".", 2) 24 | return fmt.Sprintf("%s-%d--%d-errors.%s", splitted[0], first, last, splitted[1]) 25 | } 26 | 27 | func CreateFile(name string) (io.WriteCloser, error) { 28 | file, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) 29 | if err != nil { 30 | return nil, err 31 | } 32 | if strings.HasSuffix(name, ".gz") { 33 | return gzip.NewWriter(file), nil 34 | } 35 | return file, nil 36 | } 37 | 38 | func OpenFile(name string) (io.ReadCloser, error) { 39 | file, err := os.Open(name) 40 | if err != nil { 41 | return nil, err 42 | } 43 | if strings.HasSuffix(name, ".gz") { 44 | return gzip.NewReader(file) 45 | } 46 | return file, nil 47 | } 48 | 49 | func SortU64Slice(values []uint64) { 50 | sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) 51 | } 52 | 53 | func MakeFileProcessor(f func(string) error) func(string) { 54 | return func(filename string) { 55 | log.Printf("processing %s", filename) 56 | if err := f(filename); err != nil { 57 | log.Printf("error while processing %s: %s", filename, err.Error()) 58 | } else { 59 | log.Printf("done processing %s", filename) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/base_factory.py: -------------------------------------------------------------------------------- 1 | from functools import lru_cache 2 | from abc import ABCMeta 3 | 4 | 5 | class FactoryMeta(ABCMeta): 6 | @property 7 | def _entities(cls): 8 | if not hasattr(cls, "_registered_entities"): 9 | cls._registered_entities = {} 10 | return cls._registered_entities 11 | 12 | 13 | class BaseFactory(metaclass=FactoryMeta): 14 | @classmethod 15 | def registered(cls): 16 | return list(cls._entities.keys()) 17 | 18 | @classmethod 19 | def register(cls, name: str): 20 | """decorator to register a class to the factory 21 | Should be used as follows: 22 | 23 | .. code:: python 24 | @FactorySubClass.register("name") 25 | class SubClassImpl: 26 | pass 27 | 28 | :param name: name with which the entity should be accessed 29 | """ 30 | def wrapper(klass): 31 | cls._entities[name] = klass 32 | klass.__registered_name__ = name 33 | return klass 34 | return wrapper 35 | 36 | @classmethod 37 | def get(cls, name: str): 38 | """gets an entity from the factory by named 39 | 40 | :param name: name of the entity 41 | :return: the entity class 42 | :raise ValueError: if the entity does not exist 43 | """ 44 | if name not in cls._entities: 45 | raise ValueError("{0} not registered".format(name)) 46 | return cls._entities[name] 47 | 48 | @classmethod 49 | def create(cls, name: str, *args, **kwargs): 50 | """creates an entity 51 | 52 | :param name: name of the entity 53 | :return: the entity instance 54 | """ 55 | return cls.get(name)(*args, **kwargs) 56 | -------------------------------------------------------------------------------- /processor/exporter.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "log" 5 | "path" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/danhper/blockchain-analyzer/core" 11 | "github.com/ugorji/go/codec" 12 | ) 13 | 14 | func ExportToMsgpack( 15 | blockchain core.Blockchain, 16 | globPattern string, 17 | start, end uint64, 18 | outputDir string, 19 | ) error { 20 | files, err := filepath.Glob(globPattern) 21 | if err != nil { 22 | return err 23 | } 24 | processed := 0 25 | fileDone := make(chan bool) 26 | var wg sync.WaitGroup 27 | 28 | exportFile := core.MakeFileProcessor(func(filename string) error { 29 | defer wg.Done() 30 | reader, err := core.OpenFile(filename) 31 | if err != nil { 32 | return err 33 | } 34 | defer reader.Close() 35 | 36 | outputFilename := strings.Replace(path.Base(filename), "jsonl", "dat", 1) 37 | outputFilepath := path.Join(outputDir, outputFilename) 38 | writer, err := core.CreateFile(outputFilepath) 39 | if err != nil { 40 | return err 41 | } 42 | defer writer.Close() 43 | 44 | for block := range YieldBlocks(reader, blockchain, JSONFormat) { 45 | if (start == 0 || block.Number() >= start) && 46 | (end == 0 || block.Number() <= end) { 47 | var rawBlock []byte 48 | enc := codec.NewEncoderBytes(&rawBlock, msgpackHandle) 49 | if err := enc.Encode(block); err != nil { 50 | return err 51 | } 52 | writer.Write(rawBlock) 53 | } 54 | } 55 | fileDone <- true 56 | return err 57 | }) 58 | 59 | go func() { 60 | for range fileDone { 61 | processed++ 62 | log.Printf("files processed: %d/%d", processed, len(files)) 63 | } 64 | }() 65 | 66 | log.Printf("exporting %d files", len(files)) 67 | 68 | for _, filename := range files { 69 | wg.Add(1) 70 | go exportFile(filename) 71 | } 72 | 73 | wg.Wait() 74 | close(fileDone) 75 | 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /core/test_helpers.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "path" 8 | "runtime" 9 | ) 10 | 11 | const ( 12 | XRPValidLedgersFilename string = "xrp-ledgers-54387273--54387372.jsonl.gz" 13 | XRPSimpleValidLedgersFilename string = "xrp-ledgers-simple-format-50287874--50287973.jsonl.gz" 14 | XRPMissingLedgersFilename string = "xrp-missing-block.jsonl" 15 | XRPDuplicatedLedgersFilename string = "xrp-duplicated.jsonl" 16 | 17 | EOSValidBlocksFilename string = "eos-blocks-120893532--120893631.jsonl.gz" 18 | TezosValidBlocksFilename string = "tezos-blocks.jsonl" 19 | ) 20 | 21 | func GetFixturesPath() string { 22 | _, filename, _, _ := runtime.Caller(0) 23 | return path.Join(path.Dir(filename), "fixtures") 24 | } 25 | 26 | func GetFixture(filename string) string { 27 | return path.Join(GetFixturesPath(), filename) 28 | } 29 | 30 | func GetFixtureReader(filename string) io.ReadCloser { 31 | reader, err := OpenFile(GetFixture(filename)) 32 | if err != nil { 33 | panic(err) 34 | } 35 | return reader 36 | } 37 | 38 | func ReadAllBlocks(blockchainName string) [][]byte { 39 | var filename string 40 | switch blockchainName { 41 | case "eos": 42 | filename = EOSValidBlocksFilename 43 | case "xrp": 44 | filename = XRPValidLedgersFilename 45 | case "tezos": 46 | filename = TezosValidBlocksFilename 47 | default: 48 | panic("invalid blockchain: " + blockchainName) 49 | } 50 | reader := GetFixtureReader(filename) 51 | defer reader.Close() 52 | content, err := ioutil.ReadAll(reader) 53 | if err != nil { 54 | panic(err) 55 | } 56 | return bytes.Split(content, []byte{'\n'}) 57 | } 58 | 59 | func ReadAllEOSBlocks() [][]byte { 60 | reader := GetFixtureReader(EOSValidBlocksFilename) 61 | defer reader.Close() 62 | content, err := ioutil.ReadAll(reader) 63 | if err != nil { 64 | panic(err) 65 | } 66 | return bytes.Split(content, []byte{'\n'}) 67 | } 68 | -------------------------------------------------------------------------------- /.github/workflows/release-artifacts.yml: -------------------------------------------------------------------------------- 1 | name: Generate release-artifacts 2 | 3 | # on events 4 | on: 5 | release: 6 | types: 7 | - created 8 | 9 | jobs: 10 | generate: 11 | name: Generate cross-platform builds 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout the repository 15 | uses: actions/checkout@v2 16 | - name: Generate build files 17 | uses: thatisuday/go-cross-build@v1 18 | with: 19 | platforms: 'linux/amd64, darwin/amd64, windows/amd64' 20 | package: 'cmd/blockchain-analyzer' 21 | name: 'blockchain-analyzer' 22 | compress: 'true' 23 | dest: 'dist' 24 | - name: Upload Windows binary 25 | uses: actions/upload-release-asset@v1 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | with: 29 | upload_url: ${{ github.event.release.upload_url }} 30 | asset_path: dist/blockchain-analyzer-windows-amd64.tar.gz 31 | asset_name: blockchain-analyzer-windows-amd64.tar.gz 32 | asset_content_type: application/gzip 33 | - name: Upload Linux binary 34 | uses: actions/upload-release-asset@v1 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | with: 38 | upload_url: ${{ github.event.release.upload_url }} 39 | asset_path: dist/blockchain-analyzer-linux-amd64.tar.gz 40 | asset_name: blockchain-analyzer-linux-amd64.tar.gz 41 | asset_content_type: application/gzip 42 | - name: Upload macOS binary 43 | uses: actions/upload-release-asset@v1 44 | env: 45 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 46 | with: 47 | upload_url: ${{ github.event.release.upload_url }} 48 | asset_path: dist/blockchain-analyzer-darwin-amd64.tar.gz 49 | asset_name: blockchain-analyzer-darwin-amd64.tar.gz 50 | asset_content_type: application/gzip 51 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/data/eos-categories.json: -------------------------------------------------------------------------------- 1 | { 2 | "categories": [ 3 | {"name": "tokens", "color": "blue"}, 4 | {"name": "betting", "color": "gray"}, 5 | {"name": "exchange", "color": "green"}, 6 | {"name": "games", "color": "teal"}, 7 | {"name": "pornography", "color": "violet"}, 8 | {"name": "others", "color": "brown"} 9 | ], 10 | "mapping": { 11 | "betdicegroup": "betting", 12 | "betdicetasks": "betting", 13 | "eosio.token": "tokens", 14 | "pornhashbaby": "pornography", 15 | "eosplayaloud": "others", 16 | "bluebetbcrat": "betting", 17 | "wukongmarket": "games", 18 | "bluebetproxy": "betting", 19 | "lynxtoken123": "tokens", 20 | "bluebet2user": "betting", 21 | "bulls.bg": "betting", 22 | "whaleextrust": "exchange", 23 | "newdexpublic": "exchange", 24 | "bluebetbulls": "betting", 25 | "ipsecontract": "others", 26 | "eossanguoone": "games", 27 | "walvalidator": "exchange", 28 | "ipsouminer11": "others", 29 | "eoshashhouse": "betting", 30 | "bluebetjacks": "betting", 31 | "betdiceadmin": "betting", 32 | "bluebettexas": "betting", 33 | "pokerwar.bg": "betting", 34 | "wallet.bg": "others", 35 | "skrblackjack": "betting", 36 | "skrxlotteryx": "betting", 37 | "skrxeosxdice": "betting", 38 | "tethertether": "tokens", 39 | "ghtclubtoken": "tokens", 40 | "betdicebacca": "betting", 41 | "betdicesicbo": "betting", 42 | "sanguoserver": "games", 43 | "ipsouipfseos": "others", 44 | "bountyblokio": "tokens", 45 | "krowndactokn": "tokens", 46 | "krownairdrop": "tokens", 47 | "bluebetrobot": "betting", 48 | "whaleexgate3": "exchange", 49 | "vegasdealer1": "betting", 50 | "ghtclubbank1": "others", 51 | "eoshashdices": "betting", 52 | "eosio.null": "others", 53 | "eosknightsio": "games", 54 | "prospectorsc": "games", 55 | "phwvxs3lbdn5": "others", 56 | "deposoracle1": "others", 57 | "thedeposbank": "others", 58 | "eoshashbulls": "betting", 59 | "dice.bg": "betting" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /bc-data-analyzer/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/python 3 | # Edit at https://www.gitignore.io/?templates=python 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # pyenv 72 | .python-version 73 | 74 | # pipenv 75 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 76 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 77 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 78 | # install all needed dependencies. 79 | #Pipfile.lock 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | .spyproject 90 | 91 | # Rope project settings 92 | .ropeproject 93 | 94 | # Mr Developer 95 | .mr.developer.cfg 96 | .project 97 | .pydevproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | .dmypy.json 105 | dmypy.json 106 | 107 | # Pyre type checker 108 | .pyre/ 109 | 110 | # End of https://www.gitignore.io/api/python 111 | -------------------------------------------------------------------------------- /eos/transfer.go: -------------------------------------------------------------------------------- 1 | package eos 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/danhper/blockchain-analyzer/core" 11 | "github.com/danhper/blockchain-analyzer/processor" 12 | ) 13 | 14 | type TransferData struct { 15 | From string 16 | To string 17 | Quantity string 18 | Memo string 19 | } 20 | 21 | func parseTransferQuantity(rawQuantity string) (string, string, error) { 22 | tokens := strings.Split(rawQuantity, " ") 23 | if len(tokens) != 2 { 24 | return "", "", fmt.Errorf("expected 2 tokens, got %d", len(tokens)) 25 | } 26 | if _, err := strconv.ParseFloat(tokens[0], 64); err != nil { 27 | return "", "", fmt.Errorf("quantity %s was not a valid float", tokens[0]) 28 | } 29 | return tokens[0], tokens[1], nil 30 | } 31 | 32 | func ExportTransfers(globPattern string, start, end uint64, output string) error { 33 | writer, err := core.CreateFile(output) 34 | if err != nil { 35 | return err 36 | } 37 | defer writer.Close() 38 | csvWriter := csv.NewWriter(writer) 39 | 40 | headers := []string{ 41 | "block", 42 | "tx", 43 | "account", 44 | "symbol", 45 | "from", 46 | "to", 47 | "quantity", 48 | "memo", 49 | } 50 | if err := csvWriter.Write(headers); err != nil { 51 | return err 52 | } 53 | 54 | blocks, err := processor.YieldAllBlocks(globPattern, New(), start, end) 55 | if err != nil { 56 | return err 57 | } 58 | for block := range blocks { 59 | eosBlock, ok := block.(*Block) 60 | if !ok { 61 | return err 62 | } 63 | 64 | for _, transaction := range eosBlock.Transactions { 65 | for _, action := range transaction.Trx.Transaction.Actions { 66 | if action.ActionName != "transfer" { 67 | continue 68 | } 69 | var transferData TransferData 70 | if err := fastJson.Unmarshal(action.Data, &transferData); err != nil { 71 | continue 72 | } 73 | quantity, symbol, err := parseTransferQuantity(transferData.Quantity) 74 | if err != nil { 75 | continue 76 | } 77 | 78 | row := []string{ 79 | strconv.FormatUint(block.Number(), 10), 80 | transaction.Trx.Id, 81 | action.Account, 82 | symbol, 83 | transferData.From, 84 | transferData.To, 85 | quantity, 86 | transferData.Memo, 87 | } 88 | if err = csvWriter.Write(row); err != nil { 89 | log.Printf("could not write row: %s", err.Error()) 90 | } 91 | } 92 | } 93 | } 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /xrp/xrp.go: -------------------------------------------------------------------------------- 1 | package xrp 2 | 3 | import ( 4 | "time" 5 | 6 | jsoniter "github.com/json-iterator/go" 7 | 8 | "github.com/danhper/blockchain-analyzer/core" 9 | ) 10 | 11 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 12 | 13 | const rippleEpochOffset int64 = 946684800 14 | 15 | type XRP struct { 16 | } 17 | 18 | func New() *XRP { 19 | return &XRP{} 20 | } 21 | 22 | type Transaction struct { 23 | Account string 24 | TransactionType string 25 | Destination string 26 | } 27 | 28 | type Ledger struct { 29 | Index uint64 `json:"-"` 30 | CloseTimestamp int64 `json:"close_time"` 31 | parsedCloseTime time.Time 32 | Transactions []Transaction 33 | } 34 | 35 | type xrpLedger struct { 36 | LedgerIndex uint64 `json:"ledger_index"` 37 | Ledger Ledger 38 | Validated bool 39 | } 40 | 41 | type xrpLedgerResponse struct { 42 | Result xrpLedger 43 | } 44 | 45 | func ParseRawLedger(rawLedger []byte) (*Ledger, error) { 46 | var response xrpLedgerResponse 47 | if err := json.Unmarshal(rawLedger, &response); err != nil { 48 | return nil, err 49 | } 50 | result := response.Result 51 | if result.LedgerIndex == uint64(0) && !result.Validated { 52 | if err := json.Unmarshal(rawLedger, &result); err != nil { 53 | return nil, err 54 | } 55 | } 56 | ledger := result.Ledger 57 | ledger.parsedCloseTime = time.Unix(ledger.CloseTimestamp+rippleEpochOffset, 0).UTC() 58 | ledger.Index = result.LedgerIndex 59 | return &ledger, nil 60 | } 61 | 62 | func (x *XRP) ParseBlock(rawLine []byte) (core.Block, error) { 63 | return ParseRawLedger(rawLine) 64 | } 65 | 66 | func (x *XRP) EmptyBlock() core.Block { 67 | return &Ledger{} 68 | } 69 | 70 | func (x *XRP) FetchData(filepath string, start, end uint64) error { 71 | return fetchXRPData(filepath, start, end) 72 | } 73 | 74 | func (l *Ledger) Number() uint64 { 75 | return l.Index 76 | } 77 | 78 | func (l *Ledger) Time() time.Time { 79 | return l.parsedCloseTime 80 | } 81 | 82 | func (l *Ledger) TransactionsCount() int { 83 | return len(l.Transactions) 84 | } 85 | 86 | func (l *Ledger) ListActions() []core.Action { 87 | var actions []core.Action 88 | for _, t := range l.Transactions { 89 | actions = append(actions, t) 90 | } 91 | return actions 92 | } 93 | 94 | func (t Transaction) Sender() string { 95 | return t.Account 96 | } 97 | 98 | func (t Transaction) Receiver() string { 99 | return t.Destination 100 | } 101 | 102 | func (t Transaction) Name() string { 103 | return t.TransactionType 104 | } 105 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/plot_utils.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | import datetime as dt 3 | 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | import matplotlib.dates as mdates 7 | import matplotlib 8 | 9 | from bc_data_analyzer import settings 10 | 11 | 12 | COLORS = { 13 | "green": "olivedrab", 14 | "blue": "cornflowerblue", 15 | "brown": "darkgoldenrod", 16 | "gray": "darkgray", 17 | "violet": "violet", 18 | "coral": "lightcoral", 19 | "teal": "teal", 20 | } 21 | 22 | 23 | def make_palette(*colors): 24 | return [COLORS[c] for c in colors] 25 | 26 | 27 | def show_plot(filename): 28 | if filename is None: 29 | plt.show() 30 | else: 31 | plt.savefig(filename) 32 | 33 | 34 | def find_range(dates): 35 | start_date, end_date = settings.START_DATE, settings.END_DATE 36 | if dates[0].tzinfo is None: 37 | start_date, end_date = start_date.replace(tzinfo=None), end_date.replace(tzinfo=None) 38 | start_index = bisect.bisect_right(dates, start_date) - 1 39 | end_index = bisect.bisect_left(dates, end_date) 40 | return max(start_index, 0), end_index 41 | 42 | 43 | def adjust_series(x, ys): 44 | if len(x) and isinstance(x[0], np.datetime64): 45 | x = [dt.datetime.utcfromtimestamp(v / 1e9) for v in x.tolist()] 46 | start_index, end_index = find_range(x) 47 | x = x[start_index:end_index] 48 | ys = [y[start_index:end_index] for y in ys] 49 | return x, ys 50 | 51 | 52 | def plot_chart_area(labels, x, *ys, filename=None, **kwargs): 53 | matplotlib.rc("font", size=14) 54 | x, ys = adjust_series(x, ys) 55 | 56 | fig, ax = plt.subplots(figsize=(10, 7)) 57 | if "ylim" in kwargs: 58 | plt.ylim(top=kwargs.pop("ylim")) 59 | plt.xticks(rotation=45) 60 | plt.setp(ax.xaxis.get_majorticklabels(), ha="right") 61 | ax.set_ylabel("Number of Actions") 62 | ax.stackplot(x, *ys, labels=labels, **kwargs) 63 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y-%m-%d")) 64 | ax.ticklabel_format(scilimits=(0, 0), axis="y") 65 | plt.legend(loc="upper left") 66 | plt.tight_layout() 67 | show_plot(filename) 68 | 69 | 70 | def plot_transaction_volume(x, y_tx_count, y_amount_usd, filename=None): 71 | x, (y_tx_count, y_amount_usd) = adjust_series(x, [y_tx_count, y_amount_usd]) 72 | 73 | _fig, ax = plt.subplots() 74 | plt.xticks(rotation=45) 75 | plt.setp(ax.xaxis.get_majorticklabels(), ha="right") 76 | ax.plot(x, y_tx_count, color=COLORS["blue"]) 77 | ax2 = ax.twinx() 78 | ax2.plot(x, y_amount_usd, color=COLORS["green"]) 79 | ax.figure.legend(["Transactions count", "USD volume"], 80 | bbox_to_anchor=(1.0, 0.1), frameon=False) 81 | ax2.set_yscale("log") 82 | ax.set_xlabel("Time") 83 | ax.set_ylabel("Transactions count") 84 | ax2.set_ylabel("USD volume") 85 | ax.xaxis.set_major_formatter(mdates.DateFormatter("%m-%d")) 86 | plt.tight_layout() 87 | if filename: 88 | plt.savefig(filename) 89 | else: 90 | plt.show() 91 | -------------------------------------------------------------------------------- /tezos/tezos.go: -------------------------------------------------------------------------------- 1 | package tezos 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "time" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/danhper/blockchain-analyzer/core" 12 | "github.com/danhper/blockchain-analyzer/fetcher" 13 | ) 14 | 15 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 16 | 17 | const defaultRPCEndpoint string = "https://api.tezos.org.ua" 18 | 19 | type Tezos struct { 20 | RPCEndpoint string 21 | } 22 | 23 | func (t *Tezos) makeRequest(client *http.Client, blockNumber uint64) (*http.Response, error) { 24 | url := fmt.Sprintf("%s/chains/main/blocks/%d", t.RPCEndpoint, blockNumber) 25 | return client.Get(url) 26 | } 27 | 28 | func (t *Tezos) FetchData(filepath string, start, end uint64) error { 29 | context := fetcher.NewHTTPContext(start, end, t.makeRequest) 30 | return fetcher.FetchHTTPData(filepath, context) 31 | } 32 | 33 | type Content struct { 34 | Kind string 35 | Source string 36 | Destination string 37 | Amount string 38 | } 39 | 40 | type Operation struct { 41 | Hash string 42 | Contents []Content 43 | } 44 | 45 | type BlockHeader struct { 46 | Level uint64 47 | Timestamp string 48 | ParsedTimestamp time.Time 49 | } 50 | 51 | type Block struct { 52 | Header BlockHeader 53 | Operations [][]Operation 54 | actions []core.Action 55 | } 56 | 57 | func New() *Tezos { 58 | rpcEndpoint := os.Getenv("TEZOS_RPC_ENDPOINT") 59 | if rpcEndpoint == "" { 60 | rpcEndpoint = defaultRPCEndpoint 61 | } 62 | 63 | return &Tezos{ 64 | RPCEndpoint: rpcEndpoint, 65 | } 66 | } 67 | 68 | func (t *Tezos) ParseBlock(rawLine []byte) (core.Block, error) { 69 | var block Block 70 | if err := json.Unmarshal(rawLine, &block); err != nil { 71 | return nil, err 72 | } 73 | parsedTime, err := time.Parse(time.RFC3339, block.Header.Timestamp) 74 | if err != nil { 75 | return nil, err 76 | } 77 | block.Header.ParsedTimestamp = parsedTime 78 | return &block, nil 79 | } 80 | 81 | func (t *Tezos) EmptyBlock() core.Block { 82 | return &Block{} 83 | } 84 | 85 | func (b *Block) Number() uint64 { 86 | return b.Header.Level 87 | } 88 | 89 | func (b *Block) Time() time.Time { 90 | return b.Header.ParsedTimestamp 91 | } 92 | 93 | func (b *Block) TransactionsCount() int { 94 | total := 0 95 | for _, operations := range b.Operations { 96 | total += len(operations) 97 | } 98 | return total 99 | } 100 | 101 | func (b *Block) ListActions() []core.Action { 102 | if len(b.actions) > 0 { 103 | return b.actions 104 | } 105 | var result []core.Action 106 | for _, operations := range b.Operations { 107 | for _, operation := range operations { 108 | for _, content := range operation.Contents { 109 | result = append(result, content) 110 | } 111 | } 112 | } 113 | b.actions = result 114 | return result 115 | } 116 | 117 | func (c Content) Name() string { 118 | return c.Kind 119 | } 120 | 121 | func (c Content) Receiver() string { 122 | return c.Destination 123 | } 124 | 125 | func (c Content) Sender() string { 126 | return c.Source 127 | } 128 | -------------------------------------------------------------------------------- /processor/bulk.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/danhper/blockchain-analyzer/core" 8 | ) 9 | 10 | type Aggregator interface { 11 | AddBlock(block core.Block) 12 | Result() interface{} 13 | } 14 | 15 | type Processor struct { 16 | Aggregator Aggregator 17 | Name string 18 | } 19 | 20 | func NewProcessor(name string, aggregator Aggregator) Processor { 21 | return Processor{ 22 | Aggregator: aggregator, 23 | Name: name, 24 | } 25 | } 26 | 27 | type groupActionsParams struct { 28 | By core.ActionProperty 29 | Detailed bool 30 | } 31 | 32 | type groupActionsOverTimeParams struct { 33 | By core.ActionProperty 34 | Duration core.Duration 35 | } 36 | 37 | type countTransactionsOverTimeParams struct { 38 | Duration core.Duration 39 | } 40 | 41 | type BulkConfig struct { 42 | Pattern string 43 | StartBlock uint64 44 | EndBlock uint64 45 | RawProcessors []struct { 46 | Name string 47 | Type string 48 | Params json.RawMessage 49 | } `json:"Processors"` 50 | Processors []Processor `json:"-"` 51 | } 52 | 53 | func (c *BulkConfig) UnmarshalJSON(data []byte) error { 54 | type rawConfig BulkConfig 55 | if err := json.Unmarshal(data, (*rawConfig)(c)); err != nil { 56 | return err 57 | } 58 | for _, rawProcessor := range c.RawProcessors { 59 | var aggregator Aggregator 60 | switch rawProcessor.Type { 61 | case "group-actions": 62 | var params groupActionsParams 63 | if err := json.Unmarshal(rawProcessor.Params, ¶ms); err != nil { 64 | return err 65 | } 66 | aggregator = core.NewGroupedActions(params.By, params.Detailed) 67 | 68 | case "count-transactions": 69 | aggregator = core.NewTransactionCounter() 70 | 71 | case "count-transactions-over-time": 72 | var params countTransactionsOverTimeParams 73 | if err := json.Unmarshal(rawProcessor.Params, ¶ms); err != nil { 74 | return err 75 | } 76 | aggregator = core.NewTimeGroupedTransactionCount(params.Duration.Duration) 77 | 78 | case "group-actions-over-time": 79 | var params groupActionsOverTimeParams 80 | if err := json.Unmarshal(rawProcessor.Params, ¶ms); err != nil { 81 | return err 82 | } 83 | aggregator = core.NewTimeGroupedActions(params.Duration.Duration, params.By) 84 | 85 | default: 86 | return fmt.Errorf("unknown processor %s", rawProcessor.Name) 87 | } 88 | processor := NewProcessor(rawProcessor.Name, aggregator) 89 | c.Processors = append(c.Processors, processor) 90 | } 91 | return nil 92 | } 93 | 94 | func RunBulkActions(blockchain core.Blockchain, config BulkConfig) (map[string]interface{}, error) { 95 | missingBlockProcessor := NewProcessor("MissingBlocks", core.NewMissingBlocks(config.StartBlock, config.EndBlock)) 96 | config.Processors = append(config.Processors, missingBlockProcessor) 97 | blocks, err := YieldAllBlocks(config.Pattern, blockchain, config.StartBlock, config.EndBlock) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | for block := range blocks { 103 | for _, processor := range config.Processors { 104 | processor.Aggregator.AddBlock(block) 105 | } 106 | } 107 | 108 | result := make(map[string]interface{}) 109 | result["Config"] = config 110 | processorResults := make(map[string]interface{}) 111 | for _, processor := range config.Processors { 112 | processorResults[processor.Name] = processor.Aggregator.Result() 113 | } 114 | result["Results"] = processorResults 115 | return result, nil 116 | } 117 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/blockchains/tezos.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | from typing import List, Tuple 3 | 4 | import numpy as np 5 | 6 | from bc_data_analyzer import plot_utils 7 | from bc_data_analyzer.blockchain import Blockchain 8 | from bc_data_analyzer.aggregator import count_actions_over_time 9 | 10 | 11 | OTHER_KEY = "other" 12 | 13 | 14 | @Blockchain.register("tezos") 15 | class Tezos(Blockchain): 16 | def __init__(self, max_actions=2): 17 | self.max_actions = max_actions 18 | 19 | def transform_actions_over_time( 20 | self, actions: List[Tuple[dt.datetime, dict]] 21 | ) -> Tuple[List[dt.datetime], List[str], np.ndarray, List[str]]: 22 | actions_count = count_actions_over_time(actions) 23 | sorted_actions = sorted(actions_count.items(), key=lambda v: -v[1]) 24 | top_actions = [k for k, _ in sorted_actions[: self.max_actions]] 25 | 26 | labels = [a.capitalize() for a in top_actions] + ["Other"] 27 | dates = [a[0] for a in actions] 28 | ys = np.array( 29 | [self._transform_action(a["Actions"], top_actions) 30 | for _, a in actions] 31 | ).T 32 | return labels, dates, ys, plot_utils.make_palette("blue", "green", "brown") 33 | 34 | @staticmethod 35 | def _find_action_count(actions: List[dict], name: str) -> int: 36 | for action in actions: 37 | if action["Name"] == name: 38 | return action["Count"] 39 | return 0 40 | 41 | def _transform_action(self, actions: dict, top_actions: dict) -> dict: 42 | result = [] 43 | for action in top_actions: 44 | result.append(self._find_action_count(actions, action)) 45 | result.append(sum(a["Count"] 46 | for a in actions if a["Name"] not in top_actions)) 47 | return result 48 | 49 | @property 50 | def available_tables(self): 51 | return ["top-senders"] 52 | 53 | def _generate_table(self, table_name: str, data: dict) -> str: 54 | if table_name == "top-senders": 55 | return self._output_top_senders_table(data["Results"]["ActionsBySender"]) 56 | raise ValueError("unknown table type {0}".format(table_name)) 57 | 58 | def _output_top_senders_table(self, data: dict) -> str: 59 | def make_row(row): 60 | receivers_count = row["Receivers"]["UniqueCount"] 61 | count = row["Count"] 62 | row_data = dict( 63 | name=row["Name"], 64 | count=count, 65 | avg=count / receivers_count, 66 | unique_count=row["Receivers"]["UniqueCount"], 67 | ) 68 | return ( 69 | r"\tezaddr{{{name}}} & {count:,} & {unique_count:,} & {avg:.2f}".format( 70 | **row_data 71 | ) 72 | ) 73 | 74 | rows = "\\\\\n ".join([make_row(row) 75 | for row in data["Actions"][:5]]) 76 | return r"""\begin{{figure*}}[tbp] 77 | \footnotesize 78 | \centering 79 | \begin{{tabular}}{{l r r r}} 80 | \toprule 81 | & & & \bf Avg. \#\\ 82 | & & \bf Unique & \bf of transactions\\ 83 | \bf Sender & \bf Sent count & \bf receivers & \bf per receiver\\ 84 | \midrule 85 | {rows}\\ 86 | \bottomrule 87 | \end{{tabular}} 88 | \caption{{Tezos accounts with the highest number of sent transactions.}} 89 | \label{{tab:tezos-account-edges}} 90 | \end{{figure*}}""".format( 91 | rows=rows 92 | ) 93 | -------------------------------------------------------------------------------- /bc-data-analyzer/bc_data_analyzer/blockchains/eos.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | from typing import List, Tuple 3 | import json 4 | import pkgutil 5 | 6 | import numpy as np 7 | 8 | from bc_data_analyzer import plot_utils 9 | from bc_data_analyzer import settings 10 | from bc_data_analyzer.blockchain import Blockchain 11 | 12 | 13 | @Blockchain.register("eos") 14 | class EOS(Blockchain): 15 | def __init__(self): 16 | categories = pkgutil.get_data( 17 | settings.PACKAGE_NAME, "data/eos-categories.json") 18 | self.categories = json.loads(categories) 19 | self.category_indexes = { 20 | category["name"]: i 21 | for i, category in enumerate(self.categories["categories"]) 22 | } 23 | self.accounts = json.loads( 24 | pkgutil.get_data(settings.PACKAGE_NAME, "data/eos-accounts.json") 25 | ) 26 | 27 | def compute_categories(self, actions_count): 28 | category_counts = [0 for _ in self.categories["categories"]] 29 | for action in actions_count["Actions"]: 30 | category = self.categories["mapping"].get(action["Name"], "others") 31 | category_counts[self.category_indexes[category]] += action["Count"] 32 | return category_counts 33 | 34 | def transform_actions_over_time( 35 | self, actions: List[Tuple[dt.datetime, dict]] 36 | ) -> Tuple[List[dt.datetime], List[str], np.ndarray, List[str]]: 37 | labels = [a["name"].capitalize() 38 | for a in self.categories["categories"]] 39 | dates = [a[0] for a in actions] 40 | ys = zip(*[self.compute_categories(a) for _, a in actions]) 41 | colors = plot_utils.make_palette( 42 | *[a["color"] for a in self.categories["categories"]] 43 | ) 44 | return labels, dates, ys, colors 45 | 46 | @property 47 | def available_tables(self): 48 | return ["top-actions"] 49 | 50 | def _generate_table(self, table_name: str, data: dict) -> str: 51 | if table_name == "top-actions": 52 | return self._output_top_actions_table(data["Results"]["ActionsByReceiver"]) 53 | raise ValueError("unknown table type {0}".format(table_name)) 54 | 55 | def _output_top_actions_table(self, actions: dict) -> str: 56 | def format_account(account_actions): 57 | total_actions_count = account_actions["Names"]["TotalCount"] 58 | actions = [ 59 | a 60 | for a in account_actions["Names"]["Actions"][:3] 61 | if a["Count"] / total_actions_count > 0.1 62 | ] 63 | 64 | def multirow(text, n=len(actions)): 65 | if n <= 1: 66 | return text 67 | else: 68 | return f"\\multirow{{{n}}}{{*}}{{{text}}}" 69 | 70 | def make_action(action): 71 | name = action["Name"] 72 | percentage = action["Count"] / total_actions_count * 100 73 | return f"{name} & {percentage:.2f}\\%" 74 | 75 | name = account_actions["Name"] 76 | description = self.accounts.get(name, "Unknown") 77 | formatted_total = f"{total_actions_count:,}" 78 | rows = [ 79 | f"{multirow(name)} & {multirow(description)} & " 80 | f"{multirow(formatted_total)} & {make_action(actions[0])}" 81 | ] 82 | for action in actions[1:]: 83 | rows.append(f"& & & {make_action(action)}") 84 | return "\\\\\n".join(rows) 85 | 86 | rows = [format_account(account) for account in actions["Actions"][:5]] 87 | return "\\\\\n\\midrule\n".join(rows) 88 | -------------------------------------------------------------------------------- /fetcher/http.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/danhper/blockchain-analyzer/core" 11 | ) 12 | 13 | type RequestSender func(*http.Client, uint64) (*http.Response, error) 14 | 15 | func fetchBlockWithRetry( 16 | client *http.Client, context *HTTPContext, 17 | blockNumber uint64, retries int, 18 | ) (result []byte, err error) { 19 | resp, err := context.MakeRequest(client, blockNumber) 20 | if err == nil && resp.StatusCode == 200 { 21 | result, err = ioutil.ReadAll(resp.Body) 22 | } 23 | if (err != nil || resp.StatusCode != 200) && retries > 0 { 24 | log.Printf("error: %s (status %d), retrying", err.Error(), resp.StatusCode) 25 | time.Sleep(time.Second) 26 | return fetchBlockWithRetry(client, context, blockNumber, retries-1) 27 | } 28 | return 29 | } 30 | 31 | func fetchBlock(blockNumber uint64, client *http.Client, context *HTTPContext) ([]byte, error) { 32 | return fetchBlockWithRetry(client, context, blockNumber, 3) 33 | } 34 | 35 | type HTTPContext struct { 36 | DoneCount uint64 37 | Start uint64 38 | End uint64 39 | MakeRequest RequestSender 40 | } 41 | 42 | func NewHTTPContext(start, end uint64, makeRequest RequestSender) *HTTPContext { 43 | return &HTTPContext{ 44 | DoneCount: 0, 45 | Start: start, 46 | End: end, 47 | MakeRequest: makeRequest, 48 | } 49 | } 50 | 51 | func (c *HTTPContext) TotalCount() uint64 { 52 | return c.End - c.Start + 1 53 | } 54 | 55 | func fetchBlocks(context *HTTPContext, blocks <-chan uint64, results chan<- []byte) { 56 | tr := &http.Transport{ 57 | MaxIdleConns: 10, 58 | IdleConnTimeout: 30 * time.Second, 59 | DisableCompression: true, 60 | } 61 | client := &http.Client{Transport: tr} 62 | for block := range blocks { 63 | result, err := fetchBlock(block, client, context) 64 | if err != nil { 65 | log.Printf("could not fetch block %d: %s", block, err.Error()) 66 | } 67 | results <- result 68 | } 69 | } 70 | 71 | func fetchBatch(filepath string, start, end uint64, context *HTTPContext) error { 72 | gzipFile, err := core.CreateFile(core.MakeFilename(filepath, start, end)) 73 | if err != nil { 74 | return err 75 | } 76 | defer gzipFile.Close() 77 | 78 | workersCount := 10 79 | blocksCount := end - start + 1 80 | jobs := make(chan uint64, blocksCount) 81 | results := make(chan []byte, blocksCount) 82 | 83 | for w := 1; w <= workersCount; w++ { 84 | go fetchBlocks(context, jobs, results) 85 | } 86 | 87 | for block := end; block >= start; block-- { 88 | jobs <- block 89 | } 90 | close(jobs) 91 | for i := uint64(0); i < blocksCount; i++ { 92 | result := <-results 93 | result = append(bytes.TrimSpace(result), '\n') 94 | gzipFile.Write(result) 95 | 96 | context.DoneCount++ 97 | if context.DoneCount%100 == 0 { 98 | log.Printf("%d/%d", context.DoneCount, context.TotalCount()) 99 | } 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func FetchHTTPData(filepath string, context *HTTPContext) error { 106 | log.Printf("fetching %d blocks", context.TotalCount()) 107 | getNext := func(num uint64) uint64 { 108 | if num >= core.BatchSize { 109 | return num - core.BatchSize 110 | } else { 111 | return context.Start 112 | } 113 | } 114 | for block := context.End; block > context.Start; block = getNext(block) { 115 | var currentFirst uint64 116 | if block+1 < core.BatchSize || block+1-core.BatchSize < context.Start { 117 | currentFirst = context.Start 118 | } else { 119 | currentFirst = block + 1 - core.BatchSize 120 | } 121 | if err := fetchBatch(filepath, currentFirst, block, context); err != nil { 122 | return err 123 | } 124 | } 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/danhper/structomap v0.6.2 h1:VryUhPR3Ju+FeXHXTL2RGPbRmIX/mN768jULh9weylM= 5 | github.com/danhper/structomap v0.6.2/go.mod h1:qmif0PLXZftsShsS0miHLhWv4VRR8awMUPnYxHAJCjg= 6 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= 11 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 12 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 13 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 14 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 15 | github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= 16 | github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 17 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 18 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 19 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 20 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 21 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 22 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 23 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 24 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 25 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 26 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 27 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 28 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 29 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 30 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 31 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 32 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 33 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 34 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 35 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 36 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 37 | github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= 38 | github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 40 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 41 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 42 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 43 | -------------------------------------------------------------------------------- /eos/eos.go: -------------------------------------------------------------------------------- 1 | package eos 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "strings" 9 | "time" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | 13 | "github.com/danhper/blockchain-analyzer/core" 14 | "github.com/danhper/blockchain-analyzer/fetcher" 15 | ) 16 | 17 | var fastJson = jsoniter.ConfigCompatibleWithStandardLibrary 18 | 19 | const ( 20 | defaultProducerURL string = "https://api.main.alohaeos.com:443" 21 | timeLayout string = "2006-01-02T15:04:05.999" 22 | ) 23 | 24 | type EOS struct { 25 | ProducerURL string 26 | } 27 | 28 | func (e *EOS) makeRequest(client *http.Client, blockNumber uint64) (*http.Response, error) { 29 | url := fmt.Sprintf("%s/v1/chain/get_block", e.ProducerURL) 30 | data := fmt.Sprintf("{\"block_num_or_id\": %d}", blockNumber) 31 | return client.Post(url, "application/json", strings.NewReader(data)) 32 | } 33 | 34 | func (e *EOS) FetchData(filepath string, start, end uint64) error { 35 | context := fetcher.NewHTTPContext(start, end, e.makeRequest) 36 | return fetcher.FetchHTTPData(filepath, context) 37 | } 38 | 39 | type Action struct { 40 | Account string 41 | ActionName string `json:"name"` 42 | Authorization []struct { 43 | Actor string 44 | Permission string 45 | } 46 | Data json.RawMessage 47 | } 48 | 49 | type Transaction struct { 50 | Actions []Action 51 | Expiration string 52 | RefBlockNum int `json:"ref_block_num"` 53 | } 54 | 55 | type Trx struct { 56 | Id string 57 | Signatures []string 58 | Transaction Transaction 59 | } 60 | 61 | type TrxOrString Trx 62 | 63 | func (t *TrxOrString) UnmarshalJSON(b []byte) error { 64 | if b[0] == '"' { 65 | var id string 66 | if err := fastJson.Unmarshal(b, &id); err != nil { 67 | return err 68 | } 69 | *t = TrxOrString{Id: id} 70 | return nil 71 | } else if b[0] == '{' { 72 | return fastJson.Unmarshal(b, (*Trx)(t)) 73 | } 74 | return fmt.Errorf("expected '{' or '\"', got %s", string(b)) 75 | } 76 | 77 | type FullTransaction struct { 78 | Status string 79 | Trx TrxOrString 80 | } 81 | 82 | type Block struct { 83 | BlockNumber uint64 `json:"block_num"` 84 | Timestamp string 85 | parsedTime time.Time 86 | Transactions []FullTransaction 87 | actions []core.Action 88 | } 89 | 90 | func New() *EOS { 91 | producerURL := os.Getenv("EOS_PRODUCER_URL") 92 | if producerURL == "" { 93 | producerURL = defaultProducerURL 94 | } 95 | 96 | return &EOS{ 97 | ProducerURL: producerURL, 98 | } 99 | } 100 | 101 | func (e *EOS) ParseBlock(rawBlock []byte) (core.Block, error) { 102 | var block Block 103 | if err := fastJson.Unmarshal(rawBlock, &block); err != nil { 104 | return nil, err 105 | } 106 | parsedTime, err := time.Parse(timeLayout, block.Timestamp) 107 | if err != nil { 108 | return nil, err 109 | } 110 | block.parsedTime = parsedTime 111 | return &block, fastJson.Unmarshal(rawBlock, &block) 112 | } 113 | 114 | func (e *EOS) EmptyBlock() core.Block { 115 | return &Block{} 116 | } 117 | 118 | func (b *Block) Number() uint64 { 119 | return b.BlockNumber 120 | } 121 | 122 | func (b *Block) Time() time.Time { 123 | return b.parsedTime 124 | } 125 | 126 | func (b *Block) TransactionsCount() int { 127 | return len(b.Transactions) 128 | } 129 | 130 | func (b *Block) ListActions() []core.Action { 131 | if len(b.actions) > 0 { 132 | return b.actions 133 | } 134 | var actions []core.Action 135 | for _, transaction := range b.Transactions { 136 | for _, action := range transaction.Trx.Transaction.Actions { 137 | actions = append(actions, &action) 138 | } 139 | } 140 | b.actions = actions 141 | return actions 142 | } 143 | 144 | func (a *Action) Name() string { 145 | return a.ActionName 146 | } 147 | 148 | func (a *Action) Sender() string { 149 | if len(a.Authorization) == 0 { 150 | return "" 151 | } 152 | return a.Authorization[0].Actor 153 | } 154 | 155 | func (a *Action) Receiver() string { 156 | return a.Account 157 | } 158 | -------------------------------------------------------------------------------- /processor/processor_test.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "io" 5 | "testing" 6 | "time" 7 | 8 | "github.com/danhper/blockchain-analyzer/core" 9 | "github.com/danhper/blockchain-analyzer/xrp" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func computeBlockNumbers(reader io.Reader, blockchain core.Blockchain, start, end uint64) *core.MissingBlocks { 14 | missingBlocks := core.NewMissingBlocks(start, end) 15 | for block := range YieldBlocks(reader, blockchain, JSONFormat) { 16 | missingBlocks.AddBlock(block) 17 | } 18 | return missingBlocks 19 | } 20 | 21 | func TestComputeBlockNumbers(t *testing.T) { 22 | reader := core.GetFixtureReader(core.XRPValidLedgersFilename) 23 | blockchain := xrp.New() 24 | blocks := computeBlockNumbers(reader, blockchain, 0, 0) 25 | assert.Len(t, blocks.Seen, 100) 26 | assert.Contains(t, blocks.Seen, uint64(54387329)) 27 | } 28 | 29 | func TestGetMissingBlockNumbersValid(t *testing.T) { 30 | reader := core.GetFixtureReader(core.XRPValidLedgersFilename) 31 | blockchain := xrp.New() 32 | blockNumbers := computeBlockNumbers(reader, blockchain, 54387321, 54387329) 33 | missing := blockNumbers.Result() 34 | assert.Len(t, missing, 0) 35 | } 36 | 37 | func TestGetMissingBlockNumbersInvalid(t *testing.T) { 38 | reader := core.GetFixtureReader(core.XRPMissingLedgersFilename) 39 | blockchain := xrp.New() 40 | missingBlockNumbers := computeBlockNumbers(reader, blockchain, 123, 126) 41 | missing := missingBlockNumbers.Compute() 42 | assert.Len(t, missing, 1) 43 | assert.Equal(t, missing[0], uint64(124)) 44 | } 45 | 46 | func TestCountTransactions(t *testing.T) { 47 | blockchain := xrp.New() 48 | filepath := core.GetFixture(core.XRPValidLedgersFilename) 49 | count, err := CountTransactions(blockchain, filepath, uint64(0), uint64(0)) 50 | assert.Nil(t, err) 51 | assert.Equal(t, 4518, count) 52 | } 53 | 54 | func TestYieldAllDuplicated(t *testing.T) { 55 | blockchain := xrp.New() 56 | fixtures := core.GetFixture(core.XRPDuplicatedLedgersFilename) 57 | blocksChan, err := YieldAllBlocks(fixtures, blockchain, uint64(0), uint64(0)) 58 | assert.Nil(t, err) 59 | var blocks []core.Block 60 | for block := range blocksChan { 61 | blocks = append(blocks, block) 62 | } 63 | assert.Equal(t, 3, len(blocks)) 64 | } 65 | 66 | func TestCountActionsOverTime(t *testing.T) { 67 | blockchain := xrp.New() 68 | filepath := core.GetFixture(core.XRPValidLedgersFilename) 69 | actionsCount, err := CountActionsOverTime( 70 | blockchain, filepath, uint64(0), uint64(0), time.Minute, core.ActionName) 71 | assert.Nil(t, err) 72 | assert.Len(t, actionsCount.Actions, 7) 73 | lastGroup := time.Date(2020, 3, 27, 20, 55, 0, 0, time.UTC) 74 | assert.Contains(t, actionsCount.Actions, lastGroup) 75 | assert.Equal(t, uint64(96), actionsCount.Actions[lastGroup].GetCount("Payment")) 76 | beforeLastGroup := time.Date(2020, 3, 27, 20, 54, 0, 0, time.UTC) 77 | assert.Contains(t, actionsCount.Actions, beforeLastGroup) 78 | assert.Equal(t, uint64(519), actionsCount.Actions[beforeLastGroup].GetCount("OfferCreate")) 79 | } 80 | 81 | func TestCountTransactionsOverTime(t *testing.T) { 82 | blockchain := xrp.New() 83 | filepath := core.GetFixture(core.XRPValidLedgersFilename) 84 | actionsCount, err := CountTransactionsOverTime( 85 | blockchain, filepath, uint64(0), uint64(0), time.Minute) 86 | assert.Nil(t, err) 87 | assert.Len(t, actionsCount.TransactionCounts, 7) 88 | lastGroup := time.Date(2020, 3, 27, 20, 55, 0, 0, time.UTC) 89 | assert.Equal(t, 451, actionsCount.TransactionCounts[lastGroup]) 90 | beforeLastGroup := time.Date(2020, 3, 27, 20, 54, 0, 0, time.UTC) 91 | assert.Equal(t, 803, actionsCount.TransactionCounts[beforeLastGroup]) 92 | } 93 | 94 | func TestGroupActions(t *testing.T) { 95 | blockchain := xrp.New() 96 | filepath := core.GetFixture(core.XRPValidLedgersFilename) 97 | actionsCount, err := GroupActions( 98 | blockchain, filepath, uint64(0), uint64(0), core.ActionName, false) 99 | assert.Nil(t, err) 100 | assert.Equal(t, uint64(1129), actionsCount.GetCount("Payment")) 101 | assert.Equal(t, uint64(3088), actionsCount.GetCount("OfferCreate")) 102 | } 103 | -------------------------------------------------------------------------------- /processor/processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/danhper/blockchain-analyzer/core" 16 | "github.com/ugorji/go/codec" 17 | ) 18 | 19 | const logInterval int = 10000 20 | 21 | type FileFormat int 22 | 23 | const ( 24 | JSONFormat FileFormat = iota 25 | MsgpackFormat 26 | ) 27 | 28 | var ( 29 | msgpackHandle = &codec.MsgpackHandle{} 30 | ) 31 | 32 | func InferFormat(filepath string) (FileFormat, error) { 33 | if strings.Contains(filepath, ".jsonl") { 34 | return JSONFormat, nil 35 | } 36 | if strings.Contains(filepath, ".dat") { 37 | return MsgpackFormat, nil 38 | } 39 | return JSONFormat, fmt.Errorf("invalid filename %s", filepath) 40 | } 41 | 42 | func YieldBlocks(reader io.Reader, blockchain core.Blockchain, format FileFormat) <-chan core.Block { 43 | stream := bufio.NewReader(reader) 44 | blocks := make(chan core.Block) 45 | 46 | var decoder *codec.Decoder 47 | if format == MsgpackFormat { 48 | decoder = codec.NewDecoder(stream, msgpackHandle) 49 | } 50 | 51 | go func() { 52 | defer close(blocks) 53 | 54 | for i := 0; ; i++ { 55 | if i%logInterval == 0 { 56 | log.Printf("processed: %d", i) 57 | } 58 | block := blockchain.EmptyBlock() 59 | var err error 60 | switch format { 61 | case JSONFormat: 62 | rawLine, err := stream.ReadBytes('\n') 63 | if err == io.EOF { 64 | return 65 | } 66 | if err != nil { 67 | log.Printf("failed to read line %s\n", err.Error()) 68 | return 69 | } 70 | rawLine = bytes.ToValidUTF8(rawLine, []byte{}) 71 | block, err = blockchain.ParseBlock(rawLine) 72 | case MsgpackFormat: 73 | err = decoder.Decode(&block) 74 | } 75 | 76 | if err == io.EOF { 77 | break 78 | } else if err != nil { 79 | log.Printf("could not parse: %s", err.Error()) 80 | continue 81 | } 82 | 83 | if block != nil { 84 | blocks <- block 85 | } 86 | } 87 | }() 88 | 89 | return blocks 90 | } 91 | 92 | func YieldAllBlocks( 93 | globPattern string, 94 | blockchain core.Blockchain, 95 | start, end uint64) (<-chan core.Block, error) { 96 | files, err := filepath.Glob(globPattern) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | log.Printf("starting for %d files", len(files)) 102 | blocks := make(chan core.Block) 103 | uniqueBlocks := make(chan core.Block) 104 | 105 | processed := 0 106 | fileDone := make(chan bool) 107 | 108 | var wg sync.WaitGroup 109 | run := core.MakeFileProcessor(func(filename string) error { 110 | defer wg.Done() 111 | fileFormat, err := InferFormat(filename) 112 | if err != nil { 113 | return err 114 | } 115 | reader, err := core.OpenFile(filename) 116 | if err != nil { 117 | return err 118 | } 119 | defer reader.Close() 120 | for block := range YieldBlocks(reader, blockchain, fileFormat) { 121 | if (start == 0 || block.Number() >= start) && 122 | (end == 0 || block.Number() <= end) { 123 | blocks <- block 124 | } 125 | } 126 | fileDone <- true 127 | return err 128 | }) 129 | 130 | seen := make(map[uint64]bool) 131 | go func() { 132 | for block := range blocks { 133 | if _, ok := seen[block.Number()]; !ok { 134 | uniqueBlocks <- block 135 | seen[block.Number()] = true 136 | } 137 | } 138 | close(uniqueBlocks) 139 | }() 140 | 141 | for _, filename := range files { 142 | wg.Add(1) 143 | go run(filename) 144 | } 145 | 146 | go func() { 147 | for range fileDone { 148 | processed++ 149 | log.Printf("files processed: %d/%d", processed, len(files)) 150 | } 151 | }() 152 | 153 | go func() { 154 | wg.Wait() 155 | close(blocks) 156 | close(fileDone) 157 | }() 158 | 159 | return uniqueBlocks, nil 160 | } 161 | 162 | func ComputeMissingBlockNumbers(blockNumbers map[uint64]bool, start, end uint64) []uint64 { 163 | missing := make([]uint64, 0) 164 | for blockNumber := start; blockNumber <= end; blockNumber++ { 165 | if _, ok := blockNumbers[blockNumber]; !ok { 166 | missing = append(missing, blockNumber) 167 | } 168 | } 169 | 170 | return missing 171 | } 172 | 173 | func OutputAllMissingBlockNumbers( 174 | blockchain core.Blockchain, globPattern string, 175 | outputPath string, start, end uint64) error { 176 | 177 | blocks, err := YieldAllBlocks(globPattern, blockchain, start, 0) 178 | if err != nil { 179 | return err 180 | } 181 | 182 | outputFile, err := core.CreateFile(outputPath) 183 | defer outputFile.Close() 184 | 185 | missingBlockNumbers := core.NewMissingBlocks(start, end) 186 | for block := range blocks { 187 | missingBlockNumbers.AddBlock(block) 188 | } 189 | 190 | missing := missingBlockNumbers.Compute() 191 | for _, number := range missing { 192 | fmt.Fprintf(outputFile, "{\"block\": %d}\n", number) 193 | } 194 | 195 | if len(missing) > 0 { 196 | return fmt.Errorf("%d missing blocks written to %s", len(missing), outputPath) 197 | } 198 | 199 | os.Remove(outputPath) 200 | 201 | return nil 202 | } 203 | 204 | func CountTransactions(blockchain core.Blockchain, globPattern string, start, end uint64) (int, error) { 205 | blocks, err := YieldAllBlocks(globPattern, blockchain, start, end) 206 | if err != nil { 207 | return 0, err 208 | } 209 | txCounter := core.NewTransactionCounter() 210 | for block := range blocks { 211 | txCounter.AddBlock(block) 212 | } 213 | return (int)(*txCounter), nil 214 | } 215 | 216 | func CountActionsOverTime( 217 | blockchain core.Blockchain, 218 | globPattern string, 219 | start, end uint64, 220 | duration time.Duration, 221 | actionProperty core.ActionProperty) (*core.TimeGroupedActions, error) { 222 | blocks, err := YieldAllBlocks(globPattern, blockchain, start, end) 223 | if err != nil { 224 | return nil, err 225 | } 226 | result := core.NewTimeGroupedActions(duration, actionProperty) 227 | for block := range blocks { 228 | result.AddBlock(block) 229 | } 230 | return result, nil 231 | } 232 | 233 | func CountTransactionsOverTime(blockchain core.Blockchain, globPattern string, 234 | start, end uint64, duration time.Duration, 235 | ) (*core.TimeGroupedTransactionCount, error) { 236 | blocks, err := YieldAllBlocks(globPattern, blockchain, start, end) 237 | if err != nil { 238 | return nil, err 239 | } 240 | result := core.NewTimeGroupedTransactionCount(duration) 241 | for block := range blocks { 242 | result.AddBlock(block) 243 | } 244 | return result, nil 245 | } 246 | 247 | func GroupActions(blockchain core.Blockchain, globPattern string, 248 | start, end uint64, by core.ActionProperty, detailed bool, 249 | ) (*core.GroupedActions, error) { 250 | blocks, err := YieldAllBlocks(globPattern, blockchain, start, end) 251 | if err != nil { 252 | return nil, err 253 | } 254 | groupedActions := core.NewGroupedActions(by, detailed) 255 | for block := range blocks { 256 | groupedActions.AddBlock(block) 257 | } 258 | return groupedActions, nil 259 | } 260 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blockchain-analyzer 2 | 3 | [![CircleCI](https://circleci.com/gh/danhper/blockchain-analyzer.svg?style=svg)](https://circleci.com/gh/danhper/blockchain-analyzer) 4 | 5 | CLI tool to fetch and analyze transactions data from several blockchains. 6 | 7 | Currently supported blockchains: 8 | 9 | - [Tezos](https://tezos.com/) 10 | - [EOS](https://eos.io/) 11 | - [XRP](https://ripple.com/xrp/) 12 | 13 | ## Installation 14 | 15 | ### Static binaries 16 | 17 | We provide static binaries for Windows, macOS and Linux with each [release](https://github.com/danhper/blockchain-analyzer/releases) 18 | 19 | ### From source 20 | 21 | Go needs to be installed. The tool can then be installed by running 22 | 23 | ``` 24 | go get github.com/danhper/blockchain-analyzer/cmd/blockchain-analyzer 25 | ``` 26 | 27 | ## Usage 28 | 29 | ### Fetching data 30 | 31 | The `fetch` command can be used to fetch the data: 32 | 33 | ``` 34 | blockchain-analyzer BLOCKCHAIN fetch -o OUTPUT_FILE --start START_BLOCK --end END_BLOCK 35 | 36 | # for example from 500,000 to 699,999 inclusive: 37 | blockchain-analyzer eos fetch -o eos-blocks.jsonl.gz --start 500000 --end 699999 38 | ``` 39 | 40 | ### Data format 41 | 42 | The data has the following format 43 | 44 | - One block per line, including transactions, formatted in JSON. Documentation of block format can be found in each chain documentation 45 | - [EOS](https://developers.eos.io/manuals/eos/latest/nodeos/plugins/chain_api_plugin/api-reference/index#operation/get_block) 46 | - [Tezos](https://tezos.gitlab.io/api/rpc.html#get-block-id) 47 | - [XRP](https://xrpl.org/ledger.html) 48 | - Grouped in files of 100,000 blocks each, suffixed by the block range (e.g. `eos-blocks-500000--599999.jsonl` and `eos-blocks-600000--699999.jsonl` for the above) 49 | - Gziped if the `.gz` extension is added to the output file name (recommended) 50 | 51 | ### Checking data 52 | 53 | The `check` command can then be used to check the fetched data. It will ensure that all the block from `--start` to `--end` exist in the given files, and output the missing blocks into `missing.jsonl` if any. 54 | 55 | ``` 56 | blockchain-analyzer eos check -p 'eos-blocks*.jsonl.gz' -o missing.jsonl --start 500000 --end 699999 57 | ``` 58 | 59 | ### Analyzing data 60 | 61 | The simplest way to analyze the data is to provide a configuration file about what to analyze and run the tool with the following command. 62 | 63 | ``` 64 | blockchain-analyzer bulk-process -c config.json -o tmp/results.json 65 | ``` 66 | 67 | Configuration files used for [our paper](https://arxiv.org/abs/2003.02693) can be found in the [config](./config) directory. 68 | 69 | The tool's help also contains information about what other commands can be used 70 | 71 | ```plain 72 | $ ./build/blockchain-analyzer -h 73 | NAME: 74 | blockchain-analyzer - Tool to fetch and analyze blockchain transactions 75 | 76 | USAGE: 77 | blockchain-analyzer [global options] command [command options] [arguments...] 78 | 79 | COMMANDS: 80 | eos Analyze EOS data 81 | tezos Analyze Tezos data 82 | xrp Analyze XRP data 83 | help, h Shows a list of commands or help for one command 84 | 85 | GLOBAL OPTIONS: 86 | --cpu-profile value Path where to store the CPU profile 87 | --help, -h show help (default: false) 88 | 89 | # the following is also available for xrp and tezos 90 | $ ./build/blockchain-analyzer eos -h 91 | NAME: 92 | blockchain-analyzer eos - Analyze EOS data 93 | 94 | USAGE: 95 | blockchain-analyzer eos command [command options] [arguments...] 96 | 97 | COMMANDS: 98 | export-transfers Export all the transfers to a CSV file 99 | fetch Fetches blockchain data 100 | check Checks for missing blocks in data 101 | count-transactions Count the number of transactions in the data 102 | group-actions Count and groups the number of "actions" in the data 103 | group-actions-over-time Count and groups per time the number of "actions" in the data 104 | count-transactions-over-time Count number of "transactions" over time in the data 105 | bulk-process Bulk process the data according to the given configuration file 106 | export Export a subset of the fields to msgpack format for faster processing 107 | help, h Shows a list of commands or help for one command 108 | 109 | OPTIONS: 110 | --help, -h show help (default: false) 111 | ``` 112 | 113 | ### Interpreting results 114 | 115 | We provide Python scripts to plot and generate table out of the data from the analysis. 116 | Please check the [bc-data-analyzer](./bc-data-analyzer) directory for more information. 117 | 118 | ## Dataset 119 | 120 | All the data used in our paper mentioned below can be downloaded from the following link: 121 | 122 | https://imperialcollegelondon.box.com/s/jijwo76e2pxlbkuzzt1yjz0z3niqz7yy 123 | 124 | This includes data from October 1, 2019 to April 30, 2020 for EOS, Tezos and XRP, which corresponds to the following blocks: 125 | 126 | | Blockchain | Start block | End block | 127 | | ---------- | ----------: | --------: | 128 | | EOS | 82152667 | 118286375 | 129 | | XRP | 50399027 | 55152991 | 130 | | Tezos | 630709 | 932530 | 131 | 132 | Please refer to the [Data format](https://github.com/danhper/blockchain-analyzer#data-format) section above for a description of the data format. 133 | 134 | ## Supporting other blockchains 135 | 136 | Although the framework currently only supports EOS, Tezos and XRP, it has been designed to easily support other blockchains. 137 | The three following interfaces need to be implemented in order to do so: 138 | 139 | ```go 140 | type Blockchain interface { 141 | FetchData(filepath string, start, end uint64) error 142 | ParseBlock(rawLine []byte) (Block, error) 143 | EmptyBlock() Block 144 | } 145 | 146 | type Block interface { 147 | Number() uint64 148 | TransactionsCount() int 149 | Time() time.Time 150 | ListActions() []Action 151 | } 152 | 153 | type Action interface { 154 | Sender() string 155 | Receiver() string 156 | Name() string 157 | } 158 | ``` 159 | 160 | We also provide a utilities to make methods such as `FetchData` easier to implement. 161 | [Existing implementations](https://github.com/danhper/blockchain-analyzer/blob/master/tezos/tezos.go) can be used as a point of reference for how a new blockchain can be supported. 162 | 163 | ## Academic work 164 | 165 | This tool has originally been created to analyze data for the following paper: [Revisiting Transactional Statistics of High-scalability Blockchain](https://arxiv.org/abs/2003.02693), presented at [IMC'20](https://conferences.sigcomm.org/imc/2020/accepted/). 166 | If you are using this for academic work, we would be thankful if you could cite it. 167 | 168 | ``` 169 | @misc{perez2020revisiting, 170 | title={Revisiting Transactional Statistics of High-scalability Blockchain}, 171 | author={Daniel Perez and Jiahua Xu and Benjamin Livshits}, 172 | year={2020}, 173 | eprint={2003.02693}, 174 | archivePrefix={arXiv}, 175 | primaryClass={cs.CR} 176 | } 177 | ``` 178 | -------------------------------------------------------------------------------- /xrp/fetcher.go: -------------------------------------------------------------------------------- 1 | package xrp 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "os/signal" 10 | "sync" 11 | "time" 12 | 13 | "github.com/gorilla/websocket" 14 | 15 | "github.com/danhper/blockchain-analyzer/core" 16 | ) 17 | 18 | const ( 19 | defaultWSURI string = "wss://xrpl.ws" 20 | maxTries int = 5 21 | ) 22 | 23 | type WSError struct { 24 | message string 25 | } 26 | 27 | func (e *WSError) Error() string { 28 | return fmt.Sprintf("ws connection error: %s", e.message) 29 | } 30 | 31 | func makeMessage(ledgerIndex uint64) []byte { 32 | params := map[string]interface{}{ 33 | "command": "ledger", 34 | "ledger_index": ledgerIndex, 35 | "transactions": true, 36 | "expand": true, 37 | } 38 | message, err := json.Marshal(params) 39 | if err != nil { 40 | panic(err) 41 | } 42 | return message 43 | } 44 | 45 | func processWSMessages( 46 | conn *websocket.Conn, writer io.Writer, wg *sync.WaitGroup, quit chan struct{}, written chan<- uint64, 47 | abort chan<- error) { 48 | for { 49 | _, message, err := conn.ReadMessage() 50 | if err != nil { 51 | abort <- &WSError{message: err.Error()} 52 | log.Println("read:", err) 53 | return 54 | } 55 | fmt.Fprintf(writer, "%s\n", message) 56 | 57 | ledger, err := ParseRawLedger(message) 58 | if err != nil { 59 | log.Printf("error while parsing message: %s", err.Error()) 60 | } 61 | written <- ledger.Number() 62 | wg.Done() 63 | 64 | select { 65 | case <-quit: 66 | return 67 | case <-time.After(time.Millisecond): 68 | } 69 | } 70 | } 71 | 72 | func sendWSMessage(conn *websocket.Conn, ledger uint64) { 73 | message := makeMessage(ledger) 74 | if err := conn.WriteMessage(websocket.TextMessage, message); err != nil { 75 | log.Printf("could not fetch ledger %d: %s", ledger, err.Error()) 76 | } 77 | } 78 | 79 | func writeFailed(filePath string, failed map[uint64]bool) error { 80 | // write failed blocks 81 | var errWriter io.Writer 82 | errWriter, err := core.CreateFile(filePath) 83 | if err != nil { 84 | log.Printf("could not open error file: %s", err.Error()) 85 | return err 86 | } 87 | for ledger := range failed { 88 | fmt.Fprintf(errWriter, "%d\n", ledger) 89 | } 90 | return nil 91 | } 92 | 93 | func fetchLedgersRange(start, end uint64, filePath string, context *XRPContext) (stop bool, err error) { 94 | toFetch := make(map[uint64]bool) 95 | for ledger := start; ledger <= end; ledger++ { 96 | toFetch[ledger] = true 97 | } 98 | writer, err := core.CreateFile(core.MakeFilename(filePath, start, end)) 99 | if err != nil { 100 | return false, err 101 | } 102 | defer writer.Close() 103 | return fetchLedgersWithRetry(toFetch, writer, context) 104 | } 105 | 106 | func fetchLedgersWithRetry(toFetch map[uint64]bool, writer io.Writer, context *XRPContext) (stop bool, err error) { 107 | tries := 0 108 | for tries < maxTries { 109 | stop, err = fetchLedgers(toFetch, writer, context) 110 | if stop { 111 | return 112 | } 113 | 114 | // reconnect in case of websocket failures 115 | if err != nil && errors.Is(err, &WSError{}) { 116 | if err = context.Reconnect(); err != nil { 117 | log.Printf("error while reconnecting: %s\n", err.Error()) 118 | return true, err 119 | } 120 | } 121 | if len(toFetch) == 0 { 122 | break 123 | } 124 | log.Printf("%d items left in batch, retrying", len(toFetch)) 125 | } 126 | if len(toFetch) > 0 { 127 | writeFailed("failed.txt.gz", toFetch) 128 | } 129 | 130 | return 131 | } 132 | 133 | func fetchLedgers(toFetch map[uint64]bool, writer io.Writer, context *XRPContext) (bool, error) { 134 | var wg sync.WaitGroup 135 | quit := make(chan struct{}) 136 | waiting := 0 137 | bufferSize := 20 138 | written := make(chan uint64, bufferSize) 139 | abort := make(chan error, 1) 140 | 141 | go processWSMessages(context.conn, writer, &wg, quit, written, abort) 142 | 143 | shouldWait := true 144 | 145 | defer func() { 146 | if shouldWait { 147 | wg.Wait() 148 | close(written) 149 | for index := range written { 150 | delete(toFetch, index) 151 | } 152 | } 153 | close(quit) 154 | }() 155 | 156 | ledgersToFetch := make([]uint64, len(toFetch)) 157 | index := 0 158 | for ledger := range toFetch { 159 | ledgersToFetch[index] = ledger 160 | index++ 161 | } 162 | index = 0 163 | for index < len(ledgersToFetch) { 164 | if waiting < bufferSize { 165 | ledger := ledgersToFetch[index] 166 | sendWSMessage(context.conn, ledger) 167 | waiting++ 168 | wg.Add(1) 169 | if context.doneCount%100 == 0 { 170 | log.Printf("%d/%d", context.doneCount, context.totalCount) 171 | } 172 | context.doneCount++ 173 | index++ 174 | } 175 | select { 176 | case err := <-abort: 177 | shouldWait = false 178 | return false, err 179 | case <-context.interrupt: 180 | return true, nil 181 | case index := <-written: 182 | waiting-- 183 | delete(toFetch, index) 184 | case <-time.After(time.Millisecond): 185 | } 186 | } 187 | return false, nil 188 | } 189 | 190 | func closeConnection(conn *websocket.Conn) { 191 | // Cleanly close the connection by sending a close message and then 192 | // waiting (with timeout) for the server to close the connection. 193 | msg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") 194 | err := conn.WriteMessage(websocket.CloseMessage, msg) 195 | if err != nil { 196 | log.Println("write close:", err) 197 | return 198 | } 199 | done := make(chan struct{}) 200 | go func() { 201 | conn.ReadMessage() 202 | close(done) 203 | }() 204 | select { 205 | case <-done: 206 | case <-time.After(time.Second): 207 | } 208 | } 209 | 210 | type XRPContext struct { 211 | wsURI string 212 | conn *websocket.Conn 213 | interrupt chan os.Signal 214 | doneCount int 215 | totalCount uint64 216 | } 217 | 218 | func NewXRPContext(interrupt chan os.Signal, wsURI string, totalCount uint64) (*XRPContext, error) { 219 | context := &XRPContext{ 220 | interrupt: interrupt, 221 | doneCount: 0, 222 | wsURI: wsURI, 223 | totalCount: totalCount, 224 | } 225 | return context, context.Reconnect() 226 | } 227 | 228 | func (c *XRPContext) Reconnect() (err error) { 229 | c.conn, _, err = websocket.DefaultDialer.Dial(c.wsURI, nil) 230 | return 231 | } 232 | 233 | func (c *XRPContext) Cleanup() error { 234 | if c.conn != nil { 235 | closeConnection(c.conn) 236 | return c.conn.Close() 237 | } 238 | return nil 239 | } 240 | 241 | func fetchXRPData(filepath string, start, end uint64) error { 242 | interrupt := make(chan os.Signal, 1) 243 | signal.Notify(interrupt, os.Interrupt) 244 | 245 | wsURI := os.Getenv("XRP_WS_URI") 246 | if wsURI == "" { 247 | wsURI = defaultWSURI 248 | } 249 | 250 | totalCount := end - start + 1 251 | context, err := NewXRPContext(interrupt, wsURI, totalCount) 252 | if err != nil { 253 | log.Fatalln(err.Error()) 254 | } 255 | 256 | log.Printf("fetching %d ledgers", totalCount) 257 | 258 | defer context.Cleanup() 259 | 260 | for ledger := end; ledger >= start; ledger -= core.BatchSize { 261 | currentStart := ledger - core.BatchSize + 1 262 | interrupted, err := fetchLedgersRange(currentStart, ledger, filepath, context) 263 | if err != nil || interrupted { 264 | break 265 | } 266 | } 267 | 268 | return nil 269 | } 270 | -------------------------------------------------------------------------------- /core/data.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sort" 7 | "time" 8 | 9 | "github.com/danhper/structomap" 10 | ) 11 | 12 | type ActionProperty int 13 | 14 | const ( 15 | ActionName ActionProperty = iota 16 | ActionSender 17 | ActionReceiver 18 | ) 19 | 20 | const ( 21 | maxTopLevelResults = 1000 22 | maxNestedResults = 50 23 | ) 24 | 25 | func GetActionProperty(name string) (ActionProperty, error) { 26 | switch name { 27 | case "name": 28 | return ActionName, nil 29 | case "sender": 30 | return ActionSender, nil 31 | case "receiver": 32 | return ActionReceiver, nil 33 | default: 34 | return ActionName, fmt.Errorf("no property %s for actions", name) 35 | } 36 | } 37 | 38 | func (p ActionProperty) String() string { 39 | switch p { 40 | case ActionName: 41 | return "name" 42 | case ActionSender: 43 | return "sender" 44 | case ActionReceiver: 45 | return "receiver" 46 | default: 47 | panic(fmt.Errorf("no such action property")) 48 | } 49 | } 50 | 51 | func (c *ActionProperty) UnmarshalJSON(data []byte) (err error) { 52 | var rawProperty string 53 | if err = json.Unmarshal(data, &rawProperty); err != nil { 54 | return err 55 | } 56 | *c, err = GetActionProperty(rawProperty) 57 | return err 58 | } 59 | 60 | type Duration struct { 61 | time.Duration 62 | } 63 | 64 | func (d *Duration) UnmarshalJSON(b []byte) (err error) { 65 | var rawDuration string 66 | if err = json.Unmarshal(b, &rawDuration); err != nil { 67 | return err 68 | } 69 | d.Duration, err = time.ParseDuration(rawDuration) 70 | return err 71 | } 72 | 73 | type ActionsCount struct { 74 | Actions map[string]uint64 75 | UniqueCount uint64 76 | TotalCount uint64 77 | } 78 | 79 | func NewActionsCount() *ActionsCount { 80 | return &ActionsCount{ 81 | Actions: make(map[string]uint64), 82 | } 83 | } 84 | 85 | func (a *ActionsCount) Increment(key string) { 86 | a.TotalCount++ 87 | if _, ok := a.Actions[key]; !ok { 88 | a.UniqueCount++ 89 | } 90 | a.Actions[key] += 1 91 | } 92 | 93 | func (a *ActionsCount) Get(key string) uint64 { 94 | return a.Actions[key] 95 | } 96 | 97 | func (a *ActionsCount) Merge(other *ActionsCount) { 98 | for key, value := range other.Actions { 99 | a.Actions[key] += value 100 | } 101 | } 102 | 103 | type NamedCount struct { 104 | Name string 105 | Count uint64 106 | } 107 | 108 | var actionsCountSerializer = structomap.New(). 109 | PickFunc(func(actions interface{}) interface{} { 110 | var results []NamedCount 111 | for name, count := range actions.(map[string]uint64) { 112 | results = append(results, NamedCount{Name: name, Count: count}) 113 | } 114 | sort.Slice(results, func(i, j int) bool { 115 | return results[i].Count > results[j].Count 116 | }) 117 | if len(results) > maxNestedResults { 118 | results = results[:maxNestedResults] 119 | } 120 | return results 121 | }, "Actions"). 122 | Pick("UniqueCount", "TotalCount") 123 | 124 | func (a *ActionsCount) MarshalJSON() ([]byte, error) { 125 | return json.Marshal(actionsCountSerializer.Transform(a)) 126 | } 127 | 128 | func Persist(entity interface{}, outputFile string) error { 129 | file, err := CreateFile(outputFile) 130 | if err != nil { 131 | return err 132 | } 133 | defer file.Close() 134 | encoder := json.NewEncoder(file) 135 | return encoder.Encode(entity) 136 | } 137 | 138 | type TimeGroupedActions struct { 139 | Actions map[time.Time]*GroupedActions 140 | Duration time.Duration 141 | GroupedBy ActionProperty 142 | } 143 | 144 | func NewTimeGroupedActions(duration time.Duration, by ActionProperty) *TimeGroupedActions { 145 | return &TimeGroupedActions{ 146 | Actions: make(map[time.Time]*GroupedActions), 147 | Duration: duration, 148 | GroupedBy: by, 149 | } 150 | } 151 | 152 | func (g *TimeGroupedActions) AddBlock(block Block) { 153 | group := block.Time().Truncate(g.Duration) 154 | if _, ok := g.Actions[group]; !ok { 155 | g.Actions[group] = NewGroupedActions(g.GroupedBy, false) 156 | } 157 | g.Actions[group].AddBlock(block) 158 | } 159 | 160 | func (g *TimeGroupedActions) Result() interface{} { 161 | return g 162 | } 163 | 164 | type TimeGroupedTransactionCount struct { 165 | TransactionCounts map[time.Time]int 166 | GroupedBy time.Duration 167 | } 168 | 169 | func NewTimeGroupedTransactionCount(duration time.Duration) *TimeGroupedTransactionCount { 170 | return &TimeGroupedTransactionCount{ 171 | TransactionCounts: make(map[time.Time]int), 172 | GroupedBy: duration, 173 | } 174 | } 175 | 176 | func (g *TimeGroupedTransactionCount) AddBlock(block Block) { 177 | group := block.Time().Truncate(g.GroupedBy) 178 | if _, ok := g.TransactionCounts[group]; !ok { 179 | g.TransactionCounts[group] = 0 180 | } 181 | g.TransactionCounts[group] += block.TransactionsCount() 182 | } 183 | 184 | func (g *TimeGroupedTransactionCount) Result() interface{} { 185 | return g 186 | } 187 | 188 | type ActionGroup struct { 189 | Name string 190 | Count uint64 191 | Names *ActionsCount 192 | Senders *ActionsCount 193 | Receivers *ActionsCount 194 | } 195 | 196 | var actionGroupSerializer = structomap.New(). 197 | Pick("Name", "Count"). 198 | PickIf(func(a interface{}) bool { 199 | return a.(*ActionGroup).Names.TotalCount > 0 200 | }, "Names", "Senders", "Receivers") 201 | 202 | func (a *ActionGroup) MarshalJSON() ([]byte, error) { 203 | return json.Marshal(actionGroupSerializer.Transform(a)) 204 | } 205 | 206 | func NewActionGroup(name string) *ActionGroup { 207 | return &ActionGroup{ 208 | Name: name, 209 | Count: 0, 210 | Names: NewActionsCount(), 211 | Senders: NewActionsCount(), 212 | Receivers: NewActionsCount(), 213 | } 214 | } 215 | 216 | type GroupedActions struct { 217 | Actions map[string]*ActionGroup 218 | GroupedBy string 219 | BlocksCount uint64 220 | ActionsCount uint64 221 | actionProperty ActionProperty 222 | detailed bool 223 | } 224 | 225 | var groupedActionsSerializer = structomap.New(). 226 | PickFunc(func(actions interface{}) interface{} { 227 | var results []*ActionGroup 228 | for _, action := range actions.(map[string]*ActionGroup) { 229 | results = append(results, action) 230 | } 231 | sort.Slice(results, func(i, j int) bool { 232 | return results[i].Count > results[j].Count 233 | }) 234 | if len(results) > maxTopLevelResults { 235 | results = results[:maxTopLevelResults] 236 | } 237 | return results 238 | }, "Actions"). 239 | Pick("GroupedBy", "BlocksCount", "ActionsCount") 240 | 241 | func (g *GroupedActions) MarshalJSON() ([]byte, error) { 242 | return json.Marshal(groupedActionsSerializer.Transform(g)) 243 | } 244 | 245 | func (g *GroupedActions) Get(key string) *ActionGroup { 246 | return g.Actions[key] 247 | } 248 | 249 | func (g *GroupedActions) GetCount(key string) uint64 { 250 | group := g.Get(key) 251 | if group == nil { 252 | return 0 253 | } 254 | return group.Count 255 | } 256 | 257 | func NewGroupedActions(by ActionProperty, detailed bool) *GroupedActions { 258 | actions := make(map[string]*ActionGroup) 259 | return &GroupedActions{ 260 | Actions: actions, 261 | GroupedBy: by.String(), 262 | BlocksCount: 0, 263 | ActionsCount: 0, 264 | actionProperty: by, 265 | detailed: detailed, 266 | } 267 | } 268 | 269 | func (g *GroupedActions) getActionKey(action Action) string { 270 | switch g.actionProperty { 271 | case ActionName: 272 | return action.Name() 273 | case ActionSender: 274 | return action.Sender() 275 | case ActionReceiver: 276 | return action.Receiver() 277 | default: 278 | panic(fmt.Errorf("no such property %d", g.actionProperty)) 279 | } 280 | } 281 | 282 | func (g *GroupedActions) AddBlock(block Block) { 283 | g.BlocksCount += 1 284 | for _, action := range block.ListActions() { 285 | g.ActionsCount += 1 286 | key := g.getActionKey(action) 287 | if key == "" { 288 | continue 289 | } 290 | actionGroup, ok := g.Actions[key] 291 | if !ok { 292 | actionGroup = NewActionGroup(key) 293 | g.Actions[key] = actionGroup 294 | } 295 | actionGroup.Count += 1 296 | if g.detailed { 297 | actionGroup.Names.Increment(action.Name()) 298 | actionGroup.Senders.Increment(action.Sender()) 299 | actionGroup.Receivers.Increment(action.Receiver()) 300 | } 301 | } 302 | } 303 | 304 | func (g *GroupedActions) Result() interface{} { 305 | return g 306 | } 307 | 308 | type TransactionCounter int 309 | 310 | func NewTransactionCounter() *TransactionCounter { 311 | value := 0 312 | return (*TransactionCounter)(&value) 313 | } 314 | 315 | func (t *TransactionCounter) AddBlock(block Block) { 316 | *t += (TransactionCounter)(block.TransactionsCount()) 317 | } 318 | 319 | func (t *TransactionCounter) Result() interface{} { 320 | return t 321 | } 322 | 323 | type MissingBlocks struct { 324 | Start uint64 325 | End uint64 326 | Seen map[uint64]bool 327 | } 328 | 329 | func NewMissingBlocks(start, end uint64) *MissingBlocks { 330 | return &MissingBlocks{ 331 | Start: start, 332 | End: end, 333 | Seen: make(map[uint64]bool), 334 | } 335 | } 336 | 337 | func (t *MissingBlocks) AddBlock(block Block) { 338 | t.Seen[block.Number()] = true 339 | } 340 | 341 | func (t *MissingBlocks) Compute() []uint64 { 342 | missing := make([]uint64, 0) 343 | for blockNumber := t.Start; blockNumber <= t.End; blockNumber++ { 344 | if _, ok := t.Seen[blockNumber]; !ok { 345 | missing = append(missing, blockNumber) 346 | } 347 | } 348 | return missing 349 | } 350 | 351 | func (t *MissingBlocks) Result() interface{} { 352 | return t.Compute() 353 | } 354 | -------------------------------------------------------------------------------- /cmd/blockchain-analyzer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime/pprof" 9 | "time" 10 | 11 | "github.com/danhper/blockchain-analyzer/core" 12 | "github.com/danhper/blockchain-analyzer/eos" 13 | "github.com/danhper/blockchain-analyzer/processor" 14 | "github.com/danhper/blockchain-analyzer/tezos" 15 | "github.com/danhper/blockchain-analyzer/xrp" 16 | "github.com/urfave/cli/v2" 17 | ) 18 | 19 | func addStartFlag(flags []cli.Flag, required bool) []cli.Flag { 20 | return append(flags, &cli.IntFlag{ 21 | Name: "start", 22 | Aliases: []string{"s"}, 23 | Required: required, 24 | Value: 0, 25 | Usage: "Start block/ledger index", 26 | }) 27 | } 28 | 29 | func addEndFlag(flags []cli.Flag, required bool) []cli.Flag { 30 | return append(flags, &cli.IntFlag{ 31 | Name: "end", 32 | Aliases: []string{"e"}, 33 | Required: required, 34 | Value: 0, 35 | Usage: "End block/ledger index", 36 | }) 37 | } 38 | 39 | func addOutputFlag(flags []cli.Flag) []cli.Flag { 40 | return append(flags, &cli.StringFlag{ 41 | Name: "output", 42 | Aliases: []string{"o"}, 43 | Usage: "Base output filepath", 44 | Required: true, 45 | }) 46 | } 47 | 48 | func addConfigFlag(flags []cli.Flag) []cli.Flag { 49 | return append(flags, &cli.StringFlag{ 50 | Name: "config", 51 | Aliases: []string{"c"}, 52 | Usage: "Configuration file", 53 | Required: true, 54 | }) 55 | } 56 | 57 | func addActionPropertyFlag(flags []cli.Flag) []cli.Flag { 58 | return append(flags, &cli.StringFlag{ 59 | Name: "by", 60 | Value: "name", 61 | Usage: "Property to group the actions by", 62 | }) 63 | } 64 | 65 | func addRangeFlags(flags []cli.Flag, required bool) []cli.Flag { 66 | return addStartFlag(addEndFlag(flags, required), required) 67 | } 68 | 69 | func addFetchFlags(flags []cli.Flag) []cli.Flag { 70 | return addRangeFlags(addOutputFlag(flags), true) 71 | } 72 | 73 | func addPatternFlag(flags []cli.Flag) []cli.Flag { 74 | return append(flags, &cli.StringFlag{ 75 | Name: "pattern", 76 | Aliases: []string{"p"}, 77 | Value: "", 78 | Usage: "Patterns of files to check", 79 | Required: true, 80 | }) 81 | } 82 | 83 | func addGroupDurationFlag(flags []cli.Flag) []cli.Flag { 84 | return append(flags, &cli.StringFlag{ 85 | Name: "duration", 86 | Aliases: []string{"d"}, 87 | Value: "6h", 88 | Usage: "Duration to group by when counting", 89 | }) 90 | } 91 | 92 | func addDetailedFlag(flags []cli.Flag) []cli.Flag { 93 | return append(flags, &cli.BoolFlag{ 94 | Name: "detailed", 95 | Usage: "Whether to add the details about sender/receivers etc", 96 | Value: false, 97 | Required: false, 98 | }) 99 | } 100 | 101 | func addCpuProfileFlag(flags []cli.Flag) []cli.Flag { 102 | return append(flags, &cli.StringFlag{ 103 | Name: "cpu-profile", 104 | Usage: "Path where to store the CPU profile", 105 | Value: "", 106 | Required: false, 107 | }) 108 | } 109 | 110 | func makeAction(f func(*cli.Context) error) func(*cli.Context) error { 111 | return func(c *cli.Context) error { 112 | cpuProfile := c.String("cpu-profile") 113 | if cpuProfile != "" { 114 | f, err := os.Create(cpuProfile) 115 | if err != nil { 116 | return fmt.Errorf("could not create CPU profile: %s", err.Error()) 117 | } 118 | defer f.Close() 119 | if err := pprof.StartCPUProfile(f); err != nil { 120 | return fmt.Errorf("could not start CPU profile: %s", err.Error()) 121 | } 122 | defer pprof.StopCPUProfile() 123 | } 124 | 125 | return f(c) 126 | } 127 | } 128 | 129 | func addCommonCommands(blockchain core.Blockchain, commands []*cli.Command) []*cli.Command { 130 | return append(commands, []*cli.Command{ 131 | { 132 | Name: "fetch", 133 | Flags: addFetchFlags(nil), 134 | Usage: "Fetches blockchain data", 135 | Action: makeAction(func(c *cli.Context) error { 136 | return blockchain.FetchData(c.String("output"), c.Uint64("start"), c.Uint64("end")) 137 | }), 138 | }, 139 | { 140 | Name: "check", 141 | Flags: addPatternFlag(addFetchFlags(nil)), 142 | Usage: "Checks for missing blocks in data", 143 | Action: makeAction(func(c *cli.Context) error { 144 | return processor.OutputAllMissingBlockNumbers( 145 | blockchain, c.String("pattern"), c.String("output"), 146 | c.Uint64("start"), c.Uint64("end")) 147 | }), 148 | }, 149 | { 150 | Name: "count-transactions", 151 | Flags: addPatternFlag(addRangeFlags(nil, false)), 152 | Usage: "Count the number of transactions in the data", 153 | Action: makeAction(func(c *cli.Context) error { 154 | count, err := processor.CountTransactions( 155 | blockchain, c.String("pattern"), 156 | c.Uint64("start"), c.Uint64("end")) 157 | if err != nil { 158 | return err 159 | } 160 | fmt.Printf("found %d transactions\n", count) 161 | return nil 162 | }), 163 | }, 164 | { 165 | Name: "group-actions", 166 | Flags: addDetailedFlag(addActionPropertyFlag( 167 | addPatternFlag(addOutputFlag(addRangeFlags(nil, false))))), 168 | Usage: "Count and groups the number of \"actions\" in the data", 169 | Action: makeAction(func(c *cli.Context) error { 170 | actionProperty, err := core.GetActionProperty(c.String("by")) 171 | if err != nil { 172 | return err 173 | } 174 | counts, err := processor.GroupActions( 175 | blockchain, c.String("pattern"), 176 | c.Uint64("start"), c.Uint64("end"), 177 | actionProperty, c.Bool("detailed")) 178 | if err != nil { 179 | return err 180 | } 181 | return core.Persist(counts, c.String("output")) 182 | }), 183 | }, 184 | { 185 | Name: "group-actions-over-time", 186 | Flags: addActionPropertyFlag(addGroupDurationFlag( 187 | addPatternFlag(addOutputFlag(addRangeFlags(nil, false))))), 188 | Usage: "Count and groups per time the number of \"actions\" in the data", 189 | Action: makeAction(func(c *cli.Context) error { 190 | duration, err := time.ParseDuration(c.String("duration")) 191 | if err != nil { 192 | return err 193 | } 194 | actionProperty, err := core.GetActionProperty(c.String("by")) 195 | if err != nil { 196 | return err 197 | } 198 | counts, err := processor.CountActionsOverTime( 199 | blockchain, c.String("pattern"), 200 | c.Uint64("start"), c.Uint64("end"), 201 | duration, actionProperty) 202 | if err != nil { 203 | return err 204 | } 205 | return core.Persist(counts, c.String("output")) 206 | }), 207 | }, 208 | { 209 | Name: "count-transactions-over-time", 210 | Flags: addGroupDurationFlag(addPatternFlag(addOutputFlag(addRangeFlags(nil, false)))), 211 | Usage: "Count number of \"transactions\" over time in the data", 212 | Action: makeAction(func(c *cli.Context) error { 213 | duration, err := time.ParseDuration(c.String("duration")) 214 | if err != nil { 215 | return err 216 | } 217 | counts, err := processor.CountTransactionsOverTime( 218 | blockchain, c.String("pattern"), 219 | c.Uint64("start"), c.Uint64("end"), duration) 220 | if err != nil { 221 | return err 222 | } 223 | return core.Persist(counts, c.String("output")) 224 | }), 225 | }, 226 | { 227 | Name: "bulk-process", 228 | Flags: addConfigFlag(addOutputFlag(nil)), 229 | Usage: "Bulk process the data according to the given configuration file", 230 | Action: makeAction(func(c *cli.Context) error { 231 | file, err := os.Open(c.String("config")) 232 | if err != nil { 233 | return err 234 | } 235 | defer file.Close() 236 | 237 | var config processor.BulkConfig 238 | if err := json.NewDecoder(file).Decode(&config); err != nil { 239 | return err 240 | } 241 | result, err := processor.RunBulkActions(blockchain, config) 242 | if err != nil { 243 | return err 244 | } 245 | return core.Persist(result, c.String("output")) 246 | }), 247 | }, 248 | { 249 | Name: "export", 250 | Flags: addPatternFlag(addOutputFlag(addRangeFlags(nil, false))), 251 | Usage: "Export a subset of the fields to msgpack format for faster processing", 252 | Action: makeAction(func(c *cli.Context) error { 253 | return processor.ExportToMsgpack(blockchain, c.String("pattern"), 254 | c.Uint64("start"), c.Uint64("end"), c.String("output")) 255 | }), 256 | }, 257 | }...) 258 | } 259 | 260 | var eosCommands []*cli.Command = []*cli.Command{ 261 | { 262 | Name: "export-transfers", 263 | Flags: addPatternFlag(addOutputFlag(addRangeFlags(nil, false))), 264 | Usage: "Export all the transfers to a CSV file", 265 | Action: makeAction(func(c *cli.Context) error { 266 | return eos.ExportTransfers( 267 | c.String("pattern"), 268 | c.Uint64("start"), c.Uint64("end"), c.String("output")) 269 | }), 270 | }, 271 | } 272 | 273 | func main() { 274 | app := &cli.App{ 275 | Usage: "Tool to fetch and analyze blockchain transactions", 276 | Flags: addCpuProfileFlag(nil), 277 | Commands: []*cli.Command{ 278 | { 279 | Name: "eos", 280 | Usage: "Analyze EOS data", 281 | Subcommands: addCommonCommands(eos.New(), eosCommands), 282 | }, 283 | { 284 | Name: "tezos", 285 | Usage: "Analyze Tezos data", 286 | Subcommands: addCommonCommands(tezos.New(), nil), 287 | }, 288 | { 289 | Name: "xrp", 290 | Usage: "Analyze XRP data", 291 | Subcommands: addCommonCommands(xrp.New(), nil), 292 | }, 293 | }, 294 | } 295 | 296 | err := app.Run(os.Args) 297 | if err != nil { 298 | log.Fatal(err) 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /core/fixtures/tezos-blocks.jsonl: -------------------------------------------------------------------------------- 1 | {"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"BLc7tKfzia9hnaY1YTMS6RkDniQBoApM4EjKFRLucsuHbiy3eqt","header":{"level":10000,"proto":1,"predecessor":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","timestamp":"2018-07-07T17:06:27Z","validation_pass":4,"operations_hash":"LLob71uMBRtLaKGj3sDJmAT7VEdGTtEoogrbFFnPjxXiYfDmUQrgr","fitness":["00","000000000004fff6"],"context":"CoUnq1qGxUtidFCdcaCWXEQdefFDSdBTpjnYVcrHJ1cKYqL6HLiA","priority":0,"proof_of_work_nonce":"d4dac173a3904dfc","signature":"sigRg6mM8oEt5y7nzSwi34P3UEoNDYjHF2Nik9s2f7xFGzMbbgmVYrc3uXdAKPF3ayDLv7vaEN4U2ZeDC69EJp4keYphw9WQ"},"metadata":{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","next_protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","test_chain_status":{"status":"not_running"},"max_operations_ttl":60,"max_operation_data_length":16384,"max_block_header_length":238,"max_operation_list_length":[{"max_size":32768,"max_op":32},{"max_size":32768},{"max_size":135168,"max_op":132},{"max_size":524288}],"baker":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","level":{"level":10000,"level_position":9999,"cycle":2,"cycle_position":1807,"voting_period":0,"voting_period_position":9999,"expected_commitment":false},"voting_period_kind":"proposal","nonce_hash":null,"consumed_gas":"0","deactivated":[],"balance_updates":[{"kind":"contract","contract":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","change":"-16000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","level":2,"change":"16000000"}]},"operations":[[{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oo5otsjk5Rrs3dvSemoV6Ni3bVJSVt9tM51edyiPLTyzwjMJBZK","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","change":"-10000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","level":2,"change":"10000000"}],"delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","slots":[30,26,22,17,9]}}],"signature":"siggHkiwKvLDRmA7UUgWJewieSr42Dj1Xzgo2JqafxCtkoeXae8B2c8vokJbfhKzaqpftMoPvvDjXzx8k1zvSDmazC9W4nMn"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onuw2K8sTenxbVG857d64JJ9oYFm5GiUKSQGBqwPBqAvhz3jQe9","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","change":"-6000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","level":2,"change":"6000000"}],"delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","slots":[24,15,4]}}],"signature":"sigSJC4HoiA5UJo5yHeTSgT4kSAortGj6KNdRFUUmUPWUXucfgGSUEzQmaSL4JaNds6MQN1tVPGZBTkMYonxBvJB5fiH8Psa"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooC5JaC1nXfvaT89WiJKjEonWvGeWnmRTdQc1wYQ3okHx4kkGvu","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","change":"-4000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","level":2,"change":"4000000"}],"delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","slots":[28,10]}}],"signature":"sigmyorF4narTBSNMXemm6M6ggbNpWsCeFZGhYfR4QEutkBNXgKV3c33uiMx649zHCFsoKK6PX5SRbv8y1SMn94F53tANkEL"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooFBHKYXySSzanGtLhzT5TzyvSZUfA2mFXAYjJidhXdujgJvThn","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","change":"-10000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","level":2,"change":"10000000"}],"delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","slots":[29,16,5,3,0]}}],"signature":"sigSJm6WS5rxkFdZFsuBboDMNeubGE5bAScK2rVVfx9XXSmCXAAVN7vxm9LaAMAXkzunvkLK8SkQhTynGCC7PiK9UAFwchex"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onfFpbNQzmkrtsFk8tGgo8uXYRuEhjonni7paNkirMG4P47G8fQ","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","change":"-14000000"},{"kind":"freezer","category":"deposits","delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","level":2,"change":"14000000"}],"delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","slots":[27,25,23,20,12,8,6]}}],"signature":"sigufzbcHLNgdUgfg2PkQbVrJZTpE5JhgDtHZRUsSGNdxDg2zj53g2LzyVyX6nK88xDwMFh626auHgtPNssfSueEypAjoDdc"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooHTyojDYarDp12mqujgcgszaGNZjjSR8m5NTukEqxtWktv9Hpt","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","change":"-4000000"},{"kind":"freezer","category":"deposits","delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","level":2,"change":"4000000"}],"delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","slots":[13,2]}}],"signature":"sigY3X2mjS6nRGVGhDaUdZeJQeLcqbcp5ZGrMoK8EHW19k5V2xchaxp3FVF4sbrhFfZQyz4jzsU1Q8TqXWf1VtfFznZ2v9wN"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oniGQzstxfFeqfvAfSwEH632R6rP41thv8zUKKpYEBEAPezEKh7","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","level":2,"change":"8000000"}],"delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","slots":[31,19,7,1]}}],"signature":"sigePU7QxfEJUo4LnpWWMaGf5eTb95VP3BXjR5NZPNJxoKemkriftuRBZ2jfTh67xUkitKF4beNbXPDx4qvpTjmVbYtJb8Gm"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooX8hcwA2Nx8fhgGWDqaT5ZWtUeVm77TGMnGxWXM5c1CVkGALou","branch":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","contents":[{"kind":"endorsement","level":9999,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","level":2,"change":"8000000"}],"delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","slots":[21,18,14,11]}}],"signature":"sigw9g4z8mRd9JMUpxxaJRxUTbHq4evpVAHFqsTY4foh83853pmbuNXwUgRNwtRgnzBQV4iHyQ4xeUjbwk8aWDSr1FVG1qGk"}],[],[],[]]} 2 | {"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"BMG7bSzAh1is2896bUkK7RnUREqqN4BjcH4J7YgkFKcNHWNe4cM","header":{"level":9999,"proto":1,"predecessor":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","timestamp":"2018-07-07T17:05:27Z","validation_pass":4,"operations_hash":"LLoZmTxcm3ntcFGpqAPXn5uAxfb3F9o1CT5oLobVjqzeeKwhmvvu9","fitness":["00","000000000004ffd5"],"context":"CoVs31pKopFNpiRFe4VVBp5rvg6N5J53om5ARzGwS3R4SjqvTmp6","priority":0,"proof_of_work_nonce":"44f03fdaa4c64b0d","signature":"sigWtFQtNV3NG1ATAw9hQSyrG2AEQs88tFZTnYio29avmGTL2hwvfGs1YPikp5i5q4LVtHbtoYhXSdfDNNNpWJn7o2DDDHpU"},"metadata":{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","next_protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","test_chain_status":{"status":"not_running"},"max_operations_ttl":60,"max_operation_data_length":16384,"max_block_header_length":238,"max_operation_list_length":[{"max_size":32768,"max_op":32},{"max_size":32768},{"max_size":135168,"max_op":132},{"max_size":524288}],"baker":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","level":{"level":9999,"level_position":9998,"cycle":2,"cycle_position":1806,"voting_period":0,"voting_period_position":9998,"expected_commitment":false},"voting_period_kind":"proposal","nonce_hash":null,"consumed_gas":"0","deactivated":[],"balance_updates":[{"kind":"contract","contract":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","change":"-16000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","level":2,"change":"16000000"}]},"operations":[[{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onw38WZtXgqhVE6FUxdJzgL7HEt16S3j48xPWMvGqpfnag9x1qs","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","level":2,"change":"8000000"}],"delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","slots":[29,27,25,22]}}],"signature":"sigToMGzRHPdFj53hTcsa5CwxR5GRQAwv8cpWpPg1m2TKEz34QGhPDLgLjkAvxbkojMLgPBaMLqDEq2AHBiAowgqHHvKNz7o"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oo4rhnKi44yEy9hNooZNLjXWShWXNqh4A8xYT8tT7odDoYzLSoi","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","change":"-2000000"},{"kind":"freezer","category":"deposits","delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","level":2,"change":"2000000"}],"delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","slots":[11]}}],"signature":"sigmik3XwDHVEfsdHHheoHJ1rZCGyDTssAZJ3tWRqMgJSuPWd19KjnGXv2k6Rv6KwGbUW6RmUviHRMpim8Bs6TeV57uPkwW6"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oopENA5bERMCynEghpbQPJxyTNy85czbWfw7RvC1MB69x2MZt89","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","change":"-18000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","level":2,"change":"18000000"}],"delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","slots":[30,20,18,17,15,14,3,2,1]}}],"signature":"sigTqSm5Xtu4EhgZM4z8GFbMaNPd3i7xMzGWhgfvJ1cStPAwHJgmjArb1vPf5geVbKFJtTd34MNFPThypV5wSAb4GZxwNKfp"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onidm1WZehM54BJZrTxQkAWZXv63nNeWTjhhjrR93QPQ5uaxzjR","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","change":"-2000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","level":2,"change":"2000000"}],"delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","slots":[8]}}],"signature":"sigppBPLQSsSUCLbLA7ijCAU3RSEqLhEqjNUfPcvm4r8tuTa4GZcvmyNm3TBvUVcFsuxd8hVDrFdQnXM9dfoZSBUskQ76G8c"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oo2wKJx1tp97EtnaJjpZt2Sc2oiicD2z6giq32cDLSfXe8vB36x","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","change":"-6000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","level":2,"change":"6000000"}],"delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","slots":[16,13,5]}}],"signature":"sigoM2GmN3Uox3QvjfvLFV9eAErwcvotJ2yvVzgjTkXztczVAict8L27HdY6rgUxEYLARuVDrZdK4e9DhVsg7U8gLMjT7dMd"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onneC45xtUvbLZ7tDJWkJz4FaCZQRKE6zve3McuexRfRvX8FNZy","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","change":"-10000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","level":2,"change":"10000000"}],"delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","slots":[28,24,10,7,4]}}],"signature":"sigWNnEnnsknfy6sfPJPF3jc5f8TDGKBb5oFtD7xhGm5164spFtbd6PCb7HveEgdJ12CQ14HafFtv2oMFYJemDFgjDprdKf5"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"opMH8P1mZ4q52KdQ1KiThkfzzYQaCuF7RGA72LN7fFUa2VcPrgy","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","level":2,"change":"8000000"}],"delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","slots":[19,12,9,0]}}],"signature":"sigtbkh4wBqaGR6pqFiQywdVJN53SnuP6WbB89h16PVxg1T3KCmR9EWVDBxZ2MT3tRgv5bDGoVBSorDYsxF8fTi8XstMCQC1"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oowS64R3KE3eW6SEfm4XZqEbGQ5tvEJgPbefPuMQR3Tt7jEfnjJ","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"endorsement","level":9998,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","change":"-10000000"},{"kind":"freezer","category":"deposits","delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","level":2,"change":"10000000"}],"delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","slots":[31,26,23,21,6]}}],"signature":"sigSPsGh5cTP1YSiqk7UB7XVtRZDPZc5ZUCUEqbUyoXbgCNqCfrE1UErF4ds1HZdUFibLTb9GDsfKLkfNYyvpD4z9Cxn6Zim"}],[],[],[{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooVRTkR8CibdPyvq5NezNmTdaLZ5RVbSB3kjzUyXDMxbLNrDMJV","branch":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","contents":[{"kind":"delegation","source":"KT1BoXKVVDAWLNzxViExruoq1a16AT6vRmkM","fee":"0","counter":"5","gas_limit":"0","storage_limit":"0","delegate":"tz1YKh8T79LAtWxX29N5VedCSmaZGw9LNVxQ","metadata":{"balance_updates":[],"operation_result":{"status":"applied"}}}],"signature":"signHGUoMSsYQaBhzAybJKrdLeR4sfAR32uG3icETXyNXH7KuuZkPPSnjATJK9bpjow7cA4KBUr6pwdii4AJ6876pL1puGzr"}]]} 3 | {"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"BMGD4TujH9BVggcLAabrMPutmwTw354ikszPR7gazmBhwBKfTS2","header":{"level":9998,"proto":1,"predecessor":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","timestamp":"2018-07-07T17:04:27Z","validation_pass":4,"operations_hash":"LLob4EuBRmhR7mtDnMu75WL2KjHFw3u8jG5DDuMTvztKfwTbgLpw6","fitness":["00","000000000004ffb4"],"context":"CoV24RGEbVfhCWAXzivvgRYgQRe4g9bVXtGaVx7pGffkPrzCSQae","priority":0,"proof_of_work_nonce":"012f608e3c0a6273","signature":"sigpFpbFXCVDn8ybnhWJE7kn2fYnn5aVBeVzyuysuJ4byv6WtZ51gx6Z6AjhW4Z87HzYMp45fwYFNNmmVkQwiFCzKLZ3LBzn"},"metadata":{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","next_protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","test_chain_status":{"status":"not_running"},"max_operations_ttl":60,"max_operation_data_length":16384,"max_block_header_length":238,"max_operation_list_length":[{"max_size":32768,"max_op":32},{"max_size":32768},{"max_size":135168,"max_op":132},{"max_size":524288}],"baker":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","level":{"level":9998,"level_position":9997,"cycle":2,"cycle_position":1805,"voting_period":0,"voting_period_position":9997,"expected_commitment":false},"voting_period_kind":"proposal","nonce_hash":null,"consumed_gas":"0","deactivated":[],"balance_updates":[{"kind":"contract","contract":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","change":"-16000000"},{"kind":"freezer","category":"deposits","delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","level":2,"change":"16000000"}]},"operations":[[{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooAaYdahUHyazyNTn6oxoT8rzj21LyG3kU9CraBWkPgXFHqxgCR","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","change":"-12000000"},{"kind":"freezer","category":"deposits","delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","level":2,"change":"12000000"}],"delegate":"tz3WMqdzXqRWXwyvj5Hp2H7QEepaUuS7vd9K","slots":[29,22,20,18,13,10]}}],"signature":"sigiptX3rUwPZDYpL6h45us9R6ybPvfcxREBkGZkWzojgZu79EHcCyCb5SGJt9QXAeNbLTXmKAUV9w4SWsoMMpEGhz1Y3ncS"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooNB8Wac4GmY6r6cw9Dux7PBzXCKT1rtrS99sMzWvwWX2NshRv9","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","change":"-6000000"},{"kind":"freezer","category":"deposits","delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","level":2,"change":"6000000"}],"delegate":"tz3VEZ4k6a4Wx42iyev6i2aVAptTRLEAivNN","slots":[31,25,24]}}],"signature":"sighoSf7ppRtDvTQf8WMFkk7wXdy514jnWJi6VLBhQ4z5ZHgXmJ4X2DEDLrrTYu425JpSkYmpPTyCSDJvPPnG2prM4Q4Rmw3"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"onnZZTNce8bBF7cBh6zSj1Yt4mEQNwynEuEh95PM1uq5BGTAiG3","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","level":2,"change":"8000000"}],"delegate":"tz3NExpXn9aPNZPorRE4SdjJ2RGrfbJgMAaV","slots":[27,23,14,12]}}],"signature":"sigkS2eSCkRtrTGeWEUK5LTrgzpM9u4yC5JTgcdJH7JiWqFhCUjXP2mc3Yc1efZGGzAWowzdYmBHiiQRgqdpdtU8P61LcDio"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"opWmUCXy4p9a6oT7jawgMLMdoKavUZZuEyvtA1vnPJC6XVbbKTJ","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","change":"-6000000"},{"kind":"freezer","category":"deposits","delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","level":2,"change":"6000000"}],"delegate":"tz3UoffC7FG7zfpmvmjUmUeAaHvzdcUvAj6r","slots":[17,16,1]}}],"signature":"sigNRvHv5QdEnaXGejEXepT11hZAfHWoVpSkuoL4ndtu3a5EuREpsDesUzeSu8PWqaHxuxaFaoweveGEJJ8odoYzfsGxgb1n"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"oopbvgZFLh8LPeDemFdqAhbM9j4NuH3pypgTeTNqyF1oskLBQVn","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","change":"-6000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","level":2,"change":"6000000"}],"delegate":"tz3bTdwZinP8U1JmSweNzVKhmwafqWmFWRfk","slots":[30,11,0]}}],"signature":"sigkZvcXXwvYX2w9EYEyKB4tdEpHKNj29KSc7rwZM7n1QPtGZr1ZrQ7YW9v1bAuuKfkcP1NsSqXWk68od2ARphWgbodvJEPX"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooN7xrRihm8epZKfdvaCFoDNJR85iDb8i4NDu7QvKdRQG4GgK7v","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","change":"-10000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","level":2,"change":"10000000"}],"delegate":"tz3RB4aoyjov4KEVRbuhvQ1CKJgBJMWhaeB8","slots":[28,15,9,5,4]}}],"signature":"sigXtRRVzZZ854hLNSeQShBn5TwQCbs75FX5vWikTCxrfd2WPPbyczr3Xdg1tKU47mYSpoK4bXzCFR9P3nHvdbfqWAZJduB7"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"opVrcK5rRXN63JECBFfSgrwE7jf6sYuvGsWxwTPLMPuNWcEN7QQ","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","level":2,"change":"8000000"}],"delegate":"tz3bvNMQ95vfAYtG8193ymshqjSvmxiCUuR5","slots":[26,6,3,2]}}],"signature":"sigt12EQnqBUYYYv3NUFLTUV8zj1vqRz5KJYtbi3Fq5sbFy6tEZBUpDuJQv4cDtBerNzYqePksVuEbrmVyWSU1Fw2ViJcR9C"},{"protocol":"PtCJ7pwoxe8JasnHY8YonnLYjcVHmhiARPJvqcC6VfHT5s8k8sY","chain_id":"NetXdQprcVkpaWU","hash":"ooqUiphfdgNKPKceCH4mfLvg2iQapC4fGF8JSSqZF57TPWLSVv4","branch":"BLfbiAUCREfj5b3qhLBmbSPoYouas3o7y6WQ8jqzmPDrHxk9DLi","contents":[{"kind":"endorsement","level":9997,"metadata":{"balance_updates":[{"kind":"contract","contract":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","change":"-8000000"},{"kind":"freezer","category":"deposits","delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","level":2,"change":"8000000"}],"delegate":"tz3RDC3Jdn4j15J7bBHZd29EUee9gVB1CxD9","slots":[21,19,8,7]}}],"signature":"siguvpFycfbPevqW4LMYWwnyWPxTidGRRhhYtXaorUAUaVCH4j4G3fji5giEh8KHPsVzqGuTzsdFEpovAXKxghVmQRXkEDUw"}],[],[],[]]} 4 | --------------------------------------------------------------------------------