├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── README.md ├── chainsync ├── __init__.py ├── adapters │ ├── README.md │ ├── __init__.py │ ├── abstract │ │ ├── __init__.py │ │ └── adapter.py │ ├── base │ │ ├── __init__.py │ │ └── base.py │ ├── decent │ │ ├── __init__.py │ │ └── decent.py │ ├── muse │ │ ├── __init__.py │ │ └── muse.py │ ├── peerplays │ │ ├── __init__.py │ │ └── peerplays.py │ ├── steem │ │ ├── __init__.py │ │ └── steem.py │ └── steemv2 │ │ ├── __init__.py │ │ └── steem.py ├── chainsync.py └── clients │ ├── http │ └── rpc.py │ └── ws │ └── rpc.py ├── examples ├── adapter_decent.py ├── adapter_muse.py ├── adapter_peerplays.py ├── adapter_steem.py ├── adapter_steemv2.py ├── async_mongo_op_indexer.py └── simple_mongo_op_indexer.py ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── chainsync ├── test_base.py ├── test_chainsync.py ├── test_chainsync_adapter.py ├── test_chainsync_from_block_get_ops.py ├── test_chainsync_from_block_get_ops_regular.py ├── test_chainsync_get_block.py ├── test_chainsync_get_block_sequence.py ├── test_chainsync_get_blocks.py ├── test_chainsync_get_config.py ├── test_chainsync_get_head_block.py ├── test_chainsync_get_ops_in_block.py ├── test_chainsync_get_ops_in_block_sequence.py ├── test_chainsync_get_ops_in_blocks.py ├── test_chainsync_get_ops_in_transaction.py ├── test_chainsync_get_ops_in_transactions.py ├── test_chainsync_get_status.py ├── test_chainsync_get_stream.py ├── test_chainsync_get_transaction.py ├── test_chainsync_get_transactions.py ├── test_chainsync_op_formatters.py ├── test_chainsync_stream.py ├── test_chainsync_yield_event.py └── test_chainsync_yield_event_with_plugins.py ├── conftest.py └── test_import.py /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .venv 3 | __pycache__ 4 | dist 5 | *.egg-info 6 | build 7 | chainsync/adapters/steemv2test 8 | publish.sh 9 | .pytest_* 10 | *.pyc 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.4" 4 | - "3.5" 5 | - "3.5-dev" # 3.5 development branch 6 | - "3.6" 7 | - "3.6-dev" # 3.6 development branch 8 | - "3.7-dev" # 3.7 development branch 9 | install: 10 | - pip install .[test] 11 | script: 12 | - pytest -k "not transaction" # don't test transaction methods - not supported on many APIs 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Aaron Cox 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ChainSync 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/chainsync.svg)](https://github.com/aaroncox/chainsync) 4 | [![GitHub issues](https://img.shields.io/github/issues/aaroncox/chainsync.svg)](https://github.com/aaroncox/chainsync/issues) 5 | [![Build Status](https://travis-ci.org/aaroncox/chainsync.svg?branch=master)](https://travis-ci.org/aaroncox/chainsync) 6 | 7 | A simple library to stream blocks and operations for digesting into other mediums. 8 | 9 | ### Install 10 | 11 | `pip install chainsync` 12 | 13 | 14 | #### Requirements 15 | 16 | `pip install -r requirements.txt` 17 | 18 | - [jsonrpcclient](https://github.com/bcb/jsonrpcclient) 19 | 20 | ### Example - streaming blocks 21 | 22 | ``` python 23 | from chainsync import ChainSync 24 | from chainsync.adapters.steemv2 import SteemV2Adapter 25 | 26 | adapter = SteemV2Adapter(endpoints=['https://api.steemit.com']) 27 | chainsync = ChainSync(adapter) 28 | 29 | for dataType, data in chainsync.stream(['blocks']): 30 | print("{} - {}".format(data['block_num'], data['witness'])) 31 | ``` 32 | 33 | ### Example - streaming operations 34 | 35 | ``` python 36 | from chainsync import ChainSync 37 | from chainsync.adapters.steemv2 import SteemV2Adapter 38 | 39 | adapter = SteemV2Adapter(endpoints=['https://api.steemit.com']) 40 | chainsync = ChainSync(adapter) 41 | 42 | for dataType, data in chainsync.stream(['ops']): 43 | print("{} - {}".format(data['block_num'], data['operation_type'])) 44 | ``` 45 | 46 | ### Example - streaming operations with a whitelist 47 | 48 | ``` python 49 | from chainsync import ChainSync 50 | from chainsync.adapters.steemv2 import SteemV2Adapter 51 | 52 | adapter = SteemV2Adapter(endpoints=['https://api.steemit.com']) 53 | chainsync = ChainSync(adapter) 54 | 55 | for dataType, op in chainsync.stream(['ops'], whitelist=['vote']): 56 | print("{} - {} by {}".format(op['block_num'], op['operation_type'], op['voter'])) 57 | ``` 58 | 59 | ### Example - custom adapters 60 | 61 | A custom adapter can be supplied to allow parsing of a specific blockchain 62 | 63 | ``` python 64 | from chainsync import ChainSync 65 | from chainsync.adapters.decent import DecentAdapter 66 | 67 | adapter = DecentAdapter(endpoints=['http://api.decent-db.com:8090']) 68 | chainsync = ChainSync(adapter) 69 | 70 | for block in chainsync.stream(['blocks']): 71 | print("{} - {}".format(block['block_num'], block['witness'])) 72 | ``` 73 | 74 | ## Adapters 75 | 76 | Adapters can be added and configured to allow access to other similar blockchains. 77 | 78 | A current list of adapters can be found in the `./chainsync/adapters` folder. 79 | -------------------------------------------------------------------------------- /chainsync/__init__.py: -------------------------------------------------------------------------------- 1 | from .chainsync import ChainSync 2 | -------------------------------------------------------------------------------- /chainsync/adapters/README.md: -------------------------------------------------------------------------------- 1 | Each adapter should always have the following abstract methods, regardless of what blockchain it's connecting to: 2 | 3 | - get_block 4 | - get_blocks 5 | -------------------------------------------------------------------------------- /chainsync/adapters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaroncox/chainsync/1277363787e37aa595571ab8a789831aadf0d3e6/chainsync/adapters/__init__.py -------------------------------------------------------------------------------- /chainsync/adapters/abstract/__init__.py: -------------------------------------------------------------------------------- 1 | from .adapter import AbstractAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/abstract/adapter.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | class AbstractAdapter(ABC): 4 | 5 | @property 6 | @abstractmethod 7 | def config(self): 8 | """ gets current configuration for the blockchain 9 | """ 10 | pass 11 | 12 | @abstractmethod 13 | def get_config(self): 14 | pass 15 | 16 | @abstractmethod 17 | def get_methods(self): 18 | pass 19 | 20 | @abstractmethod 21 | def get_status(self): 22 | pass 23 | 24 | @abstractmethod 25 | def get_block(self, height): 26 | """ retrieve a block using the `get_block` method 27 | 28 | :param int height: the block to retrieve 29 | """ 30 | pass 31 | 32 | @abstractmethod 33 | def get_blocks(self, blocks=[]): 34 | """ retrieve a block using the `get_block` method 35 | 36 | :param int blocks: a list of the blocks to retrieve 37 | """ 38 | pass 39 | 40 | @abstractmethod 41 | def get_ops_in_block(self, block_num): 42 | pass 43 | 44 | @abstractmethod 45 | def get_ops_in_blocks(self, start_block, virtual_only, blocks): 46 | pass 47 | 48 | @abstractmethod 49 | def get_transaction(self, transaction_id=1): 50 | pass 51 | 52 | @abstractmethod 53 | def get_transactions(self, transaction_ids=[]): 54 | pass 55 | 56 | @abstractmethod 57 | def format_op_from_get_block(self, block, op, txIndex=False, opIndex=False): 58 | pass 59 | 60 | @abstractmethod 61 | def format_op_from_get_ops_in_block(self, op): 62 | pass 63 | 64 | @abstractmethod 65 | def format_op_from_get_transaction(self, tx, op, txIndex=False, opIndex=False): 66 | pass 67 | -------------------------------------------------------------------------------- /chainsync/adapters/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/base/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | import time 4 | 5 | from urllib.parse import urlparse 6 | from websocket._exceptions import WebSocketConnectionClosedException 7 | 8 | from chainsync.clients.http.rpc import Client as http_client 9 | from chainsync.clients.ws.rpc import Client as ws_client 10 | 11 | chainsync_defaults = { 12 | 'endpoint': 'http://localhost:8090', 13 | 'endpoints': ['http://localhost:8090'], 14 | } 15 | 16 | 17 | class BaseAdapter(): 18 | 19 | debug = False 20 | 21 | def __init__(self, endpoints=None, retry=True, debug=False): 22 | 23 | self.debug = debug 24 | # print("debug: {}".format(self.debug)) 25 | self.retry = retry 26 | # print("retry: {}".format(self.retry)) 27 | self.apis = set() 28 | # print("apis: {}".format(self.apis)) 29 | self.api_methods = set() 30 | # print("api_methods: {}".format(self.api_methods)) 31 | 32 | # Disable the additional logging unless debug is active 33 | if not debug: 34 | request_logger = logging.getLogger('jsonrpcclient.client.request') 35 | response_logger = logging.getLogger('jsonrpcclient.client.response') 36 | request_logger.setLevel(logging.getLevelName('WARNING')) 37 | response_logger.setLevel(logging.getLevelName('WARNING')) 38 | 39 | # Use endpoints or set defaults 40 | if isinstance(endpoints, list): 41 | self.endpoint = endpoints[0] if endpoints else chainsync_defaults['endpoint'] 42 | self.endpoints = endpoints if endpoints else chainsync_defaults['endpoints'] 43 | elif isinstance(endpoints, str): 44 | self.endpoint = endpoints if endpoints else chainsync_defaults['endpoint'] 45 | self.endpoints = [self.endpoint] 46 | else: 47 | self.endpoint = chainsync_defaults['endpoint'] 48 | self.endpoints = chainsync_defaults['endpoints'] 49 | 50 | # Determine what type of client this adapter needs 51 | self.init_client() 52 | 53 | # Save a modifiable copy of the endpoints available 54 | self.additional_endpoints = self.endpoints[:] 55 | 56 | # Remove the active endpoint from the additional_endpoints pool 57 | if self.endpoint in self.additional_endpoints: 58 | self.additional_endpoints.remove(self.endpoint) 59 | 60 | # Determine endpoint capabilities 61 | self.get_available_apis() 62 | 63 | def init_client(self): 64 | if urlparse(self.endpoint).scheme in ['ws', 'wss']: 65 | print("init rpc client") 66 | self.client = ws_client(self.endpoint) 67 | else: 68 | print("init http client") 69 | self.client = http_client(self.endpoint) 70 | 71 | def get_available_apis(self): 72 | # Get available methods from the current adapter 73 | available_methods = self.call('get_methods') 74 | # If available methods is returned as false, assume everything is valid 75 | if available_methods == 'NOT_SUPPORTED': 76 | print("get_methods not supported") 77 | else: 78 | self.apis = set() 79 | self.api_methods = set() 80 | for call in available_methods: 81 | api, method = call.split('.') 82 | self.apis.add(api) 83 | self.api_methods.add(method) 84 | 85 | def is_api_available(self, api, method, raiseException=True): 86 | if api not in self.apis: 87 | if raiseException: 88 | raise Exception('endpoint not capable of calling {}.{} ({} not available)'.format(api, method, api)) 89 | else: 90 | return False 91 | return True 92 | 93 | def call(self, method, raiseException=True, **kwargs): 94 | try: 95 | # Logging 96 | # print("\n") 97 | if self.debug: 98 | print("call: {} ({})".format(method, kwargs)) 99 | # print("endpoint: {}".format(self.endpoint)) 100 | # print("endpoints: {}".format(self.endpoints)) 101 | # print("additional_endpoints: {}".format(self.additional_endpoints)) 102 | 103 | # Execute the call against the loaded adapter 104 | response = getattr(self, method)(**kwargs) 105 | 106 | # TODO - needs better checking 107 | # if not response: 108 | # raise Exception("empty response from API") 109 | 110 | # Return the response 111 | return response 112 | 113 | except Exception as e: 114 | # If we have remaining endpoints to try, attempt them 115 | if len(self.additional_endpoints) > 0: 116 | # Get the unavailable_endpoint this failed on 117 | unavailable_endpoint = self.endpoint 118 | 119 | # Get the next additional endpoint and set as the current 120 | self.endpoint = self.additional_endpoints.pop(0) 121 | 122 | # If we are continiously retrying the servers in the pool 123 | if self.retry: 124 | # Push the previously unavailable back to the end of the list 125 | self.additional_endpoints.append(unavailable_endpoint) 126 | 127 | # If the exception is that the websocket has closed, reinit the client 128 | if isinstance(e, WebSocketConnectionClosedException) and self.client: 129 | self.init_client() 130 | 131 | # Determine endpoint capabilities 132 | self.get_available_apis() 133 | 134 | # Logging 135 | # print("-------------") 136 | print("rpcerror: '{}' on {}, swapping to {}...".format(e, unavailable_endpoint, self.endpoint)) 137 | # print("called: {}".format(method)) 138 | # print("kawrgs: {}".format(kwargs)) 139 | # print("-------------") 140 | 141 | time.sleep(0.5) 142 | 143 | # Retry the call with the new endpoint 144 | return self.call(method, **kwargs) 145 | 146 | # If no endpoints are reachable, and retry enabled, try again 147 | elif self.retry: 148 | 149 | # print("-------------") 150 | print("rpcerror: '{}' (retrying in 3 seconds)".format(e)) 151 | # print("endpoint: {}".format(self.endpoint)) 152 | # print("endpoints: {}".format(self.endpoints)) 153 | # print("additional: {}".format(self.additional_endpoints)) 154 | # print("called: {}".format(method)) 155 | # print("kawrgs: {}".format(kwargs)) 156 | # print("-------------") 157 | 158 | time.sleep(3) 159 | 160 | # If the exception is that the websocket has closed, reinit the client 161 | if isinstance(e, WebSocketConnectionClosedException) and self.client: 162 | self.init_client() 163 | 164 | # Try again 165 | return self.call(method, **kwargs) 166 | 167 | # If no endpoints are reachable, and retry disabled, raise an exception 168 | else: 169 | # print("-------------") 170 | # print("called: {}".format(method)) 171 | # print("kawrgs: {}".format(kwargs)) 172 | # print("-------------") 173 | if raiseException: 174 | raise Exception("rpcerror: '{}' (not retrying)".format(e)) 175 | -------------------------------------------------------------------------------- /chainsync/adapters/decent/__init__.py: -------------------------------------------------------------------------------- 1 | from .decent import DecentAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/decent/decent.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from chainsync.adapters.abstract import AbstractAdapter 4 | from chainsync.adapters.base import BaseAdapter 5 | from chainsync.utils.http_client import HttpClient 6 | 7 | from jsonrpcclient.request import Request 8 | 9 | class DecentAdapter(AbstractAdapter, BaseAdapter): 10 | 11 | config = { 12 | 'BLOCK_INTERVAL': 'GRAPHENE_DEFAULT_BLOCK_INTERVAL' 13 | } 14 | 15 | def opData(self, block, opType, opData, txIndex=False): 16 | # Add some useful context to the operation 17 | opData['block_num'] = block['block_num'] 18 | opData['operation_type'] = opType 19 | opData['timestamp'] = block['timestamp'] 20 | # opData['transaction_id'] = block['transaction_ids'][txIndex] 21 | return opData 22 | 23 | def vOpData(self, vop): 24 | # Extract the operation from the vop object format 25 | opType, opData = vop 26 | # Add some useful context to the operation 27 | opData['block_num'] = opData['block'] 28 | opData['operation_type'] = opType 29 | opData['timestamp'] = opData['timestamp'] 30 | # opData['transaction_id'] = vop['trx_id'] 31 | return opData 32 | 33 | def get_block(self, block_num): 34 | response = HttpClient(self.endpoint).request('get_block', [block_num]) 35 | response['block_num'] = block_num 36 | return response 37 | 38 | def get_blocks(self, blocks): 39 | for i in blocks: 40 | yield self.call('get_block', block_num=int(i)) 41 | 42 | def get_ops_in_block(self, block_num, virtual_only=False): 43 | block = self.call('get_block', block_num=block_num) 44 | for tx in block['transactions']: 45 | for op in tx['operations']: 46 | op[1]['block'] = block_num 47 | op[1]['timestamp'] = block['timestamp'] 48 | yield op 49 | 50 | def get_ops_in_blocks(self, blocks, virtual_only=False): 51 | for i in blocks: 52 | yield self.call('get_ops_in_block', block_num=i, virtual_only=virtual_only) 53 | 54 | def get_config(self): 55 | return HttpClient(self.endpoint).request('get_config') 56 | 57 | def get_methods(self): 58 | return 'NOT_SUPPORTED' 59 | 60 | def get_status(self): 61 | return HttpClient(self.endpoint).request('get_dynamic_global_properties') 62 | -------------------------------------------------------------------------------- /chainsync/adapters/muse/__init__.py: -------------------------------------------------------------------------------- 1 | from .muse import MuseAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/muse/muse.py: -------------------------------------------------------------------------------- 1 | from chainsync.adapters.abstract import AbstractAdapter 2 | from chainsync.adapters.base import BaseAdapter 3 | from chainsync.utils.http_client import HttpClient 4 | 5 | from jsonrpcclient.request import Request 6 | 7 | class MuseAdapter(AbstractAdapter, BaseAdapter): 8 | 9 | config = { 10 | 'BLOCK_INTERVAL': 'MUSE_BLOCK_INTERVAL', 11 | 'VIRTUAL_OPS': [ 12 | 'fill_convert_request', 13 | 'author_reward', 14 | 'curation_reward', 15 | 'comment_reward', 16 | 'liquidity_reward', 17 | 'interest', 18 | 'fill_vesting_withdraw', 19 | 'fill_order', 20 | 'shutdown_witness', 21 | 'fill_transfer_from_savings', 22 | 'hardfork', 23 | 'comment_payout_update', 24 | 'return_vesting_delegation', 25 | 'comment_benefactor_reward', 26 | 'producer_reward', 27 | ] 28 | } 29 | 30 | def opData(self, block, opType, opData, txIndex=False): 31 | # Add some useful context to the operation 32 | opData['block_num'] = block['block_num'] 33 | opData['operation_type'] = opType 34 | opData['timestamp'] = block['timestamp'] 35 | opData['transaction_id'] = block['transaction_ids'][txIndex] 36 | return opData 37 | 38 | def vOpData(self, vop): 39 | # Extract the operation from the vop object format 40 | opType, opData = vop['op'] 41 | # Add some useful context to the operation 42 | opData['block_num'] = vop['block_num'] 43 | opData['operation_type'] = opType 44 | opData['timestamp'] = vop['timestamp'] 45 | opData['transaction_id'] = vop['trx_id'] 46 | return opData 47 | 48 | def get_block(self, block_num): 49 | response = HttpClient(self.endpoint).request('get_block', [block_num]) 50 | response['block_num'] = block_num 51 | return response 52 | 53 | def get_blocks(self, blocks): 54 | for i in blocks: 55 | yield self.call('get_block', block_num=int(i)) 56 | 57 | def get_ops_in_block(self, block_num, virtual_only=False): 58 | block = self.get_block(block_num) 59 | for idx, tx in enumerate(block['transactions']): 60 | for op in tx['operations']: 61 | yield { 62 | 'op': op, 63 | 'block_num': block_num, 64 | 'timestamp': block['timestamp'], 65 | 'trx_id': tx['signatures'][idx], 66 | } 67 | 68 | def get_ops_in_blocks(self, blocks, virtual_only=False): 69 | for i in blocks: 70 | yield self.call('get_ops_in_block', block_num=i, virtual_only=virtual_only) 71 | 72 | def get_transaction(self, transaction_id=1): 73 | response = HttpClient(self.endpoint).request('get_transaction', [transaction_id]) 74 | try: 75 | response['block_num'] = int(str(response['block_id'])[:8], base=16) 76 | except KeyError as e: 77 | pass 78 | return response 79 | 80 | def get_transactions(self, transaction_ids=[]): 81 | for transaction_id in transaction_ids: 82 | yield self.call('get_transaction', transaction_id=transaction_id) 83 | 84 | def get_ops_in_transaction(self, transaction_id=1): 85 | tx = self.call('get_transaction', transaction_id=transaction_id) 86 | for op in tx['operations']: 87 | op[1]['block_num'] = tx['block_num'] 88 | yield op 89 | 90 | def get_ops_in_transactions(self, transaction_ids=[]): 91 | for transaction_id in transaction_ids: 92 | for op in self.call('get_ops_in_transaction', transaction_id=transaction_id): 93 | yield op 94 | 95 | def get_config(self): 96 | return HttpClient(self.endpoint).request('get_config') 97 | 98 | def get_methods(self): 99 | return 'NOT_SUPPORTED' 100 | 101 | def get_status(self): 102 | return HttpClient(self.endpoint).request('get_dynamic_global_properties') 103 | -------------------------------------------------------------------------------- /chainsync/adapters/peerplays/__init__.py: -------------------------------------------------------------------------------- 1 | from .peerplays import PeerplaysAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/peerplays/peerplays.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from chainsync.adapters.abstract import AbstractAdapter 4 | from chainsync.adapters.base import BaseAdapter 5 | from chainsync.utils.http_client import HttpClient 6 | 7 | from jsonrpcclient.request import Request 8 | 9 | class PeerplaysAdapter(AbstractAdapter, BaseAdapter): 10 | 11 | config = { 12 | 'BLOCK_INTERVAL': 'GRAPHENE_DEFAULT_BLOCK_INTERVAL' 13 | } 14 | 15 | def opData(self, block, opType, opData, txIndex=False): 16 | # Add some useful context to the operation 17 | opData['block_num'] = block['block_num'] 18 | opData['operation_type'] = opType 19 | opData['timestamp'] = block['timestamp'] 20 | # opData['transaction_id'] = block['transaction_ids'][txIndex] 21 | return opData 22 | 23 | def vOpData(self, vop): 24 | # Extract the operation from the vop object format 25 | opType, opData = vop 26 | # Add some useful context to the operation 27 | opData['block_num'] = opData['block'] 28 | opData['operation_type'] = opType 29 | opData['timestamp'] = opData['timestamp'] 30 | # opData['transaction_id'] = vop['trx_id'] 31 | return opData 32 | 33 | def get_block(self, block_num): 34 | response = HttpClient(self.endpoint).request('get_block', [block_num]) 35 | response['block_num'] = block_num 36 | return response 37 | 38 | def get_blocks(self, blocks): 39 | for i in blocks: 40 | yield self.call('get_block', block_num=int(i)) 41 | 42 | def get_ops_in_block(self, block_num, virtual_only=False): 43 | block = self.call('get_block', block_num=block_num) 44 | for tx in block['transactions']: 45 | for op in tx['operations']: 46 | op[1]['block'] = block_num 47 | op[1]['timestamp'] = block['timestamp'] 48 | yield op 49 | 50 | def get_ops_in_blocks(self, blocks, virtual_only=False): 51 | for i in blocks: 52 | yield self.call('get_ops_in_block', block_num=i, virtual_only=virtual_only) 53 | 54 | def get_config(self): 55 | return HttpClient(self.endpoint).request('get_config') 56 | 57 | def get_methods(self): 58 | return 'NOT_SUPPORTED' 59 | 60 | def get_status(self): 61 | return HttpClient(self.endpoint).request('get_dynamic_global_properties') 62 | -------------------------------------------------------------------------------- /chainsync/adapters/steem/__init__.py: -------------------------------------------------------------------------------- 1 | from .steem import SteemAdapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/steem/steem.py: -------------------------------------------------------------------------------- 1 | from chainsync.adapters.abstract import AbstractAdapter 2 | from chainsync.adapters.base import BaseAdapter 3 | 4 | 5 | class SteemAdapter(AbstractAdapter, BaseAdapter): 6 | 7 | config = { 8 | 'BLOCK_INTERVAL': 'STEEMIT_BLOCK_INTERVAL', 9 | 'HEAD_BLOCK_NUMBER': 'head_block_number', 10 | 'LAST_IRREVERSIBLE_BLOCK_NUM': 'last_irreversible_block_num', 11 | 'VIRTUAL_OPS': [ 12 | 'fill_convert_request', 13 | 'author_reward', 14 | 'curation_reward', 15 | 'comment_reward', 16 | 'liquidity_reward', 17 | 'interest', 18 | 'fill_vesting_withdraw', 19 | 'fill_order', 20 | 'shutdown_witness', 21 | 'fill_transfer_from_savings', 22 | 'hardfork', 23 | 'comment_payout_update', 24 | 'return_vesting_delegation', 25 | 'comment_benefactor_reward', 26 | 'producer_reward', 27 | ] 28 | } 29 | 30 | def format_op_from_get_block(self, block, op, txIndex=False, opIndex=False): 31 | opType, opData = op 32 | opData['block_num'] = block['block_num'] 33 | opData['op_in_trx'] = opIndex 34 | opData['operation_type'] = opType 35 | opData['timestamp'] = block['timestamp'] 36 | opData['transaction_id'] = block['transaction_ids'][txIndex] 37 | opData['trx_in_block'] = txIndex 38 | return opData 39 | 40 | def format_op_from_get_ops_in_block(self, op): 41 | opType, opData = op['op'] 42 | opData['block_num'] = op['block'] 43 | opData['op_in_trx'] = op['op_in_trx'] 44 | opData['operation_type'] = opType 45 | opData['timestamp'] = op['timestamp'] 46 | opData['transaction_id'] = op['trx_id'] 47 | opData['trx_in_block'] = op['trx_in_block'] 48 | return opData 49 | 50 | def format_op_from_get_transaction(self, tx, op, txIndex=False, opIndex=False): 51 | opType, opData = op 52 | opData['block_num'] = tx['block_num'] 53 | opData['op_in_trx'] = opIndex 54 | opData['operation_type'] = opType 55 | opData['timestamp'] = False 56 | opData['transaction_id'] = tx['transaction_id'] 57 | opData['trx_in_block'] = txIndex 58 | return opData 59 | 60 | def get_block(self, block_num): 61 | response = self.client.request('get_block', [block_num]) 62 | try: 63 | response['block_num'] = int(str(response['block_id'])[:8], base=16) 64 | except KeyError as e: 65 | print(e) 66 | print(response) 67 | return response 68 | 69 | def get_blocks(self, blocks): 70 | for i in blocks: 71 | yield self.call('get_block', block_num=int(i)) 72 | 73 | def get_config(self): 74 | return self.client.request('get_config') 75 | 76 | def get_methods(self): 77 | return 'NOT_SUPPORTED' 78 | 79 | def get_ops_in_block(self, block_num, virtual_only=False): 80 | return self.client.request('get_ops_in_block', [block_num, virtual_only]) 81 | 82 | def get_ops_in_blocks(self, blocks, virtual_only=False): 83 | for i in blocks: 84 | yield self.call('get_ops_in_block', block_num=i, virtual_only=virtual_only) 85 | 86 | def get_status(self): 87 | return self.client.request('get_dynamic_global_properties') 88 | 89 | def get_transaction(self, transaction_id=1): 90 | response = self.client.request('get_transaction', [transaction_id]) 91 | try: 92 | response['block_num'] = int(str(response['block_id'])[:8], base=16) 93 | except KeyError as e: 94 | pass 95 | return response 96 | 97 | def get_transactions(self, transaction_ids=[]): 98 | for transaction_id in transaction_ids: 99 | yield self.call('get_transaction', transaction_id=transaction_id) 100 | -------------------------------------------------------------------------------- /chainsync/adapters/steemv2/__init__.py: -------------------------------------------------------------------------------- 1 | from .steem import SteemV2Adapter 2 | -------------------------------------------------------------------------------- /chainsync/adapters/steemv2/steem.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from chainsync.adapters.abstract import AbstractAdapter 4 | from chainsync.adapters.base import BaseAdapter 5 | 6 | from jsonrpcclient.request import Request 7 | 8 | class SteemV2Adapter(AbstractAdapter, BaseAdapter): 9 | 10 | config = { 11 | 'BLOCK_INTERVAL': 'STEEM_BLOCK_INTERVAL', 12 | 'HEAD_BLOCK_NUMBER': 'head_block_number', 13 | 'LAST_IRREVERSIBLE_BLOCK_NUM': 'last_irreversible_block_num', 14 | 'VIRTUAL_OPS': [ 15 | 'fill_convert_request', 16 | 'author_reward', 17 | 'curation_reward', 18 | 'comment_reward', 19 | 'liquidity_reward', 20 | 'interest', 21 | 'fill_vesting_withdraw', 22 | 'fill_order', 23 | 'shutdown_witness', 24 | 'fill_transfer_from_savings', 25 | 'hardfork', 26 | 'comment_payout_update', 27 | 'return_vesting_delegation', 28 | 'comment_benefactor_reward', 29 | 'producer_reward', 30 | ] 31 | } 32 | 33 | def format_op_from_get_block(self, block, op, txIndex=False, opIndex=False): 34 | opType, opData = op 35 | opData['block_num'] = block['block_num'] 36 | opData['op_in_trx'] = opIndex 37 | opData['operation_type'] = opType 38 | opData['timestamp'] = block['timestamp'] 39 | opData['transaction_id'] = block['transaction_ids'][txIndex] 40 | opData['trx_in_block'] = txIndex 41 | return opData 42 | 43 | def format_op_from_get_ops_in_block(self, op): 44 | opType, opData = op['op'] 45 | opData['block_num'] = op['block'] 46 | opData['op_in_trx'] = op['op_in_trx'] 47 | opData['operation_type'] = opType 48 | opData['timestamp'] = op['timestamp'] 49 | opData['transaction_id'] = op['trx_id'] 50 | opData['trx_in_block'] = op['trx_in_block'] 51 | return opData 52 | 53 | def format_op_from_get_transaction(self, tx, op, txIndex=False, opIndex=False): 54 | opType, opData = op 55 | opData['block_num'] = tx['block_num'] 56 | opData['op_in_trx'] = opIndex 57 | opData['operation_type'] = opType 58 | opData['timestamp'] = False 59 | opData['transaction_id'] = tx['transaction_id'] 60 | opData['trx_in_block'] = txIndex 61 | return opData 62 | 63 | def get_block(self, block_num): 64 | # block_api method 65 | api = 'block_api' 66 | method = 'get_block' 67 | # ensure the API is available 68 | if self.is_api_available(api, method): 69 | response = self.client.request('.'.join([api, method]), block_num=block_num) 70 | response['block']['block_num'] = int(str(response['block']['block_id'])[:8], base=16) 71 | return response['block'] 72 | 73 | def get_blocks(self, blocks=[]): 74 | # block_api method 75 | api = 'block_api' 76 | method = 'get_block' 77 | if self.is_api_available(api, method): 78 | # assemble list with multiple requests for batch 79 | requests = [ 80 | Request('.'.join([api, method]), { 81 | 'block_num': i 82 | }) for i in blocks 83 | ] 84 | # get response 85 | response = self.client.send(requests) 86 | # return the resulting block of each result 87 | return [dict(r['result']['block'], **{'block_num': int(str(r['result']['block']['block_id'])[:8], base=16)}) for r in response] 88 | 89 | def get_config(self): 90 | # database_api method 91 | api = 'database_api' 92 | method = 'get_config' 93 | if self.is_api_available(api, method): 94 | return self.client.request('.'.join([api, method])) 95 | 96 | def get_methods(self): 97 | # jsonrpc method 98 | api = 'jsonrpc' 99 | method = 'get_methods' 100 | return self.client.request('.'.join([api, method])) 101 | 102 | def get_ops_in_block(self, block_num, virtual_only=False): 103 | if self.is_api_available('account_history_api', 'get_ops_in_block', raiseException=False): 104 | return self.get_ops_in_block_from_account_history_api(block_num=block_num, virtual_only=virtual_only) 105 | elif self.is_api_available('condenser_api', 'get_ops_in_block', raiseException=False): 106 | return self.get_ops_in_block_from_condenser_api(block_num=block_num, virtual_only=virtual_only) 107 | else: 108 | raise Exception('endpoint not capable of calling get_ops_in_block from either condenser_api or account_history_api') 109 | 110 | def get_ops_in_block_from_account_history_api(self, block_num, virtual_only=False): 111 | # account_history_api method 112 | api = 'account_history_api' 113 | method = 'get_ops_in_block' 114 | response = self.client.request('.'.join([api, method]), { 115 | 'block_num': block_num, 116 | 'only_virtual': virtual_only 117 | }) 118 | return response['ops'] 119 | 120 | def get_ops_in_block_from_condenser_api(self, block_num, virtual_only=False): 121 | # condenser_api method 122 | api = 'condenser_api' 123 | method = 'get_ops_in_block' 124 | response = Client(self.endpoint).request('.'.join([api, method]), [ 125 | block_num, 126 | virtual_only 127 | ]) 128 | return response 129 | 130 | def get_ops_in_blocks(self, blocks, virtual_only=False): 131 | if self.is_api_available('account_history_api', 'get_ops_in_block', raiseException=False): 132 | return self.get_ops_in_blocks_from_account_history_api(blocks=blocks, virtual_only=virtual_only) 133 | elif self.is_api_available('condenser_api', 'get_ops_in_block', raiseException=False): 134 | return self.get_ops_in_blocks_from_condenser_api(blocks=blocks, virtual_only=virtual_only) 135 | else: 136 | raise Exception('endpoint not capable of calling get_ops_in_block from either condenser_api or account_history_api') 137 | 138 | def get_ops_in_blocks_from_account_history_api(self, blocks, virtual_only=False): 139 | # account_history_api method 140 | api = 'account_history_api' 141 | method = 'get_ops_in_block' 142 | # assemble list with multiple requests for batch 143 | requests = [ 144 | Request('.'.join([api, method]), { 145 | 'block_num': i, 146 | 'only_virtual': virtual_only 147 | }) for i in blocks 148 | ] 149 | # get response 150 | response = self.client.send(requests) 151 | # return the resulting ops 152 | return [r['result']['ops'] for r in response] 153 | 154 | def get_ops_in_blocks_from_condenser_api(self, blocks, virtual_only=False): 155 | # condenser_api method 156 | api = 'condenser_api' 157 | method = 'get_ops_in_block' 158 | # assemble list with multiple requests for batch 159 | requests = [ 160 | Request('.'.join([api, method]), [ 161 | i, 162 | virtual_only 163 | ]) for i in blocks 164 | ] 165 | # get response 166 | response = self.client.send(requests) 167 | # return the resulting ops 168 | return [r['result'] for r in response] 169 | 170 | def get_status(self): 171 | # database_api method 172 | api = 'database_api' 173 | method = 'get_dynamic_global_properties' 174 | if self.is_api_available(api, method): 175 | return self.client.request('.'.join([api, method])) 176 | 177 | def get_transaction(self, transaction_id=1): 178 | response = self.client.request('condenser_api.get_transaction', [transaction_id]) 179 | # try: 180 | # response['block_num'] = int(str(response['block_id'])[:8], base=16) 181 | # except KeyError as e: 182 | # print(e) 183 | # print(response) 184 | print(response) 185 | # return self.call('get_transaction', transaction_id=transaction_id) 186 | 187 | def get_transactions(self, transaction_ids=[]): 188 | pass 189 | -------------------------------------------------------------------------------- /chainsync/chainsync.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import time 3 | 4 | from collections import Counter 5 | from datetime import datetime 6 | 7 | 8 | class ChainSync(): 9 | 10 | def __init__(self, adapter, endpoints=['http://localhost:8090'], plugins=[], retry=True, debug=False): 11 | self.debug = debug 12 | self.plugins = plugins 13 | if adapter: 14 | if debug: 15 | adapter.debug = debug 16 | self.adapter = adapter 17 | else: 18 | raise Exception('adapter required: you must specify a adapter') 19 | 20 | def get_config(self): 21 | return self.adapter.call('get_config') 22 | 23 | def get_status(self): 24 | return self.adapter.call('get_status') 25 | 26 | def get_head_block(self, mode='head', return_status=False): 27 | # get status from the blockchain 28 | status = self.get_status() 29 | 30 | # determine the current head block 31 | head_block = status[self.adapter.config['HEAD_BLOCK_NUMBER']] 32 | 33 | # If set to irreversible, override the head block 34 | if mode == 'irreversible': 35 | head_block = status[self.adapter.config['LAST_IRREVERSIBLE_BLOCK_NUM']] 36 | 37 | # A flag to return status alongside the block number 38 | if return_status: 39 | return (head_block, status) 40 | 41 | return head_block 42 | 43 | def get_block(self, block_num): 44 | return self.adapter.call('get_block', block_num=block_num) 45 | 46 | def get_blocks(self, blocks): 47 | if not isinstance(blocks, list): 48 | raise TypeError 49 | return self.adapter.call('get_blocks', blocks=blocks) 50 | 51 | def get_block_sequence(self, start_block, limit): 52 | blocks = list(range(int(start_block), int(start_block) + int(limit))) 53 | yield from self.get_blocks(blocks) 54 | 55 | def get_ops_in_block(self, block_num, virtual_only=False, whitelist=[]): 56 | for vop in self.adapter.call('get_ops_in_block', block_num=block_num, virtual_only=virtual_only): 57 | if not whitelist or vop['op'][0] in whitelist: 58 | yield self.adapter.format_op_from_get_ops_in_block(vop) 59 | 60 | def get_ops_in_blocks(self, blocks, virtual_only=False, whitelist=[]): 61 | if not isinstance(blocks, list): 62 | raise TypeError 63 | for data in self.adapter.call('get_ops_in_blocks', blocks=blocks, virtual_only=virtual_only): 64 | for op in data: 65 | if not whitelist or op['op'][0] in whitelist: 66 | yield self.adapter.format_op_from_get_ops_in_block(op) 67 | 68 | def get_ops_in_block_sequence(self, start_block, limit, virtual_only=False, whitelist=[]): 69 | blocks = list(range(int(start_block), int(start_block) + int(limit))) 70 | yield from self.get_ops_in_blocks(blocks, virtual_only=virtual_only, whitelist=whitelist) 71 | 72 | def get_ops_in_transaction(self, transaction_id): 73 | if not isinstance(transaction_id, str): 74 | raise TypeError 75 | transaction = self.adapter.call('get_transaction', transaction_id=transaction_id) 76 | for op in transaction['operations']: 77 | yield self.adapter.format_op_from_get_transaction(transaction, op) 78 | 79 | def get_ops_in_transactions(self, transaction_ids): 80 | if not isinstance(transaction_ids, list): 81 | raise TypeError 82 | for transaction in self.adapter.call('get_transactions', transaction_ids=transaction_ids): 83 | for op in transaction['operations']: 84 | yield self.adapter.format_op_from_get_transaction(transaction, op) 85 | 86 | def get_transaction(self, transaction_id): 87 | if not isinstance(transaction_id, str): 88 | raise TypeError 89 | return self.adapter.call('get_transaction', transaction_id=transaction_id) 90 | 91 | def get_transactions(self, transaction_ids): 92 | if not isinstance(transaction_ids, list): 93 | raise TypeError 94 | yield from self.adapter.call('get_transactions', transaction_ids=transaction_ids) 95 | 96 | def from_block_get_ops(self, block, virtual_ops=False, regular_ops=True, whitelist=[]): 97 | # ensure a block is passed (could use better checks) 98 | if not isinstance(block, dict): 99 | raise TypeError 100 | # check the whitelist for virtual operations 101 | if whitelist and 'VIRTUAL_OPS' in self.adapter.config: 102 | # get a set of all the different virtual op types 103 | virtual_ops = set(self.adapter.config['VIRTUAL_OPS']) 104 | # check for any whitelist operations that are virtual 105 | matches = [vop for vop in whitelist if vop in virtual_ops] 106 | # if any matches, force enable virtual ops 107 | if matches: 108 | virtual_ops = True 109 | # ensure regular_ops should be yielded 110 | if regular_ops: 111 | yield from self.from_block_get_ops_regular(block, whitelist=whitelist) 112 | # ensure virtual_ops should be yielded 113 | if virtual_ops: 114 | # query and yield all virtual operations within this block 115 | yield from self.get_ops_in_block(block['block_num'], True, whitelist=whitelist) 116 | 117 | def from_block_get_ops_regular(self, block, whitelist=[]): 118 | # ensure a block is passed (could use better checks) 119 | if not isinstance(block, dict): 120 | raise TypeError 121 | # Loop through all transactions within this block 122 | for txIndex, tx in enumerate(block['transactions']): 123 | # If a whitelist is defined, only allow whitelisted operations through 124 | ops = (op for op in tx['operations'] if not whitelist or op[0] in whitelist) 125 | # Iterate and yield each op 126 | for opIndex, op in enumerate(ops): 127 | yield self.adapter.format_op_from_get_block(block, op, txIndex=txIndex, opIndex=opIndex) 128 | 129 | def stream(self, what=['blocks', 'config', 'status', 'ops', 'ops_per_blocks'], start_block=None, mode='head', batch_size=10, virtual_ops=True, regular_ops=True, throttle=1, whitelist=[]): 130 | 131 | config = self.get_config() 132 | if 'config' in what: 133 | yield self.yield_event('config', config) 134 | 135 | yield from self.stream_from_rpc( 136 | what=what, 137 | config=config, 138 | start_block=start_block, 139 | mode=mode, 140 | batch_size=batch_size, 141 | virtual_ops=virtual_ops, 142 | regular_ops=regular_ops, 143 | throttle=throttle, 144 | whitelist=whitelist 145 | ) 146 | 147 | def stream_from_rpc(self, what=['blocks', 'config', 'status', 'ops', 'ops_per_blocks'], config=False, start_block=None, mode='head', batch_size=10, virtual_ops=True, regular_ops=True, throttle=1, whitelist=[]): 148 | 149 | if not config: 150 | config = self.get_config() 151 | if 'config' in what: 152 | yield ('config', config) 153 | 154 | while True: 155 | 156 | # Determine what height to stream up to 157 | end_block, status = self.get_head_block(mode, return_status=True) 158 | 159 | # Yield event if the 'status' parameter is set 160 | if 'status' in what: 161 | yield self.yield_event('status', status) 162 | 163 | # If no start block is specified, start streaming from head 164 | if start_block is None: 165 | start_block = end_block 166 | 167 | # If we need to retrieve the stream, do so. 168 | if any(dataType in what for dataType in ['blocks', 'ops', 'ops_per_blocks']): 169 | yield from self.get_stream( 170 | config=config, 171 | what=what, 172 | start_block=start_block, 173 | end_block=end_block, 174 | mode=mode, 175 | batch_size=batch_size, 176 | virtual_ops=virtual_ops, 177 | regular_ops=regular_ops, 178 | whitelist=whitelist 179 | ) 180 | 181 | # If remaining > batch_size, increment by batch size 182 | if (end_block - start_block) > batch_size: 183 | # set the next block to start on 184 | start_block = start_block + batch_size 185 | else: 186 | # else start on the next block 187 | start_block = end_block + 1 188 | 189 | # Pause the loop for based on an estimated time the next block will arrive 190 | time.sleep(self.get_approx_sleep_until_block(throttle, config, status['time'])) 191 | 192 | def get_approx_sleep_until_block(self, throttle, config, ts): 193 | # Get the rate the blockchain generates blocks 194 | block_interval = config[self.adapter.config['BLOCK_INTERVAL']] if 'BLOCK_INTERVAL' in self.adapter.config else 3 195 | 196 | # Interpret the datetime string passed from the last status 197 | blockchain_time = datetime(int(ts[:4]), int(ts[5:7]), int(ts[8:10]), int(ts[11:13]), int(ts[14:16]), int(ts[17:19])) 198 | 199 | # Determine how long it's been since that last block 200 | current_time = datetime.utcnow() 201 | since_last_block = current_time.timestamp() - blockchain_time.timestamp() 202 | 203 | # Add a delay based on when the next block is expected 204 | if since_last_block < block_interval: 205 | return block_interval - since_last_block 206 | else: 207 | # Otherwise add the throttle delay (defaults to 1 second) delay to avoid thrashing the API 208 | return throttle 209 | 210 | def yield_event(self, what, data): 211 | formatted = copy.deepcopy(data) 212 | if self.plugins: 213 | for plugin in self.plugins: 214 | formatted = getattr(plugin, what)(formatted) 215 | return (what, formatted) 216 | 217 | def get_stream(self, what=['blocks', 'config', 'status', 'ops', 'ops_per_blocks'], config=None, start_block=None, end_block=None, mode='head', batch_size=10, virtual_ops=True, regular_ops=True, whitelist=[]): 218 | 219 | if not config: 220 | config = self.get_config() 221 | if 'config' in what: 222 | yield self.yield_event('config', config) 223 | 224 | # If no end_block is specified, assume a single iteration starting at start_block and ending at start_block + batch_size 225 | if not end_block: 226 | end_block = start_block + batch_size - 1 227 | 228 | last_block_processed = start_block 229 | 230 | # While remaining blocks exist - batch load them 231 | while start_block <= end_block: 232 | 233 | # Determine how many blocks to load with this request 234 | limit = end_block - start_block + 1 if (end_block - start_block + 1) < batch_size else batch_size 235 | 236 | # Assume the last block processed 237 | last_block_processed = start_block + limit - 1 238 | 239 | # Track how many operations are in each block 240 | ops_per_blocks = [] 241 | 242 | if 'blocks' in what: 243 | for block in self.get_block_sequence(start_block, limit=limit): 244 | # Yield the block 245 | yield self.yield_event('block', block) 246 | 247 | # Set this as the last block processed 248 | last_block_processed = block['block_num'] 249 | 250 | # If 'ops' should be streamed alongside 'blocks', use the existing data and retrieve any missing vops via `self.from_block_get_ops` 251 | if 'ops' in what: 252 | for op in self.from_block_get_ops(block, virtual_ops=virtual_ops, regular_ops=regular_ops, whitelist=whitelist): 253 | # Add the data to count how many ops are in this block 254 | if 'ops_per_blocks' in what: 255 | ops_per_blocks.append(op['block_num']) 256 | # Yield the operation 257 | yield self.yield_event('op', op) 258 | 259 | # If not streaming blocks, but streaming either 'ops' or 'ops_per_blocks' 260 | elif 'ops' or 'ops_per_blocks' in what: 261 | 262 | # If virtual ops are needed (or both virtual + regular ops) 263 | if virtual_ops or (regular_ops and virtual_ops): 264 | # Retrieve ops using `get_ops_in_block` 265 | for op in self.get_ops_in_block_sequence(start_block, limit=limit, virtual_only=not regular_ops, whitelist=whitelist): 266 | # Add the data to count how many ops are in this block 267 | if 'ops_per_blocks' in what: 268 | ops_per_blocks.append(op['block_num']) 269 | # Yield the operation if 'ops' is requested 270 | if 'ops' in what: 271 | yield self.yield_event('op', op) 272 | 273 | # If only regular ops are needed 274 | elif regular_ops: 275 | for block in self.get_block_sequence(start_block, limit=limit): 276 | # Set this as the last block processed 277 | last_block_processed = block['block_num'] 278 | for op in self.from_block_get_ops(block, virtual_ops=virtual_ops, regular_ops=regular_ops, whitelist=whitelist): 279 | # Add the data to count how many ops are in this block 280 | if 'ops_per_blocks' in what: 281 | ops_per_blocks.append(op['block_num']) 282 | 283 | # Yield the operation if 'ops' is requested 284 | if 'ops' in what: 285 | yield self.yield_event('op', op) 286 | 287 | # If 'ops_per_blocks' are requested, yield the Counter generated 288 | if 'ops_per_blocks' in what: 289 | yield self.yield_event('ops_per_block', Counter(ops_per_blocks)) 290 | 291 | # Determine the next block to start on 292 | start_block = last_block_processed + 1 293 | -------------------------------------------------------------------------------- /chainsync/clients/http/rpc.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import sys 3 | import datetime 4 | import time 5 | 6 | from jsonrpcclient.http_client import HTTPClient 7 | from jsonrpcclient.request import Request 8 | 9 | from jsonrpcclient import config 10 | config.validate = False 11 | 12 | 13 | class Client(HTTPClient): 14 | 15 | def send(self, request, **kwargs): 16 | # start_time = time.time() 17 | response = super(Client, self).send(request, **kwargs) 18 | # total_time = "%.3f" % (time.time() - start_time) 19 | # print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, request, list(kwargs))) 20 | # print(response) 21 | if 'error' in response: 22 | print("response error") 23 | print(response) 24 | raise requests.RequestException() 25 | return response 26 | 27 | def request(self, method_name, *args, **kwargs): 28 | # start_time = time.time() 29 | response = super(Client, self).send(Request(method_name, *args, **kwargs)) 30 | # total_time = "%.3f" % (time.time() - start_time) 31 | # print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, method_name, list(args))) 32 | # print(response) 33 | if 'error' in response: 34 | print("response error") 35 | print(response) 36 | raise requests.RequestException() 37 | return response 38 | -------------------------------------------------------------------------------- /chainsync/clients/ws/rpc.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import sys 3 | import datetime 4 | import time 5 | import json 6 | 7 | from jsonrpcclient import config 8 | from jsonrpcclient.http_client import HTTPClient 9 | from jsonrpcclient.request import Request 10 | 11 | from websocket import create_connection 12 | 13 | config.validate = False 14 | 15 | 16 | class Client(HTTPClient): 17 | 18 | def __init__(self, endpoint): 19 | self.endpoint = endpoint 20 | self.connect() 21 | 22 | def connect(self): 23 | print("connecting: {}".format(self.endpoint)) 24 | self.websocket = create_connection(self.endpoint) 25 | 26 | def send(self, request, **kwargs): 27 | # start_time = time.time() 28 | self.websocket.send(str(request)) 29 | response = self.websocket.recv() 30 | # total_time = "%.3f" % (time.time() - start_time) 31 | # print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, request, list(kwargs))) 32 | if 'error' in response: 33 | print("response error") 34 | print(response) 35 | raise requests.RequestException() 36 | return json.loads(response)['result'] 37 | 38 | def request(self, method_name, *args, **kwargs): 39 | # start_time = time.time() 40 | self.websocket.send(str(Request(method_name, *args, **kwargs))) 41 | response = self.websocket.recv() 42 | # total_time = "%.3f" % (time.time() - start_time) 43 | # print("[{}] http request - {} kb / {} sec - {} {}".format(datetime.datetime.now(), sys.getsizeof(str(response)) / 1000, total_time, method_name, list(args))) 44 | if 'error' in response: 45 | print("response error") 46 | print(response) 47 | raise requests.RequestException() 48 | return json.loads(response)['result'] 49 | -------------------------------------------------------------------------------- /examples/adapter_decent.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from chainsync import ChainSync 3 | from chainsync.adapters.decent import DecentAdapter 4 | 5 | adapter = DecentAdapter(endpoints=['http://api.decent-db.com:8090/']) 6 | chainsync = ChainSync(adapter) 7 | 8 | print('\nGetting block 1') 9 | block = chainsync.get_block(1) 10 | print(block) 11 | 12 | print('\nGetting blocks 1, 10, 50, 250, 500') 13 | blocks = chainsync.get_blocks([1, 10, 50, 250, 500]) 14 | for block in blocks: 15 | print(block) 16 | 17 | print('\nGetting blocks 1000-1005') 18 | blocks = chainsync.get_block_sequence(1000, 5) 19 | for block in blocks: 20 | print(block) 21 | 22 | print('\nGetting all ops in block 92847...') 23 | for op in chainsync.get_ops_in_block(92847): 24 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 25 | 26 | print('\nGetting withdraw_vesting ops in block 92847...') 27 | for op in chainsync.get_ops_in_block(92847, whitelist=['withdraw_vesting']): 28 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 29 | 30 | print('\nGetting all ops in block 10000, 50000, and 20000...') 31 | for op in chainsync.get_ops_in_blocks([10000, 50000, 20000]): 32 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 33 | 34 | print('\nGetting producer_reward ops in block 10000, 50000, and 20000...') 35 | for op in chainsync.get_ops_in_blocks([10000, 50000, 20000], whitelist=['producer_reward']): 36 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 37 | 38 | print('\nStreaming blocks, 100 at a time, from the irreversible height...') 39 | for dataType, block in chainsync.stream(['blocks'], batch_size=100, mode='irreversible'): 40 | print("{}: {} - {}".format(datetime.datetime.now(), block['block_num'], block['miner'])) 41 | 42 | print('\nStreaming all ops...') 43 | for dataType, op in chainsync.stream(['ops']): 44 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 45 | 46 | print('\nStreaming all blocks + ops + virtual ops + accurate counts of ops per block...') 47 | for dataType, data in chainsync.stream(['blocks', 'ops', 'ops_per_blocks'], start_block=1045177): 48 | dataHeader = "{} #{}: {}".format(datetime.datetime.now(), data['block_num'], dataType) 49 | if dataType == "op": 50 | print("{} {}".format(dataHeader, data['operation_type'])) 51 | if dataType == "block": 52 | txCount = len(data['transactions']) 53 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 54 | print("{} - #{} - tx: {} / ops: {}".format(dataHeader, data['block_num'], txCount, opCount)) 55 | if dataType == "ops_per_blocks": 56 | for height in data: 57 | print("{} - #{}: {}".format(dataHeader, height, data[height])) 58 | -------------------------------------------------------------------------------- /examples/adapter_muse.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.muse import MuseAdapter 5 | 6 | adapter = MuseAdapter(endpoints=['http://45.79.206.79:8090/'], debug=False) 7 | chainsync = ChainSync(adapter) 8 | 9 | print('\nGetting block 1') 10 | block = chainsync.get_block(1) 11 | print(block) 12 | 13 | print('\nGetting blocks 1, 10, 50, 250, 500') 14 | blocks = chainsync.get_blocks([1, 10, 50, 250, 500]) 15 | for block in blocks: 16 | print(block) 17 | 18 | print('\nGetting blocks 1000-1005') 19 | blocks = chainsync.get_block_sequence(1000, 5) 20 | for block in blocks: 21 | print(block) 22 | 23 | print('\nGetting all ops in block 6274087...') 24 | for op in chainsync.get_ops_in_block(6274087): 25 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 26 | 27 | print('\nGetting withdraw_vesting ops in block 6274087...') 28 | for op in chainsync.get_ops_in_block(6274087, whitelist=['feed_publish']): 29 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 30 | 31 | print('\nGetting all ops in block 1000000, 5000000, and 2000000...') 32 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000]): 33 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 34 | 35 | print('\nGetting producer_reward ops in block 1000000, 5000000, and 2000000...') 36 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000], whitelist=['producer_reward']): 37 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 38 | 39 | print('\nStreaming blocks, 100 at a time, from the irreversible height...') 40 | for dataType, block in chainsync.stream(['blocks'], batch_size=100, mode='irreversible'): 41 | print("{}: {} - {}".format(datetime.datetime.now(), block['block_num'], block['witness'])) 42 | 43 | print('\nStreaming all ops...') 44 | for dataType, op in chainsync.stream(['ops']): 45 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 46 | 47 | print('\nStreaming all non-virtual ops...') 48 | for dataType, op in chainsync.stream(['ops'], virtual_ops=False): 49 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 50 | 51 | print('\nStreaming all virtual ops...') 52 | for dataType, op in chainsync.stream(['ops'], regular_ops=False): 53 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 54 | 55 | print('\nStreaming vote ops only...') 56 | for dataType, op in chainsync.stream(['ops'], whitelist=['vote']): 57 | print("{}: {} - {} by {}".format(datetime.datetime.now(), op['block_num'], op['operation_type'], op['voter'])) 58 | 59 | print('\nStreaming producer_reward virtual ops only...') 60 | for dataType, op in chainsync.stream(['ops'], whitelist=['producer_reward']): 61 | print("{}: {} - {} for {} of {}".format(datetime.datetime.now(), op['block_num'], op['operation_type'], op['producer'], op['vesting_shares'])) 62 | 63 | print('\nStreaming all blocks + ops + virtual ops + accurate counts of ops per block...') 64 | for dataType, data in chainsync.stream(['blocks', 'ops', 'ops_per_blocks']): 65 | dataHeader = "{} #{}: {}".format(datetime.datetime.now(), data['block_num'], dataType) 66 | if dataType == "op": 67 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 68 | if dataType == "block": 69 | txCount = len(data['transactions']) 70 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 71 | print("{} - #{} - tx: {} / ops: {}".format(dataHeader, data['block_num'], txCount, opCount)) 72 | if dataType == "ops_per_blocks": 73 | for height in data: 74 | print("{} - #{}: {}".format(dataHeader, height, data[height])) 75 | 76 | print('\nStreaming all blocks + ops (no virtual ops)...') 77 | for dataType, data in chainsync.stream(['blocks', 'ops'], virtual_ops=False): 78 | dataHeader = "{}: {}".format(datetime.datetime.now(), dataType) 79 | if dataType == "op": 80 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 81 | if dataType == "block": 82 | txCount = len(data['transactions']) 83 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 84 | print("{} - tx: {} / ops: {}".format(dataHeader, txCount, opCount)) 85 | 86 | print('\nStreaming all blocks + ops (no virtual ops), filtering only votes...') 87 | for dataType, data in chainsync.stream(['blocks', 'ops'], whitelist=['vote'], virtual_ops=False): 88 | dataHeader = "{}: {}".format(datetime.datetime.now(), dataType) 89 | if dataType == "op": 90 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 91 | if dataType == "block": 92 | txCount = len(data['transactions']) 93 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 94 | print("{} - tx: {} / ops: {}".format(dataHeader, txCount, opCount)) 95 | 96 | print('\nStreaming all blocks + virtual ops (no normal ops)...') 97 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False): 98 | dataHeader = "{}: {}".format(datetime.datetime.now(), dataType) 99 | if dataType == "op": 100 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 101 | if dataType == "block": 102 | txCount = len(data['transactions']) 103 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 104 | print("{} - tx: {} / ops: {}".format(dataHeader, txCount, opCount)) 105 | 106 | print('\nStreaming all blocks + virtual ops (no normal ops), filtering only producer_reward...') 107 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False, whitelist=['producer_reward']): 108 | dataHeader = "{}: {}".format(datetime.datetime.now(), dataType) 109 | if dataType == "op": 110 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 111 | if dataType == "block": 112 | txCount = len(data['transactions']) 113 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 114 | print("{} - tx: {} / ops: {}".format(dataHeader, txCount, opCount)) 115 | -------------------------------------------------------------------------------- /examples/adapter_peerplays.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from chainsync import ChainSync 3 | from chainsync.adapters.peerplays import PeerplaysAdapter 4 | 5 | adapter = PeerplaysAdapter(endpoints=['http://45.79.220.16:8090/']) 6 | chainsync = ChainSync(adapter) 7 | 8 | print('\nGetting block 1') 9 | block = chainsync.get_block(1) 10 | print(block) 11 | 12 | print('\nGetting blocks 1, 10, 50, 250, 500') 13 | blocks = chainsync.get_blocks([1, 10, 50, 250, 500]) 14 | for block in blocks: 15 | print(block) 16 | 17 | print('\nGetting blocks 1000-1005') 18 | blocks = chainsync.get_block_sequence(1000, 5) 19 | for block in blocks: 20 | print(block) 21 | 22 | print('\nGetting all ops in block 92847...') 23 | for op in chainsync.get_ops_in_block(92847): 24 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 25 | 26 | print('\nGetting withdraw_vesting ops in block 92847...') 27 | for op in chainsync.get_ops_in_block(92847, whitelist=['withdraw_vesting']): 28 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 29 | 30 | print('\nGetting all ops in block 10000, 50000, and 20000...') 31 | for op in chainsync.get_ops_in_blocks([10000, 50000, 20000]): 32 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 33 | 34 | print('\nGetting producer_reward ops in block 10000, 50000, and 20000...') 35 | for op in chainsync.get_ops_in_blocks([10000, 50000, 20000], whitelist=['producer_reward']): 36 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 37 | 38 | print('\nStreaming blocks, 100 at a time, from the irreversible height...') 39 | for dataType, block in chainsync.stream(['blocks'], batch_size=100, mode='irreversible'): 40 | print("{}: {} - {}".format(datetime.datetime.now(), block['block_num'], block['witness'])) 41 | 42 | print('\nStreaming all ops...') 43 | for dataType, op in chainsync.stream(['ops']): 44 | print("{}: {} [{}] - {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'])) 45 | 46 | print('\nStreaming all blocks + ops + virtual ops + accurate counts of ops per block...') 47 | for dataType, data in chainsync.stream(['blocks', 'ops', 'ops_per_blocks'], start_block=1045177): 48 | dataHeader = "{} #{}: {}".format(datetime.datetime.now(), data['block_num'], dataType) 49 | if dataType == "op": 50 | print("{} {}".format(dataHeader, data['operation_type'])) 51 | if dataType == "block": 52 | txCount = len(data['transactions']) 53 | opCount = sum([len(tx['operations']) for tx in data['transactions']]) 54 | print("{} - #{} - tx: {} / ops: {}".format(dataHeader, data['block_num'], txCount, opCount)) 55 | if dataType == "ops_per_blocks": 56 | for height in data: 57 | print("{} - #{}: {}".format(dataHeader, height, data[height])) 58 | -------------------------------------------------------------------------------- /examples/adapter_steem.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | adapter = SteemAdapter(endpoints=['https://direct.steemd.privex.io'], debug=False) 7 | chainsync = ChainSync(adapter) 8 | 9 | def print_event(dataType, data): 10 | if dataType == "block": 11 | print("[{}]: {} [{}] - #{}, previous: {}".format(datetime.datetime.now(), dataType, data['block_id'], data['block_num'], data['previous'])) 12 | if dataType == "op": 13 | print("[{}]: {} [{}] - {}".format(datetime.datetime.now(), dataType, data['transaction_id'], data['operation_type'])) 14 | if dataType == "ops_per_block": 15 | for height in data: 16 | print("[{}]: {} - #{} had {} ops".format(datetime.datetime.now(), dataType, height, data[height])) 17 | if dataType == 'tx': 18 | print("[{}]: {} [{}] - {} ops".format(datetime.datetime.now(), dataType, data['transaction_id'], len(data['operations']))) 19 | if dataType == 'status': 20 | print("[{}]: {} - {} h:{} / i:{}".format(datetime.datetime.now(), dataType, data['time'], data['last_irreversible_block_num'], data['head_block_number'])) 21 | 22 | print('\nGetting block 1') 23 | block = chainsync.get_block(1) 24 | print_event('block', block) 25 | 26 | print('\nGetting transaction c68435a34a7afc701771eb090f96526ed4c2a37b') 27 | tx = chainsync.get_transaction('c68435a34a7afc701771eb090f96526ed4c2a37b') 28 | print_event('tx', tx) 29 | 30 | print('\nGetting multiple transactions') 31 | transactions = [ 32 | '62f68c45f67ecbe4ac6eb8348bce44e73e46611c', 33 | '2f58f00e70b8d0f88e90350043e17ed6ea2eb223', 34 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 35 | ] 36 | for tx in chainsync.get_transactions(transactions): 37 | print_event('tx', tx) 38 | 39 | print('\nGetting ops in transaction c68435a34a7afc701771eb090f96526ed4c2a37b') 40 | for op in chainsync.get_ops_in_transaction('c68435a34a7afc701771eb090f96526ed4c2a37b'): 41 | print_event('op', op) 42 | 43 | print('\nGetting ops in multiple transactions') 44 | transactions = [ 45 | '62f68c45f67ecbe4ac6eb8348bce44e73e46611c', 46 | '2f58f00e70b8d0f88e90350043e17ed6ea2eb223', 47 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 48 | ] 49 | for op in chainsync.get_ops_in_transactions(transactions): 50 | print_event('op', op) 51 | 52 | print('\nGetting blocks 1, 10, 50, 250, 500') 53 | blocks = chainsync.get_blocks([1, 10, 50, 250, 500]) 54 | for block in blocks: 55 | print_event('block', block) 56 | 57 | print('\nGetting blocks 1000-1005') 58 | blocks = chainsync.get_block_sequence(1000, 5) 59 | for block in blocks: 60 | print_event('block', block) 61 | 62 | print('\nGetting all ops in block 9284729...') 63 | for op in chainsync.get_ops_in_block(9284729): 64 | print_event('op', op) 65 | 66 | print('\nGetting withdraw_vesting ops in block 9284729...') 67 | for op in chainsync.get_ops_in_block(9284729, whitelist=['withdraw_vesting']): 68 | print_event('op', op) 69 | 70 | print('\nGetting all ops in block 1000000, 5000000, and 2000000...') 71 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000]): 72 | print_event('op', op) 73 | 74 | print('\nGetting producer_reward ops in block 1000000, 5000000, and 2000000...') 75 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000], whitelist=['producer_reward']): 76 | print_event('op', op) 77 | 78 | print('\nStreaming blocks from head...') 79 | for dataType, data in chainsync.stream(['blocks']): 80 | print_event(dataType, data) 81 | 82 | print('\nStreaming blocks from the irreversible height...') 83 | for dataType, data in chainsync.stream(['blocks'], mode='irreversible'): 84 | print_event(dataType, data) 85 | 86 | print('\nStreaming status...') 87 | for dataType, data in chainsync.stream(['status']): 88 | print_event(dataType, data) 89 | 90 | print('\nStreaming ops_per_blocks...') 91 | for dataType, data in chainsync.stream(['ops_per_blocks']): 92 | print_event(dataType, data) 93 | 94 | print('\nStreaming blocks + status from head...') 95 | for dataType, data in chainsync.stream(['blocks', 'status']): 96 | print_event(dataType, data) 97 | 98 | print('\nStreaming all op from head...') 99 | for dataType, data in chainsync.stream(['ops']): 100 | print_event(dataType, data) 101 | 102 | print('\nStreaming all non-virtual ops...') 103 | for dataType, data in chainsync.stream(['ops'], virtual_ops=False): 104 | print_event(dataType, data) 105 | 106 | print('\nStreaming all virtual ops...') 107 | for dataType, data in chainsync.stream(['ops'], regular_ops=False): 108 | print_event(dataType, data) 109 | 110 | print('\nStreaming vote ops only...') 111 | for dataType, op in chainsync.stream(['ops'], whitelist=['vote']): 112 | print("[{}]: {} [{}] - {} by {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'], op['voter'])) 113 | 114 | print('\nStreaming producer_reward virtual ops only...') 115 | for dataType, op in chainsync.stream(['ops'], whitelist=['producer_reward']): 116 | print("[{}]: {} - {} for {} of {}".format(datetime.datetime.now(), op['block_num'], op['operation_type'], op['producer'], op['vesting_shares'])) 117 | 118 | print('\nStreaming all ops + status from head...') 119 | for dataType, data in chainsync.stream(['ops', 'status']): 120 | print_event(dataType, data) 121 | 122 | print('\nStreaming all blocks + ops (no virtual ops)...') 123 | for dataType, data in chainsync.stream(['blocks', 'ops'], virtual_ops=False): 124 | print_event(dataType, data) 125 | 126 | print('\nStreaming all blocks + ops (no virtual ops), filtering only votes...') 127 | for dataType, data in chainsync.stream(['blocks', 'ops'], whitelist=['vote']): 128 | print_event(dataType, data) 129 | 130 | print('\nStreaming all blocks + virtual ops (no normal ops)...') 131 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False): 132 | print_event(dataType, data) 133 | 134 | print('\nStreaming all blocks + virtual ops (no normal ops), filtering only producer_reward...') 135 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False, whitelist=['producer_reward']): 136 | print_event(dataType, data) 137 | 138 | print('\nStreaming all blocks + all ops + ops_per_blocks + status...') 139 | for dataType, data in chainsync.stream(['blocks', 'ops', 'ops_per_blocks', 'status']): 140 | print_event(dataType, data) 141 | -------------------------------------------------------------------------------- /examples/adapter_steemv2.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steemv2 import SteemV2Adapter 5 | 6 | adapter = SteemV2Adapter(endpoints=['https://api.steemitstage.com/']) 7 | chainsync = ChainSync(adapter) 8 | 9 | def print_event(dataType, data): 10 | if dataType == "block": 11 | print("[{}]: {} [{}] - #{}, previous: {}".format(datetime.datetime.now(), dataType, data['block_id'], data['block_num'], data['previous'])) 12 | if dataType == "op": 13 | print("[{}]: {} [{}] - {}".format(datetime.datetime.now(), dataType, data['transaction_id'], data['operation_type'])) 14 | if dataType == "ops_per_block": 15 | for height in data: 16 | print("[{}]: {} - #{} had {} ops".format(datetime.datetime.now(), dataType, height, data[height])) 17 | if dataType == 'tx': 18 | print("[{}]: {} [{}] - {} ops".format(datetime.datetime.now(), dataType, data['transaction_id'], len(data['operations']))) 19 | if dataType == 'status': 20 | print("[{}]: {} - {} h:{} / i:{}".format(datetime.datetime.now(), dataType, data['time'], data['last_irreversible_block_num'], data['head_block_number'])) 21 | 22 | print('\nGetting block 1') 23 | block = chainsync.get_block(1) 24 | print_event('block', block) 25 | 26 | print('\nGetting transaction c68435a34a7afc701771eb090f96526ed4c2a37b') 27 | tx = chainsync.get_transaction('c68435a34a7afc701771eb090f96526ed4c2a37b') 28 | print_event('tx', tx) 29 | 30 | print('\nGetting multiple transactions') 31 | transactions = [ 32 | '62f68c45f67ecbe4ac6eb8348bce44e73e46611c', 33 | '2f58f00e70b8d0f88e90350043e17ed6ea2eb223', 34 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 35 | ] 36 | for tx in chainsync.get_transactions(transactions): 37 | print_event('tx', tx) 38 | 39 | print('\nGetting ops in transaction c68435a34a7afc701771eb090f96526ed4c2a37b') 40 | for op in chainsync.get_ops_in_transaction('c68435a34a7afc701771eb090f96526ed4c2a37b'): 41 | print_event('op', op) 42 | 43 | print('\nGetting ops in multiple transactions') 44 | transactions = [ 45 | '62f68c45f67ecbe4ac6eb8348bce44e73e46611c', 46 | '2f58f00e70b8d0f88e90350043e17ed6ea2eb223', 47 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 48 | ] 49 | for op in chainsync.get_ops_in_transactions(transactions): 50 | print_event('op', op) 51 | 52 | print('\nGetting blocks 1, 10, 50, 250, 500') 53 | blocks = chainsync.get_blocks([1, 10, 50, 250, 500]) 54 | for block in blocks: 55 | print_event('block', block) 56 | 57 | print('\nGetting blocks 1000-1005') 58 | blocks = chainsync.get_block_sequence(1000, 5) 59 | for block in blocks: 60 | print_event('block', block) 61 | 62 | print('\nGetting all ops in block 9284729...') 63 | for op in chainsync.get_ops_in_block(9284729): 64 | print_event('op', op) 65 | 66 | print('\nGetting withdraw_vesting ops in block 9284729...') 67 | for op in chainsync.get_ops_in_block(9284729, whitelist=['withdraw_vesting']): 68 | print_event('op', op) 69 | 70 | print('\nGetting all ops in block 1000000, 5000000, and 2000000...') 71 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000]): 72 | print_event('op', op) 73 | 74 | print('\nGetting producer_reward ops in block 1000000, 5000000, and 2000000...') 75 | for op in chainsync.get_ops_in_blocks([1000000, 5000000, 2000000], whitelist=['producer_reward']): 76 | print_event('op', op) 77 | 78 | print('\nStreaming blocks from head...') 79 | for dataType, data in chainsync.stream(['blocks']): 80 | print_event(dataType, data) 81 | 82 | print('\nStreaming blocks from the irreversible height...') 83 | for dataType, data in chainsync.stream(['blocks'], mode='irreversible'): 84 | print_event(dataType, data) 85 | 86 | print('\nStreaming status...') 87 | for dataType, data in chainsync.stream(['status']): 88 | print_event(dataType, data) 89 | 90 | print('\nStreaming ops_per_blocks...') 91 | for dataType, data in chainsync.stream(['ops_per_blocks']): 92 | print_event(dataType, data) 93 | 94 | print('\nStreaming blocks + status from head...') 95 | for dataType, data in chainsync.stream(['blocks', 'status']): 96 | print_event(dataType, data) 97 | 98 | print('\nStreaming all op from head...') 99 | for dataType, data in chainsync.stream(['ops']): 100 | print_event(dataType, data) 101 | 102 | print('\nStreaming all non-virtual ops...') 103 | for dataType, data in chainsync.stream(['ops'], virtual_ops=False): 104 | print_event(dataType, data) 105 | 106 | print('\nStreaming all virtual ops...') 107 | for dataType, data in chainsync.stream(['ops'], regular_ops=False): 108 | print_event(dataType, data) 109 | 110 | print('\nStreaming vote ops only...') 111 | for dataType, op in chainsync.stream(['ops'], whitelist=['vote']): 112 | print("[{}]: {} [{}] - {} by {}".format(datetime.datetime.now(), op['block_num'], op['transaction_id'], op['operation_type'], op['voter'])) 113 | 114 | print('\nStreaming producer_reward virtual ops only...') 115 | for dataType, op in chainsync.stream(['ops'], whitelist=['producer_reward']): 116 | print("[{}]: {} - {} for {} of {}".format(datetime.datetime.now(), op['block_num'], op['operation_type'], op['producer'], op['vesting_shares'])) 117 | 118 | print('\nStreaming all ops + status from head...') 119 | for dataType, data in chainsync.stream(['ops', 'status']): 120 | print_event(dataType, data) 121 | 122 | print('\nStreaming all blocks + ops (no virtual ops)...') 123 | for dataType, data in chainsync.stream(['blocks', 'ops'], virtual_ops=False): 124 | print_event(dataType, data) 125 | 126 | print('\nStreaming all blocks + ops (no virtual ops), filtering only votes...') 127 | for dataType, data in chainsync.stream(['blocks', 'ops'], whitelist=['vote']): 128 | print_event(dataType, data) 129 | 130 | print('\nStreaming all blocks + virtual ops (no normal ops)...') 131 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False): 132 | print_event(dataType, data) 133 | 134 | print('\nStreaming all blocks + virtual ops (no normal ops), filtering only producer_reward...') 135 | for dataType, data in chainsync.stream(['blocks', 'ops'], regular_ops=False, whitelist=['producer_reward']): 136 | print_event(dataType, data) 137 | 138 | print('\nStreaming all blocks + all ops + ops_per_blocks + status...') 139 | for dataType, data in chainsync.stream(['blocks', 'ops', 'ops_per_blocks', 'status']): 140 | print_event(dataType, data) 141 | -------------------------------------------------------------------------------- /examples/async_mongo_op_indexer.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import multiprocessing 3 | import os 4 | import signal 5 | import time 6 | import datetime 7 | 8 | from chainsync import ChainSync 9 | from chainsync.adapters.steem import SteemAdapter 10 | 11 | from chainmodel import ChainModel 12 | from chainmodel.models.steem.schema import Schema 13 | 14 | from pymongo import MongoClient 15 | 16 | # Setup ChainSync + Steem adapter 17 | adapter = SteemAdapter(endpoints=['https://api.steemit.com/']) 18 | chainsync = ChainSync(adapter) 19 | 20 | # establish models with schema 21 | chainmodel = ChainModel(schema=Schema()) 22 | 23 | class Example(): 24 | 25 | def __init__(self): 26 | # Handling KeyboardInterrupt so the pool doesn't override 27 | # https://stackoverflow.com/questions/11312525/catch-ctrlc-sigint-and-exit-multiprocesses-gracefully-in-python 28 | original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) 29 | 30 | self.requests = multiprocessing.Queue() 31 | self.responses = multiprocessing.Queue() 32 | 33 | self.requests_pool = multiprocessing.Pool(None, self.worker, (self.requests,)) 34 | self.responses_pool = multiprocessing.Pool(None, self.saver, (self.responses,)) 35 | 36 | # Reassign signal 37 | signal.signal(signal.SIGINT, original_sigint_handler) 38 | 39 | # Get chain config 40 | self.config = chainsync.get_config() 41 | 42 | def worker(self, queue): 43 | while True: 44 | # get the height and limit from the requests queue 45 | height, limit = queue.get(True) 46 | 47 | # retrieve the requested blocks 48 | for dataType, data in chainsync.get_stream(['ops', 'ops_per_block'], start_block=height, end_block=(height + limit)): 49 | self.responses.put((dataType, data)) 50 | 51 | def saver(self, queue): 52 | # each saver gets it's own mongoclient 53 | mongo = MongoClient('mongodb://localhost', connect=False) 54 | db = mongo['steemindex'] 55 | while True: 56 | # get any opData waiting in the responses queue 57 | dataType, data = queue.get(True) 58 | 59 | dataHeader = "{} #{}: {}".format(datetime.datetime.now(), data['block_num'], dataType) 60 | 61 | # if this was an op, save it 62 | if dataType == "op": 63 | print("{} {} {}".format(dataHeader, data['transaction_id'], data['operation_type'])) 64 | # model data based on schema 65 | model = chainmodel.get(data) 66 | # insert in database 67 | db.ops.update(model.query(), model.data(), upsert=True) 68 | 69 | # if this was an ops_per_block response, save the counts per block 70 | # (this data can be used to validate that every operation in a block was successfully saved) 71 | if dataType == "ops_per_block": 72 | for height in data: 73 | print("{} - #{}: {}".format(dataHeader, height, data[height])) 74 | db.ops_per_block.update({'_id': height}, {'v': data[height]}, upsert=True) 75 | 76 | def run(self, start_block=1, batch_size=10): 77 | while True: 78 | # Get blockchain status 79 | status = chainsync.get_status() 80 | 81 | # Determine the current head block 82 | head_block = status['head_block_number'] 83 | 84 | # If no start block is specified, start streaming from head 85 | if start_block is None: 86 | start_block = head_block 87 | 88 | # Set initial remaining blocks for this stream 89 | remaining = head_block - start_block + 1 90 | 91 | # While remaining blocks exist - batch load them 92 | while remaining > 0: 93 | # Determine how many blocks to load with this request 94 | limit = batch_size 95 | 96 | # Modify the amount of blocks to load if lower than the batch_size 97 | if remaining < batch_size: 98 | limit = remaining 99 | 100 | # Track the last block successfully processed 101 | last_block_processed = start_block 102 | 103 | # Queue the requested block + limit 104 | self.requests.put((start_block, limit)) 105 | 106 | # Remaining blocks to process 107 | remaining = head_block - start_block 108 | 109 | # Next block to start on 110 | if remaining > batch_size: 111 | start_block = start_block + batch_size 112 | else: 113 | start_block = last_block_processed + 1 114 | 115 | # Pause loop based on the blockchain block time 116 | block_interval = self.config[chainsync.adapter.config['BLOCK_INTERVAL']] if 'BLOCK_INTERVAL' in chainsync.adapter.config else 3 117 | time.sleep(block_interval) 118 | 119 | if __name__ == '__main__': 120 | 121 | batch_size = 10 122 | start_block = 1000000 123 | 124 | example = Example() 125 | example.run(start_block=start_block, batch_size=batch_size) 126 | -------------------------------------------------------------------------------- /examples/simple_mongo_op_indexer.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | from chainmodel import ChainModel 7 | from chainmodel.models.steem.schema import Schema 8 | 9 | # define endpoints 10 | endpoints = [ 11 | 'https://api.steemit.com/', 12 | 'https://rpc.buildteam.io/', 13 | 'https://steemd.privex.io/', 14 | ] 15 | 16 | # setup adapter + chainsync 17 | adapter = SteemAdapter(endpoints=endpoints) 18 | chainsync = ChainSync(adapter) 19 | 20 | # establish models with schema 21 | chainmodel = ChainModel(schema=Schema()) 22 | 23 | # connect to database 24 | mongo = MongoClient('mongodb://localhost', connect=False) 25 | db = mongo['steem'] 26 | 27 | # stream all operations 28 | for dataType, opData in chainsync.stream(['ops']): 29 | 30 | # model data based on schema 31 | model = chainmodel.get(opData) 32 | 33 | # insert in database 34 | db.ops.update(model.query(), model.data(), upsert=True) 35 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jsonrpcclient 2 | requests 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file=README.md 3 | 4 | [tool:pytest] 5 | testpaths = tests 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | TEST_REQUIRED = [ 4 | 'pep8', 5 | 'pytest', 6 | 'pytest-pylint', 7 | 'pytest-xdist', 8 | 'pytest-runner', 9 | 'pytest-pep8', 10 | 'pytest-cov', 11 | 'yapf', 12 | 'autopep8' 13 | ] 14 | 15 | extras = { 16 | 'test': TEST_REQUIRED, 17 | } 18 | 19 | setup( 20 | name='chainsync', 21 | version='0.4.0', 22 | description='Utility to stream and sync blocks into other data sources', 23 | url='http://github.com/aaroncox/chainsync', 24 | author='Aaron Cox', 25 | author_email='aaron@greymass.com', 26 | license='MIT', 27 | packages=find_packages(), 28 | install_requires=['jsonrpcclient', 'requests'], 29 | setup_requires=['pytest-runner'], 30 | tests_require=TEST_REQUIRED, 31 | extras_require=extras, 32 | zip_safe=False 33 | ) 34 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaroncox/chainsync/1277363787e37aa595571ab8a789831aadf0d3e6/tests/__init__.py -------------------------------------------------------------------------------- /tests/chainsync/test_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import pytest 3 | 4 | from chainsync import ChainSync 5 | 6 | 7 | @pytest.mark.usefixtures("client") 8 | class ChainSyncBaseTestCase(unittest.TestCase): 9 | def setUp(self): 10 | self.chainsync = ChainSync(adapter=self.adapter, retry=False) 11 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncMainTestCase(ChainSyncBaseTestCase): 8 | 9 | def test_main_debug_flag_default_false(self): 10 | self.assertEqual(self.chainsync.debug, False) 11 | 12 | def test_main_debug_flag_set_true(self): 13 | adapter = SteemAdapter() 14 | custom = ChainSync(adapter, debug=True) 15 | self.assertEqual(custom.debug, True) 16 | 17 | def test_main_debug_flag_set_false_from_adapter(self): 18 | adapter = SteemAdapter(debug=True) 19 | custom = ChainSync(adapter) 20 | self.assertEqual(custom.debug, False) 21 | 22 | def test_main_debug_flag_set_true_from_main_false_for_adapter(self): 23 | adapter = SteemAdapter(debug=False) 24 | # main debug flag should override adapter 25 | custom = ChainSync(adapter, debug=True) 26 | self.assertEqual(custom.debug, True) 27 | self.assertEqual(custom.adapter.debug, True) 28 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_adapter.py: -------------------------------------------------------------------------------- 1 | from chainsync import ChainSync 2 | from chainsync.adapters.steem import SteemAdapter 3 | 4 | import unittest 5 | 6 | 7 | class ChainSyncAdapterTestCase(unittest.TestCase): 8 | def setUp(self): 9 | self.chainsync = ChainSync(adapter=SteemAdapter) 10 | 11 | def test_adapter_init_default_adapter(self): 12 | self.assertNotEqual(self.chainsync.adapter, None) 13 | 14 | def test_adapter_init_custom_adapter_custom_endpoint_no_endpoints(self): 15 | adapter = SteemAdapter() 16 | custom = ChainSync(adapter) 17 | self.assertEqual(custom.adapter.endpoint, 'http://localhost:8090') 18 | 19 | def test_adapter_init_custom_adapter_custom_endpoint_string(self): 20 | adapter = SteemAdapter(endpoints='http://localhost:8091') 21 | custom = ChainSync(adapter) 22 | self.assertEqual(custom.adapter.endpoint, 'http://localhost:8091') 23 | 24 | def test_adapter_init_custom_adapter_custom_endpoint_list(self): 25 | endpoints = ['http://localhost:8091', 'http://localhost:8090'] 26 | adapter = SteemAdapter(endpoints=endpoints) 27 | custom = ChainSync(adapter) 28 | self.assertEqual(custom.adapter.endpoint, 'http://localhost:8091') 29 | self.assertEqual(custom.adapter.endpoints, endpoints) 30 | 31 | def test_adapter_debug_flag_default_false(self): 32 | self.assertEqual(self.chainsync.adapter.debug, False) 33 | 34 | def test_adapter_debug_flag_set_true(self): 35 | adapter = SteemAdapter(debug=True) 36 | custom = ChainSync(adapter) 37 | self.assertEqual(custom.adapter.debug, True) 38 | 39 | def test_adapter_debug_flag_set_true_from_main(self): 40 | adapter = SteemAdapter() 41 | custom = ChainSync(adapter, debug=True) 42 | self.assertEqual(custom.adapter.debug, True) 43 | 44 | def test_adapter_debug_flag_set_true_from_main_false_for_adapter(self): 45 | adapter = SteemAdapter(debug=False) 46 | # main debug flag should override adapter 47 | custom = ChainSync(adapter, debug=True) 48 | self.assertEqual(custom.adapter.debug, True) 49 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_from_block_get_ops.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncFromBlockGetOpsTestCase(ChainSyncBaseTestCase): 5 | 6 | def setUp(self): 7 | # initialize parent 8 | super(ChainSyncFromBlockGetOpsTestCase, self).setUp() 9 | # has both virtual ops and regular ops 10 | self.block = self.chainsync.get_block(1093) 11 | 12 | def test_from_block_get_ops(self): 13 | for result in self.chainsync.from_block_get_ops(self.block): 14 | self.assertEqual(result['block_num'], 1093) 15 | 16 | def test_from_block_get_ops_no_value(self): 17 | with self.assertRaises(TypeError) as context: 18 | self.chainsync.from_block_get_ops() 19 | 20 | def test_from_block_get_ops_non_block_value(self): 21 | with self.assertRaises(TypeError) as context: 22 | result = [result for result in self.chainsync.from_block_get_ops("1")] 23 | 24 | def test_from_block_get_ops_whitelist(self): 25 | whitelist = ['pow'] 26 | for result in self.chainsync.from_block_get_ops(self.block, virtual_ops=True, whitelist=whitelist): 27 | self.assertTrue(result['operation_type'] in whitelist) 28 | 29 | def test_from_block_get_ops_detect_virtual_ops_in_whitelist(self): 30 | whitelist = ['producer_reward'] 31 | # virtual_ops is not enabled, but the whitelist is requesting a virtual op 32 | results = [result for result in self.chainsync.from_block_get_ops(self.block, whitelist=whitelist)] 33 | # we should still get the virtual op back 34 | self.assertTrue(len(results), 1) 35 | for result in results: 36 | # and it should be in the whitelist 37 | self.assertTrue(result['operation_type'] in whitelist) 38 | 39 | def test_from_block_get_ops_with_virtual_ops_enabled(self): 40 | for result in self.chainsync.from_block_get_ops(self.block, virtual_ops=True): 41 | self.assertTrue(result['operation_type'] in ['producer_reward', 'pow']) 42 | 43 | def test_from_block_get_ops_with_virtual_ops_disabled(self): 44 | for result in self.chainsync.from_block_get_ops(self.block, virtual_ops=False): 45 | self.assertEqual(result['operation_type'], 'pow') 46 | 47 | def test_from_block_get_ops_with_regular_ops_enabled(self): 48 | for result in self.chainsync.from_block_get_ops(self.block, regular_ops=True): 49 | self.assertTrue(result['operation_type'] in ['producer_reward', 'pow']) 50 | 51 | def test_from_block_get_ops_with_regular_ops_enabled_virtual_ops_enabled(self): 52 | results = [result for result in self.chainsync.from_block_get_ops(self.block, regular_ops=True, virtual_ops=True) if result] 53 | self.assertEqual(len(results), 5) # 4 pow + 1 producer_reward 54 | for result in results: 55 | self.assertTrue(result['operation_type'] in ['pow', 'producer_reward']) 56 | 57 | def test_from_block_get_ops_with_regular_ops_disabled(self): 58 | results = [result for result in self.chainsync.from_block_get_ops(self.block, regular_ops=False) if result] 59 | # Should return 0 results, virtual ops aren't loaded by default (requires extra API call) 60 | self.assertEqual(len(results), 0) 61 | 62 | def test_from_block_get_ops_with_regular_ops_disabled_virtual_ops_enabled(self): 63 | results = [result for result in self.chainsync.from_block_get_ops(self.block, regular_ops=False, virtual_ops=True) if result] 64 | # Should return a single producer_reward 65 | self.assertEqual(len(results), 1) 66 | for result in results: 67 | self.assertEqual(result['operation_type'], 'producer_reward') 68 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_from_block_get_ops_regular.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncFromBlockGetOpsRegularTestCase(ChainSyncBaseTestCase): 5 | 6 | def setUp(self): 7 | # initialize parent 8 | super(ChainSyncFromBlockGetOpsRegularTestCase, self).setUp() 9 | # has both virtual ops and regular ops 10 | self.block = self.chainsync.get_block(1093) 11 | 12 | def test_from_block_get_ops_regular(self): 13 | for result in self.chainsync.from_block_get_ops_regular(self.block): 14 | self.assertEqual(result['block_num'], 1093) 15 | 16 | def test_from_block_get_ops_regular_no_value(self): 17 | with self.assertRaises(TypeError) as context: 18 | result = [result for result in self.chainsync.from_block_get_ops_regular("1")] 19 | 20 | def test_from_block_get_ops_regular_non_block_value(self): 21 | with self.assertRaises(TypeError) as context: 22 | result = [result for result in self.chainsync.from_block_get_ops_regular("1")] 23 | 24 | def test_from_block_get_ops_regular_whitelist(self): 25 | whitelist = ['pow'] 26 | for result in self.chainsync.from_block_get_ops_regular(self.block, whitelist=whitelist): 27 | self.assertTrue(result['operation_type'] in whitelist) 28 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_block.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetBlockTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_block(self): 7 | result = self.chainsync.get_block(1) 8 | self.assertEqual(result['block_num'], 1) 9 | 10 | def test_get_block_no_value(self): 11 | with self.assertRaises(TypeError) as context: 12 | self.chainsync.get_block() 13 | 14 | def test_get_block_string_value(self): 15 | result = self.chainsync.get_block("1") 16 | self.assertEqual(result['block_num'], 1) 17 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_block_sequence.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetBlockSequenceTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_block_sequence(self): 7 | blocks = [8, 9, 10] 8 | results = list(self.chainsync.get_block_sequence(8, 3)) 9 | for idx, result in enumerate(results): 10 | self.assertEqual(result['block_num'], blocks[idx]) 11 | self.assertEqual(len(results), 3) 12 | 13 | def test_get_block_sequence_string_value(self): 14 | blocks = [7, 8, 9, 10] 15 | results = list(self.chainsync.get_block_sequence('7', '4')) 16 | for idx, result in enumerate(results): 17 | self.assertEqual(result['block_num'], blocks[idx]) 18 | self.assertEqual(len(results), 4) 19 | 20 | def test_get_block_sequence_no_start_block(self): 21 | with self.assertRaises(TypeError) as context: 22 | self.chainsync.get_block_sequence() 23 | 24 | def test_get_block_sequence_no_height(self): 25 | with self.assertRaises(TypeError) as context: 26 | self.chainsync.get_block_sequence(1) 27 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_blocks.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetBlocksTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_blocks(self): 7 | blocks = [5, 10] 8 | results = self.chainsync.get_blocks(blocks) 9 | for idx, result in enumerate(results): 10 | self.assertEqual(result['block_num'], blocks[idx]) 11 | 12 | def test_get_blocks_no_value(self): 13 | with self.assertRaises(TypeError) as context: 14 | self.chainsync.get_blocks() 15 | 16 | def test_get_blocks_int_value(self): 17 | with self.assertRaises(TypeError) as context: 18 | self.chainsync.get_blocks(10) 19 | 20 | def test_get_blocks_string_value(self): 21 | with self.assertRaises(TypeError) as context: 22 | self.chainsync.get_blocks("10") 23 | 24 | def test_get_blocks_list_of_string_values(self): 25 | blocks = ["5", "10"] 26 | results = self.chainsync.get_blocks(blocks) 27 | for idx, result in enumerate(results): 28 | self.assertEqual(result['block_num'], int(blocks[idx])) 29 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_config.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetConfigTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_config(self): 7 | result = self.chainsync.get_config() 8 | self.assertTrue('IS_TEST_NET' in result) 9 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_head_block.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetStatusTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_head_block(self): 7 | height = self.chainsync.get_head_block() 8 | # ensure it returns a number 9 | self.assertIsInstance(height, int) 10 | 11 | def test_get_head_block_with_status(self): 12 | height, status = self.chainsync.get_head_block(return_status=True) 13 | # ensure it returns a number 14 | self.assertIsInstance(height, int) 15 | # as well as a dict with the status 16 | self.assertIsInstance(status, dict) 17 | 18 | def test_get_head_block_mode_irreversible(self): 19 | head = self.chainsync.get_head_block() 20 | irreversible = self.chainsync.get_head_block(mode='irreversible') 21 | # Ensure they are both numbers 22 | self.assertIsInstance(head, int) 23 | self.assertIsInstance(irreversible, int) 24 | # These two values will likely never be the same 25 | self.assertTrue(head != irreversible) 26 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_ops_in_block.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetBlockTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_ops_in_block(self): 7 | results = self.chainsync.get_ops_in_block(1) 8 | for result in results: 9 | self.assertEqual(result['block_num'], 1) 10 | 11 | def test_get_ops_in_block_no_value(self): 12 | with self.assertRaises(TypeError) as context: 13 | self.chainsync.get_ops_in_block() 14 | 15 | def test_get_ops_in_block_string_value(self): 16 | results = self.chainsync.get_ops_in_block("1") 17 | for result in results: 18 | self.assertEqual(result['block_num'], 1) 19 | 20 | def test_get_ops_in_block_filtered(self): 21 | block = 1 22 | whitelist = ['producer_reward'] 23 | results = self.chainsync.get_ops_in_block(block, whitelist=whitelist) 24 | for idx, result in enumerate(results): 25 | self.assertTrue(result['operation_type'] in whitelist) 26 | 27 | def test_get_ops_in_block_regular_and_virtual(self): 28 | block = 1093 # contains pow (non-virtual) + producer (virtual) ops 29 | results = self.chainsync.get_ops_in_block(block, virtual_only=False) 30 | for idx, result in enumerate(results): 31 | self.assertTrue(result['operation_type'] in ['pow', 'producer_reward']) 32 | 33 | def test_get_ops_in_block_virtual_only(self): 34 | block = 1093 # contains pow (non-virtual) + producer (virtual) ops 35 | results = self.chainsync.get_ops_in_block(block, virtual_only=True) 36 | for idx, result in enumerate(results): 37 | self.assertTrue(result['operation_type'] in ['producer_reward']) 38 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_ops_in_block_sequence.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetOpsInBlockSequenceTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_ops_in_block_sequence(self): 7 | blocks = [200000, 200001] 8 | start_block = 200000 9 | limit = 1 10 | results = self.chainsync.get_ops_in_block_sequence(start_block, limit) 11 | for idx, result in enumerate(results): 12 | self.assertTrue(result['block_num'] in blocks) 13 | 14 | def test_get_ops_in_block_sequence_no_start_block(self): 15 | with self.assertRaises(TypeError) as context: 16 | self.chainsync.get_ops_in_block_sequence() 17 | 18 | def test_get_ops_in_block_sequence_no_limit(self): 19 | with self.assertRaises(TypeError) as context: 20 | self.chainsync.get_ops_in_block_sequence(1) 21 | 22 | def test_get_ops_in_block_sequence_string_values(self): 23 | blocks = [1093, 1094] 24 | results = self.chainsync.get_ops_in_block_sequence('1093', '2') 25 | for idx, result in enumerate(results): 26 | self.assertTrue(result['block_num'] in blocks) 27 | 28 | def test_get_ops_in_block_sequence_filtered(self): 29 | blocks = [1093, 1094] # contains pow (non-virtual) + producer (virtual) ops 30 | whitelist = ['pow'] 31 | results = self.chainsync.get_ops_in_block_sequence(1093, 2, whitelist=whitelist) 32 | for idx, result in enumerate(results): 33 | self.assertTrue(result['operation_type'] in whitelist) 34 | self.assertTrue(result['block_num'] in blocks) 35 | 36 | def test_get_ops_in_block_sequence_regular_and_virtual(self): 37 | blocks = [1093, 1094] # contains pow (non-virtual) + producer (virtual) ops 38 | results = self.chainsync.get_ops_in_block_sequence(1093, 2, virtual_only=False) 39 | for idx, result in enumerate(results): 40 | self.assertTrue(result['operation_type'] in ['pow', 'producer_reward']) 41 | self.assertTrue(result['block_num'] in blocks) 42 | 43 | def test_get_ops_in_block_sequence_virtual_only(self): 44 | blocks = [1093, 1094] # contains pow (non-virtual) + producer (virtual) ops 45 | results = self.chainsync.get_ops_in_block_sequence(1093, 2, virtual_only=True) 46 | for idx, result in enumerate(results): 47 | self.assertTrue(result['operation_type'] in ['producer_reward']) 48 | self.assertTrue(result['block_num'] in blocks) 49 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_ops_in_blocks.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetOpsInBlocksTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_ops_in_blocks(self): 7 | blocks = [200000] 8 | results = self.chainsync.get_ops_in_blocks(blocks) 9 | for idx, result in enumerate(results): 10 | self.assertTrue(result['block_num'] in blocks) 11 | 12 | def test_get_ops_in_blocks_no_value(self): 13 | with self.assertRaises(TypeError) as context: 14 | self.chainsync.get_ops_in_blocks() 15 | 16 | def test_get_ops_in_blocks_non_list_value(self): 17 | with self.assertRaises(TypeError) as context: 18 | results = self.chainsync.get_ops_in_blocks(10) 19 | for result in results: 20 | print(result) 21 | 22 | def test_get_ops_in_blocks_list_of_string_values(self): 23 | blocks = ["5", "10"] 24 | blocks_as_int = [5, 10] 25 | results = self.chainsync.get_ops_in_blocks(blocks) 26 | for idx, result in enumerate(results): 27 | self.assertTrue(result['block_num'] in blocks_as_int) 28 | 29 | def test_get_ops_in_blocks_filtered(self): 30 | blocks = [10000000, 11000000] 31 | whitelist = ['producer_reward'] 32 | results = self.chainsync.get_ops_in_blocks(blocks, whitelist=whitelist) 33 | for idx, result in enumerate(results): 34 | self.assertTrue(result['operation_type'] in whitelist) 35 | 36 | def test_get_ops_in_blocks_regular_and_virtual(self): 37 | blocks = [1093, 1094] # contains pow (non-virtual) + producer (virtual) ops 38 | results = self.chainsync.get_ops_in_blocks(blocks, virtual_only=False) 39 | for idx, result in enumerate(results): 40 | self.assertTrue(result['operation_type'] in ['pow', 'producer_reward']) 41 | 42 | def test_get_ops_in_blocks_virtual_only(self): 43 | blocks = [1093, 1094] # contains pow (non-virtual) + producer (virtual) ops 44 | results = self.chainsync.get_ops_in_blocks(blocks, virtual_only=True) 45 | for idx, result in enumerate(results): 46 | self.assertTrue(result['operation_type'] in ['producer_reward']) 47 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_ops_in_transaction.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncGetOpsInTransactionTestCase(ChainSyncBaseTestCase): 8 | 9 | def setUp(self): 10 | adapter = SteemAdapter( 11 | endpoints='https://steemd.pevo.science', 12 | retry=False 13 | ) 14 | self.chainsync = ChainSync(adapter=adapter, retry=False) 15 | 16 | def test_get_ops_in_transaction(self): 17 | tx_id = 'c68435a34a7afc701771eb090f96526ed4c2a37b' 18 | result = self.chainsync.get_ops_in_transaction(tx_id) 19 | for op in result: 20 | self.assertEqual(op['block_num'], 20905025) 21 | 22 | def test_get_ops_in_transaction_exception_no_transaction_id(self): 23 | with self.assertRaises(TypeError) as context: 24 | self.chainsync.get_ops_in_transaction() 25 | 26 | def test_get_ops_in_transaction_exception_invalid_transaction_id(self): 27 | with self.assertRaises(Exception) as context: 28 | results = [op in self.chainsync.get_ops_in_transaction('0000000000000000000000000000000000000000')] 29 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_ops_in_transactions.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncGetTransactionsTestCase(ChainSyncBaseTestCase): 8 | 9 | def setUp(self): 10 | adapter = SteemAdapter( 11 | endpoints='https://steemd.pevo.science', 12 | retry=False 13 | ) 14 | self.chainsync = ChainSync(adapter=adapter, retry=False) 15 | 16 | def test_get_ops_in_transactions(self): 17 | blocks = [ 18 | 20905050, 19 | 20905025, 20 | ] 21 | txs = [ 22 | 'a3815d4a17f1331481ec6bf89ba0844ce16175bc', 23 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 24 | ] 25 | result = self.chainsync.get_ops_in_transactions(txs) 26 | for op in result: 27 | self.assertTrue(op['block_num'] in blocks) 28 | 29 | def test_get_ops_in_transactions_exception_no_transaction_id(self): 30 | with self.assertRaises(TypeError) as context: 31 | self.chainsync.get_transactions() 32 | 33 | def test_get_ops_in_transactions_exception_invalid_transaction_id(self): 34 | with self.assertRaises(Exception) as context: 35 | txs = [ 36 | 'a3815d4a17f1331481ec6bf89ba0844ce16175bc', 37 | '0000000000000000000000000000000000000000', # invalid tx 38 | ] 39 | results = [op in self.chainsync.get_ops_in_transactions(txs)] 40 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_status.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetStatusTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_status(self): 7 | result = self.chainsync.get_status() 8 | self.assertTrue('time' in result) 9 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_stream.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncGetStreamTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_get_stream_blocks(self): 7 | stream_data = list(self.chainsync.get_stream(['blocks'], start_block=1)) 8 | # Should only retrive the blocks within the initial batch size 9 | self.assertEqual(len(stream_data), 10) 10 | # Should only be returning the data type requested 11 | self.assertTrue(stream_data[0][0] in ['block']) 12 | # Should be block 1 - 10 (default batch size is 10) 13 | self.assertTrue(stream_data[0][1]['block_num'] <= 10) 14 | 15 | def test_get_stream_blocks_with_batch_size(self): 16 | stream_data = list(self.chainsync.get_stream(['blocks'], start_block=1, batch_size=3)) 17 | # Should only retrive the blocks within the initial batch size 18 | self.assertEqual(len(stream_data), 3) 19 | # Should only be returning the data type requested 20 | self.assertTrue(stream_data[0][0] in ['block']) 21 | # Should be block 1 - 10 (default batch size is 10) 22 | self.assertTrue(stream_data[0][1]['block_num'] <= 3) 23 | 24 | def test_get_stream_blocks_with_end(self): 25 | stream_data = list(self.chainsync.get_stream(['blocks'], start_block=1, end_block=4)) 26 | # Should only retrieve a single event 27 | self.assertEqual(len(stream_data), 4) 28 | # Should only be returning the data types requested 29 | self.assertTrue([d[0] for d in stream_data][0] in ['block']) 30 | # Should be returning both blocks 1 & 2 31 | for block in [d[1] for d in stream_data]: 32 | self.assertTrue(block['block_num'] in [1, 2, 3, 4]) 33 | 34 | def test_get_stream_ops(self): 35 | stream_data = list(self.chainsync.get_stream(['ops'], start_block=1)) 36 | # Should only retrieve a single event (op) - even though there's 10 blocks 37 | self.assertEqual(len(stream_data), 1) 38 | # Should only be returning the data type requested 39 | self.assertTrue(stream_data[0][0] in ['op']) 40 | # Should be block 1 - 10 (default batch size is 10) 41 | self.assertTrue(stream_data[0][1]['block_num'] <= 10) 42 | 43 | def test_get_stream_ops_with_batch_size(self): 44 | stream_data = list(self.chainsync.get_stream(['ops'], start_block=3000000, batch_size=4)) 45 | # Should retrieve 23 events (ops) 46 | self.assertEqual(len(stream_data), 23) 47 | self.assertEqual(len([d[1] for d in stream_data if d[0] == 'op']), 23) 48 | # Should only be returning the data type requested 49 | self.assertTrue(stream_data[0][0] in ['op']) 50 | for op in [d[1] for d in stream_data]: 51 | # Should be blocks 3000000 - 3000003 52 | self.assertTrue(op['block_num'] in [3000000, 3000001, 3000002, 3000003]) 53 | 54 | def test_get_stream_ops_with_whitelist(self): 55 | stream_data = list(self.chainsync.get_stream(['ops'], start_block=3000000, whitelist=['vote', 'comment'])) 56 | # Should retrieve 8 events (ops, 6 vote and 2 comment) 57 | self.assertEqual(len(stream_data), 8) 58 | self.assertEqual(len([d[1] for d in stream_data if d[0] == 'op' and d[1]['operation_type'] == 'vote']), 6) 59 | self.assertEqual(len([d[1] for d in stream_data if d[0] == 'op' and d[1]['operation_type'] == 'comment']), 2) 60 | # Should only be returning the data type requested 61 | self.assertTrue(stream_data[0][0] in ['op']) 62 | for op in [d[1] for d in stream_data]: 63 | # Should be blocks 3000000 - 3000003 64 | self.assertTrue(op['block_num'] in range(3000000, 3000010)) 65 | # Should all be votes 66 | self.assertTrue(op['operation_type'] in ['vote', 'comment']) 67 | 68 | def test_get_stream_ops_with_end(self): 69 | stream_data = list(self.chainsync.get_stream(['ops'], start_block=3000000, end_block=3000003)) 70 | # Should be retrieving a total of 23 ops from these 4 blocks 71 | self.assertEqual(len(stream_data), 23) 72 | # Should only be returning the data types requested 73 | self.assertTrue([d[0] for d in stream_data][0] in ['op']) 74 | # Should be returning ops from all blocks requested 75 | for op in [d[1] for d in stream_data]: 76 | self.assertTrue(op['block_num'] in [3000000, 3000001, 3000002, 3000003]) 77 | 78 | def test_get_stream_ops_per_blocks(self): 79 | stream_data = list(self.chainsync.get_stream(['ops', 'ops_per_blocks'], start_block=1)) 80 | # Should retrieve two events (1 op and 1 ops_per_blocks) 81 | self.assertEqual(len(stream_data), 2) 82 | # Should only be returning the data types requested 83 | self.assertTrue([ 84 | d[0] for d in stream_data 85 | ][0] in ['op', 'ops_per_block']) 86 | # The ops should return the proper block number 87 | for op in [d[1] for d in stream_data if d[0] == 'op']: 88 | # Should be block 1 - 10 (default batch size is 10) 89 | self.assertTrue(op['block_num'] <= 10) 90 | # The ops_per_block should return the number of ops in the block 91 | for counter in [d[1] for d in stream_data if d[0] == 'ops_per_block']: 92 | self.assertEqual(counter[1], 1) 93 | 94 | def test_get_stream_ops_per_blocks_with_end(self): 95 | stream_data = list(self.chainsync.get_stream(['ops', 'ops_per_blocks'], start_block=3000000, end_block=3000003)) 96 | # Should retrieve 24 events (23 ops and 1 ops_per_blocks) 97 | self.assertEqual(len(stream_data), 24) 98 | self.assertEqual(len([d[1] for d in stream_data if d[0] == 'op']), 23) 99 | self.assertEqual(len([d[1] for d in stream_data if d[0] == 'ops_per_block']), 1) 100 | # Should only be returning the data types requested 101 | self.assertTrue([ 102 | d[0] for d in stream_data 103 | ][0] in ['op', 'ops_per_block']) 104 | # The ops should return the proper block number 105 | for op in [d[1] for d in stream_data if d[0] == 'op']: 106 | # Should be block 1 - 10 (default batch size is 10) 107 | self.assertTrue(op['block_num'] in [3000000, 3000001, 3000002, 3000003]) 108 | # The ops_per_block should return the number of ops in the block 109 | for counter in [d[1] for d in stream_data if d[0] == 'ops_per_block']: 110 | print(counter) 111 | for height in [3000000, 3000001, 3000002, 3000003]: 112 | self.assertTrue(counter[height] in [3, 15, 4, 1]) 113 | 114 | def test_get_stream_all_events(self): 115 | stream_data = list(self.chainsync.get_stream(start_block=3000000, batch_size=3)) 116 | types_expected = ['config', 'block', 'op', 'ops_per_block'] 117 | types_returned = [d[0] for d in stream_data] 118 | self.assertTrue(all([data_type in types_returned for data_type in types_expected])) 119 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_transaction.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncGetTransactionTestCase(ChainSyncBaseTestCase): 8 | 9 | def setUp(self): 10 | adapter = SteemAdapter( 11 | endpoints='https://steemd.pevo.science', 12 | retry=False 13 | ) 14 | self.chainsync = ChainSync(adapter=adapter, retry=False) 15 | 16 | def test_get_transaction(self): 17 | tx_id = 'c68435a34a7afc701771eb090f96526ed4c2a37b' 18 | result = self.chainsync.get_transaction(tx_id) 19 | self.assertEqual(result['block_num'], 20905025) 20 | 21 | def test_get_transaction_exception_no_transaction_id(self): 22 | with self.assertRaises(TypeError) as context: 23 | self.chainsync.get_transaction() 24 | 25 | def test_get_transaction_exception_invalid_transaction_id(self): 26 | with self.assertRaises(Exception) as context: 27 | self.chainsync.get_transaction('0000000000000000000000000000000000000000') 28 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_get_transactions.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncGetTransactionsTestCase(ChainSyncBaseTestCase): 8 | 9 | def setUp(self): 10 | adapter = SteemAdapter( 11 | endpoints='https://steemd.pevo.science', 12 | retry=False 13 | ) 14 | self.chainsync = ChainSync(adapter=adapter, retry=False) 15 | 16 | def test_get_transactions(self): 17 | txs = [ 18 | 'a3815d4a17f1331481ec6bf89ba0844ce16175bc', 19 | 'c68435a34a7afc701771eb090f96526ed4c2a37b', 20 | ] 21 | result = self.chainsync.get_transactions(txs) 22 | for tx in result: 23 | self.assertTrue(tx['transaction_id'] in txs) 24 | 25 | def test_get_transactions_exception_no_transaction_id(self): 26 | with self.assertRaises(TypeError) as context: 27 | self.chainsync.get_transactions() 28 | 29 | def test_get_transactions_exception_invalid_transaction_id(self): 30 | with self.assertRaises(Exception) as context: 31 | txs = [ 32 | 'a3815d4a17f1331481ec6bf89ba0844ce16175bc', 33 | '0000000000000000000000000000000000000000', 34 | ] 35 | result = self.chainsync.get_transactions(txs) 36 | for tx in result: 37 | self.assertTrue(tx['transaction_id'] in txs) 38 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_op_formatters.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from chainsync import ChainSync 4 | from chainsync.adapters.steem import SteemAdapter 5 | 6 | 7 | class ChainSyncOpDataFormatTestCase(ChainSyncBaseTestCase): 8 | 9 | # Tests data format returned by `get_block` 10 | def test_format_op_from_get_block(self): 11 | block_num = 10000000 12 | for block in self.chainsync.get_block_sequence(block_num, limit=1): 13 | for op in self.chainsync.from_block_get_ops(block): 14 | self.assertEqual(op['block_num'], block_num) 15 | self.assertTrue(op['op_in_trx'] >= 0) 16 | self.assertTrue(op['trx_in_block'] >= 0) 17 | self.assertTrue('operation_type' in op and op['operation_type'] is not False) 18 | self.assertTrue('transaction_id' in op and op['transaction_id'] is not False) 19 | self.assertTrue('timestamp' in op and op['timestamp'] is not False) 20 | 21 | # Tests data format returned by `get_ops_in_block` 22 | def test_format_op_from_get_ops_in_block(self): 23 | block_num = 10000000 24 | for op in self.chainsync.get_ops_in_block(block_num): 25 | self.assertEqual(op['block_num'], block_num) 26 | self.assertTrue(op['op_in_trx'] >= 0) 27 | self.assertTrue(op['trx_in_block'] >= 0) 28 | self.assertTrue('operation_type' in op and op['operation_type'] is not False) 29 | self.assertTrue('transaction_id' in op and op['transaction_id'] is not False) 30 | self.assertTrue('timestamp' in op and op['timestamp'] is not False) 31 | 32 | # Tests data format returned by `get_transaction` 33 | def test_txopdata_format(self): 34 | # Specify custom node since api.steemit.com doesn't have these endpoints enabled 35 | adapter = SteemAdapter( 36 | endpoints='https://steemd.pevo.science', 37 | retry=False 38 | ) 39 | chainsync = ChainSync(adapter=adapter, retry=False) 40 | # Load a transaction by ID 41 | transaction = chainsync.get_transaction('04008551461adb9a48c5c9eac8be6f63f9c840d9') 42 | for opIndex, op in enumerate(transaction['operations']): 43 | formatted = adapter.format_op_from_get_transaction(transaction, op, opIndex=opIndex) 44 | self.assertEqual(formatted['block_num'], 4042653) 45 | self.assertTrue(formatted['op_in_trx'] >= 0) 46 | self.assertTrue(formatted['trx_in_block'] >= 0) 47 | self.assertTrue('operation_type' in formatted and formatted['operation_type'] is not False) 48 | self.assertTrue('transaction_id' in formatted and formatted['transaction_id'] is not False) 49 | # self.assertTrue('timestamp' in formatted and formatted['timestamp'] is not False) # NYI - https://github.com/steemit/steem/issues/2310 50 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_stream.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | 4 | class ChainSyncStreamTestCase(ChainSyncBaseTestCase): 5 | 6 | def test_stream_blocks(self): 7 | for dataType, block in self.chainsync.stream(['blocks']): 8 | self.assertEqual(dataType, 'block') 9 | self.assertTrue(block['block_num'] > 1) 10 | break # Kill the generator loop 11 | 12 | def test_stream_blocks_from_irreversible(self): 13 | for dataType, block in self.chainsync.stream(['blocks'], mode='irreversible'): 14 | self.assertEqual(dataType, 'block') 15 | self.assertTrue(block['block_num'] > 1) 16 | break # Kill the generator loop 17 | 18 | def test_stream_blocks_from_block_height(self): 19 | for dataType, block in self.chainsync.stream(['blocks'], start_block=10): 20 | self.assertEqual(dataType, 'block') 21 | self.assertEqual(block['block_num'], 10) 22 | break # Kill the generator loop 23 | 24 | def test_stream_blocks(self): 25 | for dataType, block in self.chainsync.stream(['blocks']): 26 | self.assertEqual(dataType, 'block') 27 | self.assertTrue(block['block_num'] > 1) 28 | break # Kill the generator loop 29 | 30 | def test_stream_ops(self): 31 | for dataType, block in self.chainsync.stream(['ops']): 32 | self.assertEqual(dataType, 'op') 33 | self.assertTrue(block['block_num'] > 1) 34 | break # Kill the generator loop 35 | 36 | def test_stream_ops_from_block_height(self): 37 | for dataType, block in self.chainsync.stream(['ops'], start_block=3000000): 38 | self.assertEqual(dataType, 'op') 39 | self.assertEqual(block['block_num'], 3000000) 40 | break # Kill the generator loop 41 | 42 | def test_stream_ops_per_block(self): 43 | start_block = 3000000 44 | for dataType, ops_per_block in self.chainsync.stream(['ops_per_blocks'], start_block=start_block, batch_size=1): 45 | self.assertEqual(dataType, 'ops_per_block') 46 | self.assertTrue(start_block in ops_per_block) 47 | self.assertEqual(ops_per_block[start_block], 3) 48 | break # Kill the generator loop 49 | 50 | def test_stream_config(self): 51 | start_block = 3000000 52 | for dataType, config in self.chainsync.stream(['config'], start_block=start_block, batch_size=1): 53 | print(config) 54 | self.assertEqual(dataType, 'config') 55 | self.assertTrue('IS_TEST_NET' in config) 56 | break # Kill the generator loop 57 | 58 | def test_stream_status(self): 59 | start_block = 3000000 60 | for dataType, status in self.chainsync.stream(['status'], start_block=start_block, batch_size=1): 61 | print(status) 62 | self.assertEqual(dataType, 'status') 63 | self.assertTrue('head_block_number' in status) 64 | break # Kill the generator loop 65 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_yield_event.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from datetime import datetime 4 | from abc import ABC, abstractmethod 5 | 6 | from chainsync import ChainSync 7 | from chainsync.adapters.steem import SteemAdapter 8 | 9 | class ChainSyncYieldEventTestCase(ChainSyncBaseTestCase): 10 | 11 | def test_yield_event(self): 12 | what = 'foo' 13 | data = 'bar' 14 | tupled = self.chainsync.yield_event(what, data) 15 | result1, result2 = tupled 16 | self.assertEqual(result1, what) 17 | self.assertEqual(result2, data) 18 | -------------------------------------------------------------------------------- /tests/chainsync/test_chainsync_yield_event_with_plugins.py: -------------------------------------------------------------------------------- 1 | from test_base import ChainSyncBaseTestCase 2 | 3 | from datetime import datetime 4 | from abc import ABC, abstractmethod 5 | 6 | from chainsync import ChainSync 7 | from chainsync.adapters.steem import SteemAdapter 8 | 9 | class ChainSyncYieldEventTestCase(ChainSyncBaseTestCase): 10 | 11 | def test_yield_event_plugin(self): 12 | # configure with adapter + plugin 13 | chainsync = ChainSync(adapter=SteemAdapter(), plugins=[SamplePlugin1()]) 14 | # sample 'what' and 'data' for the event 15 | what = 'block' 16 | data = { 17 | 'timestamp': '2018-02-19T23:54:09' 18 | } 19 | # yield the data in an event 20 | tupled = chainsync.yield_event(what, data) 21 | eventType, eventData = tupled 22 | # they should still match 23 | self.assertEqual(eventType, what) 24 | self.assertEqual(eventData, data) 25 | # ensure a datetime is returned 26 | self.assertIsInstance(eventData['timestamp'], datetime) 27 | 28 | def test_yield_event_plugin(self): 29 | # configure with adapter + plugin 30 | chainsync = ChainSync(adapter=SteemAdapter(), plugins=[SamplePlugin1(), SamplePlugin2()]) 31 | # sample 'what' and 'data' for the event 32 | what = 'block' 33 | data = { 34 | 'string': 'convert to upper', 35 | 'timestamp': '2018-02-19T23:54:09' 36 | } 37 | # yield the data in an event 38 | tupled = chainsync.yield_event(what, data) 39 | eventType, eventData = tupled 40 | # they should still match 41 | self.assertEqual(eventType, what) 42 | # ensure a datetime is returned (via SamplePlugin1) 43 | self.assertIsInstance(eventData['timestamp'], datetime) 44 | # ensure the string is now upper (via SamplePlugin2) 45 | self.assertEqual(eventData['string'], data['string'].upper()) 46 | 47 | class AbstractPlugin(ABC): 48 | 49 | @abstractmethod 50 | def block(self, data): 51 | """ a method name matching the 'what' value and transforms 52 | the data as needed. 53 | """ 54 | pass 55 | 56 | class SamplePlugin1(AbstractPlugin): 57 | 58 | """ a plugin that will convert a timestamp string into a datetime 59 | """ 60 | def block(self, data): 61 | ts = data['timestamp'] 62 | data['timestamp'] = datetime(int(ts[:4]), int(ts[5:7]), int(ts[8:10]), int(ts[11:13]), int(ts[14:16]), int(ts[17:19])) 63 | return data 64 | 65 | class SamplePlugin2(AbstractPlugin): 66 | 67 | """ a plugin that will upper() a single field 68 | """ 69 | def block(self, data): 70 | data['string'] = data['string'].upper() 71 | return data 72 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from chainsync.adapters.steem import SteemAdapter 4 | from chainsync.adapters.steemv2 import SteemV2Adapter 5 | 6 | 7 | def pytest_addoption(parser): 8 | parser.addoption("--adapter", action="store", default="steem", help="steem, steemv2, etc") 9 | parser.addoption("--endpoint", action="store", default="https://api.steemit.com", help="http://localhost:8090, https://public-endpoint.com") 10 | 11 | 12 | @pytest.fixture(scope="class") 13 | def client(request): 14 | adapter = request.config.getoption("--adapter") 15 | endpoint = request.config.getoption("--endpoint") 16 | test_data = { 17 | 'block_with_ops': 1093, 18 | 'blocks_with_ops': 1094, 19 | } 20 | 21 | if adapter == 'steem_appbase': 22 | request.cls.adapter = SteemV2Adapter(endpoints=[endpoint]) 23 | else: 24 | request.cls.adapter = SteemAdapter(endpoints=[endpoint]) 25 | 26 | request.cls.test_data = test_data 27 | -------------------------------------------------------------------------------- /tests/test_import.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sys 4 | sys.path.insert(0, os.path.abspath('..')) 5 | 6 | from chainsync import * # noqa 7 | from chainsync.adapters.steem import * # noqa 8 | 9 | 10 | # pylint: disable=unused-import,unused-variable 11 | def test_import(): 12 | _ = ChainSync(adapter=SteemAdapter) 13 | _ = SteemAdapter() 14 | --------------------------------------------------------------------------------