├── coinmetrics ├── __init__.py ├── utils │ ├── __init__.py │ ├── event.py │ ├── arguments.py │ ├── execution.py │ ├── postgres.py │ ├── timeutil.py │ ├── jsonrpc.py │ ├── file.py │ ├── eta.py │ ├── bech32.py │ └── pipelines.py ├── bitsql │ ├── omni │ │ ├── aggregator.py │ │ ├── exporter.py │ │ ├── __init__.py │ │ ├── query.py │ │ ├── node.py │ │ └── schema.py │ ├── constants.py │ ├── applications │ │ ├── export.py │ │ ├── dbcontrol.py │ │ └── metricmaker.py │ ├── data.py │ ├── exporter.py │ ├── aggregator.py │ ├── __init__.py │ ├── schema.py │ ├── node.py │ └── query.py └── applications │ └── utxo_csvmaker.py ├── requirements.txt ├── .gitignore ├── LICENSE └── README.md /coinmetrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /coinmetrics/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.18.4 2 | psycopg2==2.7.5 3 | python-dateutil==2.7.5 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sublime-project 3 | *.sublime-workspace 4 | dev.py 5 | .mypy_cache/* 6 | *.csv 7 | venv/ 8 | .idea/ 9 | -------------------------------------------------------------------------------- /coinmetrics/utils/event.py: -------------------------------------------------------------------------------- 1 | class EventEmitter(object): 2 | 3 | def __init__(self): 4 | self.subscribers = [] 5 | 6 | def subscribe(self, subscriber): 7 | self.subscribers.append(subscriber) 8 | 9 | def unsubscribe(self, subscriber): 10 | self.subscribers.remove(subscriber) 11 | 12 | def trigger(self, *args): 13 | for subscriber in self.subscribers: 14 | subscriber(*args) 15 | -------------------------------------------------------------------------------- /coinmetrics/utils/arguments.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, Dict 2 | 3 | 4 | def postgres_connection_argument(value: str) -> Tuple[str, int, str, str, str]: 5 | pieces = value.split(":") 6 | if len(pieces) != 5: 7 | raise ValueError() 8 | return (pieces[0], int(pieces[1]), pieces[2], pieces[3], pieces[4]) 9 | 10 | 11 | def bitcoin_node_connection_argument(value: str) -> Tuple[str, int, str, str]: 12 | pieces = value.split(":") 13 | if len(pieces) != 4: 14 | raise ValueError() 15 | return (pieces[0], int(pieces[1]), pieces[2], pieces[3]) 16 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/aggregator.py: -------------------------------------------------------------------------------- 1 | from coinmetrics.bitsql.aggregator import * 2 | 3 | 4 | class OmniManagedPropertyAggregator(DailyAggregator): 5 | 6 | def createDailyMetrics(self): 7 | self.addMetric(DailyTxCountStatistic(self.dbAccess, self.query)) 8 | self.addMetric(DailyTxVolumeStatistic(self.dbAccess, self.query)) 9 | self.addMetric(DailyMedianTransactionValueStatistic(self.dbAccess, self.query)) 10 | self.addMetric(DailyActiveAddressesStatistic(self.dbAccess, self.query)) 11 | self.addMetric(DailyRewardStatistic(self.dbAccess, self.query)) 12 | 13 | 14 | class OmniCrowdsalePropertyAggregator(DailyAggregator): 15 | 16 | def createDailyMetrics(self): 17 | self.addMetric(DailyTxCountStatistic(self.dbAccess, self.query)) 18 | self.addMetric(DailyTxVolumeStatistic(self.dbAccess, self.query)) 19 | self.addMetric(DailyMedianTransactionValueStatistic(self.dbAccess, self.query)) 20 | self.addMetric(DailyActiveAddressesStatistic(self.dbAccess, self.query)) 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Coinmetrics.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/constants.py: -------------------------------------------------------------------------------- 1 | HASH_PRECISION = 78 2 | CHAINWORK_PRECISION = 48 3 | OUTPUT_VALUE_PRECISION = 32 4 | MAX_ADDRESS_LENGTH = 64 5 | OUTPUT_TYPES = { 6 | "nulldata": 0, 7 | "nonstandard": 1, 8 | "pubkeyhash": 2, 9 | "pubkey": 3, 10 | "multisig": 4, 11 | "scripthash": 5, 12 | "witness_v0_keyhash": 6, 13 | "witness_v0_scripthash": 7, 14 | # decred 15 | "stakegen": 64, 16 | "stakesubmission": 65, 17 | "sstxcommitment": 66, 18 | "sstxchange": 67, 19 | "stakerevoke": 68, 20 | "pubkeyalt": 69, 21 | } 22 | 23 | BLOCK_TIMES = { 24 | "btc": 600, 25 | "pivx": 60, 26 | "dash": 160, 27 | "ltc": 150, 28 | "vtc": 150, 29 | "xmr": 120, 30 | "etc": 15, 31 | "eth": 15, 32 | "zec": 150, 33 | "dgb": 15, 34 | "doge": 60, 35 | "btg": 600, 36 | "bch": 600, 37 | "xvg": 30, 38 | "dcr": 150, 39 | "omnilayer": 600, 40 | "bsv": 600, 41 | "btcp": 150, 42 | } 43 | 44 | SUPPORTED_ASSETS = ["btc", "ltc", "vtc", "dash", "doge", "zec", "dgb", "xvg", "pivx", "dcr", "bch", 45 | "btg", "omnilayer", "usdt", "maid", "bsv", "btcp"] 46 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/applications/export.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | from coinmetrics.bitsql import runExport, dbObjectsFactory, postgresFactory 4 | from coinmetrics.bitsql.constants import SUPPORTED_ASSETS 5 | from coinmetrics.utils.arguments import postgres_connection_argument, bitcoin_node_connection_argument 6 | 7 | 8 | argParser = argparse.ArgumentParser() 9 | argParser.add_argument("asset", type=str, choices=SUPPORTED_ASSETS) 10 | argParser.add_argument("database", type=postgres_connection_argument, help="Database parameters dbHost:dbPort:dbName:dbUser:dbPassword") 11 | argParser.add_argument("nodes", type=bitcoin_node_connection_argument, nargs="+", 12 | help="Node parameters host:port:rpcUser:rpcPassword") 13 | argParser.add_argument("--rpcthreads", type=int, default=8, help="Maximum amount of simultaneous RPC requests") 14 | argParser.add_argument("--loop", action="store_true", help="run export continously, in a loop") 15 | args = argParser.parse_args() 16 | 17 | 18 | logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') 19 | appLog = logging.getLogger("bitsql:{0}".format(args.asset)) 20 | appLog.setLevel(logging.DEBUG) 21 | 22 | runExport(args.asset, args.nodes, args.database, appLog, loop=args.loop, rpcThreads=args.rpcthreads) 23 | -------------------------------------------------------------------------------- /coinmetrics/utils/execution.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | from typing import Callable, Union, Tuple, List, Any 4 | 5 | 6 | def executeWithRetries(proc: Callable, args: Union[Tuple, List], maxRetries: int, sleepTime: float=0): 7 | retries = 0 8 | while True: 9 | try: 10 | return proc(*args) 11 | except KeyboardInterrupt: 12 | raise 13 | except Exception as e: 14 | if retries >= maxRetries: 15 | raise e 16 | else: 17 | retries += 1 18 | if sleepTime > 0: 19 | time.sleep(sleepTime) 20 | 21 | 22 | def executeInParallel(procs: List[Callable], args: List[Union[Tuple, List]]) -> List[Tuple[Any, Exception]]: 23 | def threadWrap(index, proc, args): 24 | try: 25 | partialResult = proc(*args) 26 | exception = None 27 | except Exception as e: 28 | partialResult = None 29 | exception = e 30 | result[index] = (partialResult, exception) 31 | 32 | assert len(procs) == len(args) 33 | result = [] 34 | threads = [] 35 | for index in range(len(procs)): 36 | result.append(None) 37 | t = threading.Thread(target=threadWrap, args=(index, procs[index], args[index])) 38 | t.start() 39 | threads.append(t) 40 | 41 | for t in threads: 42 | t.join() 43 | 44 | return result 45 | 46 | 47 | def executeInParallelSameProc(proc: Callable, args: List[Union[Tuple, List]]) -> List[Tuple[Any, Exception]]: 48 | return executeInParallel([proc for _ in range(len(args))], args) 49 | -------------------------------------------------------------------------------- /coinmetrics/utils/postgres.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | import psycopg2.extras 3 | 4 | 5 | class PostgresAccess(object): 6 | 7 | def __init__(self, dbHost, dbPort, dbName, dbUser, dbPassword): 8 | self.connection = psycopg2.connect("host=%s port=%s dbname=%s user=%s password=%s" % (dbHost, dbPort, dbName, dbUser, dbPassword)) 9 | self.cursor = self.connection.cursor() 10 | 11 | def __del__(self): 12 | self.close() 13 | 14 | def queryNoReturnCommit(self, text, params=None): 15 | self.cursor.execute(text, params) 16 | self.connection.commit() 17 | 18 | def queryNoReturnNoCommit(self, text, params=None): 19 | self.cursor.execute(text, params) 20 | 21 | def queryReturnOne(self, text, params=None): 22 | self.cursor.execute(text, params) 23 | return self.cursor.fetchone() 24 | 25 | def queryReturnAll(self, text, params=None): 26 | self.cursor.execute(text, params) 27 | return self.cursor.fetchall() 28 | 29 | def executeValues(self, sql, rows, batchSize): 30 | return psycopg2.extras.execute_values(self.cursor, sql, rows, page_size=batchSize) 31 | 32 | def commit(self): 33 | self.connection.commit() 34 | 35 | def close(self): 36 | if self.cursor is not None: 37 | self.cursor.close() 38 | self.cursor = None 39 | if self.connection is not None: 40 | self.connection.close() 41 | self.connection = None 42 | 43 | def getTableNames(self): 44 | return [row[0] for row in self.queryReturnAll(""" 45 | SELECT 46 | table_name 47 | FROM 48 | information_schema.tables 49 | WHERE 50 | table_type = 'BASE TABLE' AND 51 | table_schema NOT IN ('pg_catalog', 'information_schema')""")] 52 | -------------------------------------------------------------------------------- /coinmetrics/utils/timeutil.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | from datetime import datetime, timedelta 3 | from typing import Any, List 4 | 5 | 6 | def datetimeToTimestamp(value: datetime) -> float: 7 | return (value - datetime(1970, 1, 1)).total_seconds() 8 | 9 | 10 | def datetimeFromMilliseconds(value: int) -> datetime: 11 | seconds = value // 1000 12 | milliseconds = value % 1000 13 | return datetime.utcfromtimestamp(seconds) + timedelta(milliseconds=milliseconds) 14 | 15 | 16 | def alignDateToInterval(dt: datetime, interval: timedelta) -> datetime: 17 | intervalCount = datetimeToIntervalCount(dt, interval) 18 | return datetime.utcfromtimestamp(intervalCount * int(interval.total_seconds())) 19 | 20 | 21 | def datetimeToIntervalCount(dt: datetime, interval: timedelta) -> int: 22 | dateSeconds = int(datetimeToTimestamp(dt)) 23 | intervalSeconds = int(interval.total_seconds()) 24 | return dateSeconds // intervalSeconds 25 | 26 | 27 | class Timeline(object): 28 | 29 | def __init__(self, maxAgeInSeconds: float): 30 | self._events = deque() 31 | self._maxAge = maxAgeInSeconds 32 | assert self._maxAge > 0, "maximum age must be positive" 33 | 34 | def add(self, value: Any): 35 | now = datetime.now() 36 | self._events.append((now, value)) 37 | while now - self._events[0][0] > self._maxAge: 38 | self._events.popleft() 39 | 40 | def getAllYoungerThan(self, ageInSeconds: float) -> List[Any]: 41 | result = [] 42 | now = datetime.now() 43 | maxAgeTimedelta = timedelta(seconds=ageInSeconds) 44 | for eventTime, eventValue in reversed(self._events): 45 | if now - eventTime <= maxAgeTimedelta: 46 | result.append(eventValue) 47 | else: 48 | break 49 | return result 50 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/applications/dbcontrol.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | from coinmetrics.bitsql import runExport, dbObjectsFactory, postgresFactory 4 | from coinmetrics.bitsql.constants import SUPPORTED_ASSETS 5 | from coinmetrics.utils.arguments import postgres_connection_argument 6 | 7 | 8 | argParser = argparse.ArgumentParser() 9 | argParser.add_argument("asset", type=str, choices=SUPPORTED_ASSETS) 10 | argParser.add_argument("database", type=postgres_connection_argument, help="Database parameters dbHost:dbPort:dbName:dbUser:dbPassword") 11 | argParser.add_argument("--drop-db", dest="dropDb", action="store_true", 12 | help="drop tables that contain data of the given asset") 13 | argParser.add_argument("--add-index", dest="addIndex", action="store_true", help="add indexes to tables") 14 | argParser.add_argument("--drop-index", dest="dropIndex", action="store_true", help="remove table indexes") 15 | argParser.add_argument("--vacuum", action="store_true", help="vacuum outputs table") 16 | args = argParser.parse_args() 17 | 18 | 19 | logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') 20 | appLog = logging.getLogger("bitsql:{0}".format(args.asset)) 21 | appLog.setLevel(logging.DEBUG) 22 | 23 | 24 | if args.dropDb: 25 | schema, _, _, _ = dbObjectsFactory(args.asset, postgresFactory(*args.database), appLog) 26 | schema.drop() 27 | elif args.addIndex: 28 | schema, _, _, _ = dbObjectsFactory(args.asset, postgresFactory(*args.database), appLog) 29 | schema.addIndexes() 30 | elif args.dropIndex: 31 | schema, _, _, _ = dbObjectsFactory(args.asset, postgresFactory(*args.database), appLog) 32 | schema.dropIndexes() 33 | elif args.vacuum: 34 | schema, _, _, _ = dbObjectsFactory(args.asset, postgresFactory(*args.database), appLog) 35 | schema.vacuum() 36 | else: 37 | print("no action chosen, exiting") -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-tools 2 | This repository contains source code for tools used by coinmetrics.io to collect data from Bitcoin and its clones and forks. Currently, we support BTC, BCH, LTC, DOGE, ZEC, DCR, PIVX, XVG, DASH, VTC, DGB, BTG, BSV and two assets based on Omni protocol: USDT and MAID. 3 | 4 | ## Prerequisites 5 | Python 3.6, PostgreSQL 9 or 10, Python modules `psycopg2`, `requests`, `python-dateutil`. 6 | 7 | ## Reproducing coinmetrics.io data 8 | We'll use LTC as an example, due to relatively small size of its blockchain. We presume that the tool, postgresql database and LTC node live on the same machine. 9 | 10 | * LTC node should be installed and synced with the network. It is essential to launch the node with `txindex=1` flag set. 11 | * Clone this repository and launch the following command from the root directory: ```python3 -m coinmetrics.bitsql.applications.export ltc localhost:db_port:db_name:db_user:db_password localhost:node_rpc_port:node_rpc_user:node_rpc_password```. This will export node data to PostgreSQL database and may take a while. 12 | * After initial export is completed, vacuum tables by running `python3 -m coinmetrics.bitsql.applications.dbcontrol ltc localhost:db_port:db_name:db_user:db_password --vacuum` and then create database indices by running `python3 -m coinmetrics.bitsql.applications.dbcontrol ltc localhost:db_port:db_name:db_user:db_password --add-index`. 13 | * Compute metrics and store them in PostgreSQL tables by running `python3 -m coinmetrics.bitsql.applications.metricmaker ltc localhost:db_port:db_name:db_user:db_password --save`. 14 | * Optionally, create CSV from metric tables: `python3 -m coinmetrics.applications.utxo_csvmaker ltc localhost:db_port:db_name:db_user:db_password`. 15 | * Produced CSV contains only on-chain data denominated in satoshis. CSVs available at coinmetrics.io can be obtained by combining on-chain and price data collected from, for instance, coinmarketcap.com. 16 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/applications/metricmaker.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import dateutil.parser 4 | import time 5 | from datetime import datetime 6 | from coinmetrics.bitsql import dbObjectsFactory, postgresFactory 7 | from coinmetrics.bitsql.constants import SUPPORTED_ASSETS 8 | from coinmetrics.utils.arguments import postgres_connection_argument 9 | 10 | 11 | argParser = argparse.ArgumentParser() 12 | argParser.add_argument("assets", type=str, nargs="+", choices=SUPPORTED_ASSETS) 13 | argParser.add_argument("database", type=postgres_connection_argument, help="Database parameters dbHost:dbPort:dbName:dbUser:dbPassword") 14 | argParser.add_argument("--startdate", type=dateutil.parser.parse, default=datetime(2009, 1, 1)) 15 | argParser.add_argument("--enddate", type=dateutil.parser.parse, default=datetime.utcnow()) 16 | argParser.add_argument("--save", action="store_true", default=False, help="If set, calculcated metrics will be stored in the database") 17 | argParser.add_argument("--force", action="store_true", default=False, help="If set, will trigger re-computation of metrics") 18 | argParser.add_argument("--drop", action="store_true", help="Drop calculated metrics from database") 19 | argParser.add_argument("--metrics", type=str, default=[], nargs="+", help="Metrics on which operation will act, all by default") 20 | argParser.add_argument("--excludemetrics", type=str, default=[], nargs="+", help="Metrics on which operation will not act, none by default") 21 | argParser.add_argument("--list", action="store_true", default=False, help="Will list metrics' names") 22 | argParser.add_argument("--loop", action="store_true", default=False, help="Continously calculate metrics") 23 | args = argParser.parse_args() 24 | 25 | 26 | logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') 27 | appLog = logging.getLogger("bitsql-metric:{0}".format(args.assets)) 28 | appLog.setLevel(logging.DEBUG) 29 | 30 | 31 | def proc(startDate, endDate, save, force): 32 | for asset in args.assets: 33 | db = postgresFactory(*args.database) 34 | _, _, _, aggregator = dbObjectsFactory(asset, db, appLog) 35 | 36 | metrics = aggregator.getMetricNames() if len(args.metrics) == 0 else args.metrics 37 | metrics = [metric for metric in metrics if len(args.excludemetrics) == 0 or metric not in args.excludemetrics] 38 | 39 | if args.drop: 40 | aggregator.drop(metrics) 41 | elif args.list: 42 | print(aggregator.getMetricNames()) 43 | else: 44 | aggregator.run(metrics, startDate, endDate, save, force) 45 | 46 | 47 | if args.loop: 48 | while True: 49 | proc(datetime(2009, 1, 1), datetime.utcnow(), True, False) 50 | appLog.info("going to sleep...") 51 | time.sleep(300.0) 52 | else: 53 | proc(args.startdate, args.enddate, args.save, args.force) 54 | -------------------------------------------------------------------------------- /coinmetrics/utils/jsonrpc.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import base64 4 | import time 5 | 6 | 7 | class RpcCallFailedException(Exception): 8 | pass 9 | 10 | 11 | class JsonRpcCaller(object): 12 | 13 | def __init__(self, host, port, user, password, queryPath="", tls=False, tlsVerify=True): 14 | self.host = host 15 | self.port = str(port) 16 | self.user = user 17 | self.password = password 18 | self.queryPath = queryPath 19 | self.tls = tls 20 | self.tlsVerify = tlsVerify 21 | 22 | def getAddress(self): 23 | return self.host + ":" + self.port 24 | 25 | def makeRpcCall(self, headers, payload): 26 | try: 27 | protocol = "https" if self.tls else "http" 28 | response = requests.post("{0}://{1}:{2}/{3}".format(protocol, self.host, self.port, self.queryPath), 29 | headers=headers, data=payload, verify=(self.tls and self.tlsVerify)) 30 | except Exception as e: 31 | raise RpcCallFailedException(e) 32 | 33 | if response.status_code != 200: 34 | raise RpcCallFailedException("Invalid status code: %s" % response.status_code) 35 | 36 | responseJson = response.json(parse_float=lambda f: f) 37 | if type(responseJson) != list: 38 | if "error" in responseJson and responseJson["error"] is not None: 39 | raise RpcCallFailedException("RPC call error: %s" % responseJson["error"]) 40 | else: 41 | return responseJson["result"] 42 | else: 43 | result = [] 44 | for subResult in responseJson: 45 | if "error" in subResult and subResult["error"] is not None: 46 | raise RpcCallFailedException("RPC call error: %s" % subResult["error"]) 47 | else: 48 | result.append(subResult["result"]) 49 | return result 50 | 51 | def call(self, method, params=None): 52 | if params is None: 53 | params = [] 54 | authString = str(base64.b64encode("{0}:{1}".format(self.user, self.password).encode()))[2:-1] 55 | headers = {'content-type': 'application/json', 'Authorization': 'Basic ' + (authString)} 56 | payload = json.dumps({"jsonrpc": "2.0", "id": "0", "method": method, "params": params}) 57 | return self.makeRpcCall(headers, payload) 58 | 59 | def bulkCall(self, methodParamsTuples): 60 | authString = str(base64.b64encode("{0}:{1}".format(self.user, self.password).encode()))[2:-1] 61 | headers = {'content-type': 'application/json', 'Authorization': 'Basic ' + (authString)} 62 | payload = json.dumps([{"jsonrpc": "2.0", "id": "0", "method": method, "params": params} 63 | for method, params in methodParamsTuples]) 64 | return self.makeRpcCall(headers, payload) 65 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/exporter.py: -------------------------------------------------------------------------------- 1 | class OmniExporter(object): 2 | 3 | def __init__(self, dbAccess, schema, log): 4 | self.dbAccess = dbAccess 5 | self.schema = schema 6 | self.log = log 7 | 8 | def pushBlock(self, blockData): 9 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getBlocksTableName() + "\ 10 | (block_hash, block_height, block_time) \ 11 | VALUES (%s, %s, %s)", (blockData.blockHash, blockData.blockHeight, blockData.blockTime)) 12 | 13 | self.txsInBlock = 0 14 | self.insertTransactions(blockData.simpleSendTransactions, self.schema.getSimpleSendTxTableName(), self.schema.getSimpleSendTxPrefix()) 15 | self.insertTransactions(blockData.sendOwnersTransactions, self.schema.getSendOwnersTxTableName(), self.schema.getSendOwnersTxPrefix()) 16 | self.insertTransactions(blockData.sellForBitcoinTransactions, self.schema.getSellForBitcoinTxTableName(), self.schema.getSellForBitcoinTxPrefix()) 17 | self.insertTransactions(blockData.acceptSellForBitcoinTransactions, self.schema.getAcceptSellForBitcoinTxTableName(), self.schema.getAcceptSellForBitcoinTxPrefix()) 18 | self.insertTransactions(blockData.dexPurchaseTransactions, self.schema.getDexPurchaseTxTableName(), self.schema.getDexPurchaseTxPrefix()) 19 | self.insertTransactions(blockData.createFixedPropertyTransactions, self.schema.getCreateFixedPropertyTxTableName(), self.schema.getCreateFixedPropertyTxPrefix()) 20 | self.insertTransactions(blockData.createCrowdsalePropertyTransactions, self.schema.getCreateCrowdsalePropertyTxTableName(), self.schema.getCreateCrowdsalePropertyTxPrefix()) 21 | self.insertTransactions(blockData.createManagedPropertyTransactions, self.schema.getCreateManagedPropertyTxTableName(), self.schema.getCreateManagedPropertyTxPrefix()) 22 | self.insertTransactions(blockData.closeCrowdsaleTransactions, self.schema.getCloseCrowdsaleTxTableName(), self.schema.getCloseCrowdsaleTxPrefix()) 23 | self.insertTransactions(blockData.grantTokensTransactions, self.schema.getGrantTokensTxTableName(), self.schema.getGrantTokensTxPrefix()) 24 | self.insertTransactions(blockData.revokeTokensTransactions, self.schema.getRevokeTokensTxTableName(), self.schema.getRevokeTokensTxPrefix()) 25 | self.insertTransactions(blockData.sendAllTransactions, self.schema.getSendAllTxTableName(), self.schema.getSendAllTxPrefix()) 26 | self.log.info("exported {0} txs".format(self.txsInBlock)) 27 | 28 | self.dbAccess.commit() 29 | 30 | def insertTransactions(self, txList, tableName, prefix): 31 | if len(txList) == 0: 32 | return 33 | 34 | self.txsInBlock += len(txList) 35 | 36 | keys, _ = txList[0].getAttributes(prefix) 37 | keysString = "(" + ", ".join([key for key in keys]) + ")" 38 | 39 | tuples = [] 40 | for tx in txList: 41 | _, values = tx.getAttributes(prefix) 42 | tuples.append(values) 43 | 44 | self.dbAccess.executeValues("INSERT INTO " + tableName + " " + keysString + " VALUES %s", tuples, 512) 45 | -------------------------------------------------------------------------------- /coinmetrics/utils/file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from typing import List 4 | from random import randint 5 | from datetime import datetime 6 | from coinmetrics.utils.timeutil import datetimeToTimestamp 7 | 8 | 9 | class AtomicallySwappableFile(object): 10 | 11 | def __init__(self, fileName: str, path: str): 12 | self._fileName = fileName 13 | self._path = path 14 | if self._path[-1] != "/": 15 | self._path += "/" 16 | 17 | self._currentVersion = self._initSymlink() 18 | self._cullOldVersions() 19 | 20 | def update(self, content: str): 21 | newVersion = self._createNewVersionFile() 22 | newVersionPath = self._getVersionPath(newVersion) 23 | with open(newVersionPath, "w") as f: 24 | f.write(content) 25 | os.fsync(f) 26 | 27 | tmpName = self._path + self._getNewVersionId() 28 | os.symlink(self._getVersionFileName(newVersion), tmpName) 29 | shutil.move(tmpName, self._path + self._fileName) 30 | 31 | self._currentVersion = newVersion 32 | self._cullOldVersions() 33 | 34 | def read(self) -> str: 35 | if self._currentVersion is not None: 36 | with open(self._getVersionPath(self._currentVersion), "r") as f: 37 | return f.read() 38 | else: 39 | return "" 40 | 41 | def _initSymlink(self) -> str: 42 | symlinkPath = self._path + self._fileName 43 | if not os.path.islink(symlinkPath): 44 | newVersionId = self._createNewVersionFile() 45 | os.symlink(self._getVersionFileName(newVersionId), symlinkPath) 46 | return newVersionId 47 | else: 48 | versions = self._collectVersions() 49 | try: 50 | currentVersion = os.readlink(symlinkPath).split("/")[-1][len(self._fileName) + 1:] 51 | if currentVersion not in versions: 52 | raise Exception("file {0} is corrupted: symlink points to non-existent version".format(symlinkPath)) 53 | return currentVersion 54 | except Exception as e: 55 | return None 56 | 57 | def _collectVersions(self) -> List[str]: 58 | result = [] 59 | files = [f for f in os.listdir(self._path) if os.path.isfile(os.path.join(self._path, f))] 60 | for f in files: 61 | pieces = f.split(".") 62 | if ".".join(pieces[0:-1]) == self._fileName: 63 | result.append(pieces[-1]) 64 | return result 65 | 66 | def _createNewVersionFile(self) -> str: 67 | versionId = self._getNewVersionId() 68 | with open(self._getVersionPath(versionId), "w") as f: 69 | f.write("\n") 70 | return versionId 71 | 72 | def _getVersionPath(self, version: str) -> str: 73 | return self._path + self._fileName + "." + version 74 | 75 | def _getVersionFileName(self, version: str) -> str: 76 | return self._fileName + "." + version 77 | 78 | def _cullOldVersions(self): 79 | versions = self._collectVersions() 80 | for version in versions: 81 | if version != self._currentVersion: 82 | os.remove(self._getVersionPath(version)) 83 | 84 | def _getNewVersionId(self) -> str: 85 | return str(int(datetimeToTimestamp(datetime.utcnow()) * 1000)) + "_" + str(randint(1, 2**64)) 86 | 87 | def _getTimeFromVersionId(self, version: str) -> datetime: 88 | return datetime.utcfromtimestamp(version.split("_")[0]) 89 | -------------------------------------------------------------------------------- /coinmetrics/applications/utxo_csvmaker.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | import argparse 4 | import logging 5 | import dateutil.parser 6 | from datetime import datetime 7 | from collections import defaultdict 8 | from coinmetrics.bitsql import dbObjectsFactory, postgresFactory 9 | from coinmetrics.bitsql.constants import SUPPORTED_ASSETS 10 | from coinmetrics.utils.arguments import postgres_connection_argument 11 | from coinmetrics.utils.file import AtomicallySwappableFile 12 | 13 | 14 | argParser = argparse.ArgumentParser() 15 | argParser.add_argument("assets", type=str, nargs="+", choices=SUPPORTED_ASSETS) 16 | argParser.add_argument("database", type=postgres_connection_argument, help="Database parameters dbHost:dbPort:dbName:dbUser:dbPassword") 17 | argParser.add_argument("--directory", default="./", help="directory for generated file") 18 | argParser.add_argument("--metrics", type=str, default=[], nargs="+", help="Metrics on which operation will act, all by default") 19 | argParser.add_argument("--excludemetrics", type=str, default=[], nargs="+", help="Metrics which will not be included into CSV, none by default") 20 | argParser.add_argument("--loop", action="store_true", default=False, help="Continously rebuild CSV file from metrics") 21 | args = argParser.parse_args() 22 | 23 | 24 | directory = args.directory 25 | if directory[-1] != "/": 26 | directory += "/" 27 | 28 | 29 | logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') 30 | appLog = logging.getLogger("bitsql-gen:{0}".format(args.assets)) 31 | appLog.setLevel(logging.DEBUG) 32 | 33 | 34 | def proc(): 35 | db = postgresFactory(*args.database) 36 | tableNames = sorted(db.getTableNames()) 37 | 38 | for asset in args.assets: 39 | pattern = re.compile("^statistic_([a-z_]+)_{0}$".format(asset)) 40 | dates = defaultdict(dict) 41 | metricNames = set() 42 | 43 | for tableName in tableNames: 44 | match = pattern.match(tableName) 45 | if match is not None and match.group(1) not in args.excludemetrics and (len(args.metrics) == 0 or match.group(1) in args.metrics): 46 | metricName = match.group(1) 47 | metricNames.add(metricName) 48 | dateValues = db.queryReturnAll("""SELECT date, value FROM {0}""".format(tableName)) 49 | for date, value in dateValues: 50 | dates[date][metricName] = value 51 | 52 | datesList = sorted([(date, metrics) for date, metrics in dates.items()], key=lambda pair: pair[0]) 53 | 54 | csvContent = [] 55 | metricList = sorted(list(metricNames) if len(args.metrics) == 0 else args.metrics) 56 | header = ",".join(["date"] + metricList) 57 | csvContent.append(header + "\n") 58 | for date, metrics in datesList: 59 | csvContent.append(date.strftime('%Y-%m-%d') + ",") 60 | values = [] 61 | for metric in metricList: 62 | if metric in metrics: 63 | metricValue = metrics[metric] 64 | metricValueString = str(metricValue) if metricValue is not None else "" 65 | values.append(metricValueString) 66 | else: 67 | values.append("") 68 | csvContent.append(",".join(values) + "\n") 69 | 70 | AtomicallySwappableFile("{0}.csv".format(asset), directory).update("".join(csvContent)) 71 | 72 | 73 | if args.loop: 74 | while True: 75 | proc() 76 | appLog.info("going to sleep...") 77 | time.sleep(300.0) 78 | else: 79 | proc() 80 | -------------------------------------------------------------------------------- /coinmetrics/utils/eta.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from logging import Logger 3 | from collections import deque 4 | 5 | 6 | class _ObservationHistory(object): 7 | 8 | def __init__(self, maxObservations: int): 9 | self._maxObservations = maxObservations 10 | self._observations = deque() 11 | 12 | def addObservation(self, timeSpent: float, workDone: int): 13 | self._observations.append((timeSpent, workDone)) 14 | if len(self._observations) > self._maxObservations: 15 | self._observations.popleft() 16 | 17 | def getAverage(self) -> float: 18 | totalWork = 0 19 | totalTime = 0 20 | for timeSpent, workDone in self._observations: 21 | totalTime += timeSpent 22 | totalWork += workDone 23 | return totalTime / totalWork 24 | 25 | 26 | class ETA(object): 27 | 28 | def __init__(self, 29 | log: Logger, 30 | totalWorkAmount: int, 31 | maxObservations: int, 32 | outputInterval: int, 33 | printPrefix: str=""): 34 | assert totalWorkAmount > 0, "total work amount should be positive" 35 | self._log = log 36 | self._totalWorkAmount = totalWorkAmount 37 | self._printPrefix = printPrefix 38 | if len(self._printPrefix) > 0: 39 | self._printPrefix += " " 40 | self._workDone = 0 41 | self._history = _ObservationHistory(maxObservations) 42 | self._workStartedFlag = False 43 | self._workStartedTime = datetime(1970, 1, 1) 44 | self._outputInterval = outputInterval 45 | self._workSteps = 0 46 | 47 | def workStarted(self): 48 | assert not self._workStartedFlag, "work shouldn't have been already started" 49 | self._workStartedFlag = True 50 | self._workStartedTime = datetime.now() 51 | 52 | def workFinished(self, workDone: int): 53 | assert self._workStartedFlag, "work should've been started" 54 | self._workStartedFlag = False 55 | self._updateWork(workDone, datetime.now() - self._workStartedTime) 56 | 57 | def _updateWork(self, workDone: int, timeSpent: timedelta) -> bool: 58 | self._history.addObservation(timeSpent.total_seconds(), workDone) 59 | self._workDone += workDone 60 | self._workSteps += 1 61 | 62 | if self._workSteps % self._outputInterval == 0: 63 | self.output() 64 | return True 65 | else: 66 | return False 67 | 68 | def getETA(self) -> float: 69 | return (self._totalWorkAmount - self._workDone) * self._history.getAverage() 70 | 71 | def getPercentDone(self) -> float: 72 | return float(self._workDone) / self._totalWorkAmount 73 | 74 | def output(self): 75 | self._log.info("{0}{1:0.2f}% done, eta is {2}, speed is {3:0.4f}/s".format( 76 | self._printPrefix, 77 | 100.0 * self.getPercentDone(), 78 | self.prettyStringForETA(self.getETA()), 79 | 1.0 / self._history.getAverage())) 80 | 81 | def prettyStringForETA(self, etaInSeconds: float) -> str: 82 | totalSeconds = int(etaInSeconds) 83 | 84 | daysModulo = totalSeconds % (3600 * 24) 85 | days = (totalSeconds - daysModulo) // (3600 * 24) 86 | totalSeconds -= days * (3600 * 24) 87 | 88 | hoursModulo = totalSeconds % 3600 89 | hours = (totalSeconds - hoursModulo) // 3600 90 | totalSeconds -= hours * 3600 91 | 92 | minutesModulo = totalSeconds % 60 93 | minutes = (totalSeconds - minutesModulo) // 60 94 | seconds = totalSeconds - minutes * 60 95 | 96 | return "{0} days {1} hours {2} minutes {3} seconds".format(days, hours, minutes, seconds) 97 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/data.py: -------------------------------------------------------------------------------- 1 | class BitcoinBlockData(object): 2 | 3 | def __init__(self, height, hashAsNumber, chainworkAsNumber, blockTime, blockMedianTime, blockSize, difficulty): 4 | self.transactions = [] 5 | self.blockHeight = height 6 | self.hashAsNumber = hashAsNumber 7 | self.chainworkAsNumber = chainworkAsNumber 8 | self.blockTime = blockTime 9 | self.blockMedianTime = blockMedianTime 10 | self.blockSize = blockSize 11 | self.difficulty = difficulty 12 | 13 | def addTransaction(self, transaction): 14 | self.transactions.append(transaction) 15 | 16 | def getTransactions(self): 17 | return self.transactions 18 | 19 | def __repr__(self): 20 | header = "block height: %d, time: %s, tx count %d" % (self.blockHeight, self.blockTime, len(self.transactions)) 21 | transactions = [] 22 | for tx in self.transactions: 23 | transactions.append(str(tx)) 24 | return "\n".join([header] + transactions) 25 | 26 | 27 | class BitcoinTransactionData(object): 28 | 29 | def __init__(self, txHash, txSize, txTime, txMedianTime, coinbase): 30 | self.txHash = txHash 31 | self.txSize = txSize 32 | self.txTime = txTime 33 | self.txMedianTime = txMedianTime 34 | self.coinbase = coinbase 35 | self.coinbaseScript = None 36 | self.inputs = [] 37 | self.outputs = [] 38 | 39 | def getInputs(self): 40 | return self.inputs 41 | 42 | def getOutputs(self): 43 | return self.outputs 44 | 45 | def addInput(self, inputData): 46 | self.inputs.append(inputData) 47 | 48 | def addOutput(self, outputData): 49 | self.outputs.append(outputData) 50 | 51 | def setCoinbaseScript(self, coinbaseScript): 52 | self.coinbaseScript = coinbaseScript 53 | 54 | def getCoinbaseScript(self): 55 | return self.coinbaseScript 56 | 57 | def getHashString(self): 58 | result = hex(self.txHash)[2:] 59 | while len(result) < 64: 60 | result = "0" + result 61 | return result 62 | 63 | def __repr__(self): 64 | header = "tx %s %s\n" % (self.txHash, "coinbase" if self.coinbase else "") 65 | inputsRepr = "\n".join(["inputs:"] + [str(i) for i in self.inputs]) 66 | outputsRepr = "\n".join(["\noutputs:"] + [str(o) for o in self.outputs]) 67 | return header + inputsRepr + outputsRepr 68 | 69 | 70 | class ZcashTransactionData(BitcoinTransactionData): 71 | 72 | def __init__(self, txHash, txSize, txTime, txMedianTime, coinbase): 73 | super(ZcashTransactionData, self).__init__(txHash, txSize, txTime, txMedianTime, coinbase) 74 | self.joinSplits = [] 75 | self.saplingPayments = [] 76 | 77 | def getJoinSplits(self): 78 | return self.joinSplits 79 | 80 | def getSaplingPayments(self): 81 | return self.saplingPayments 82 | 83 | def addJoinSplit(self, joinSplitData): 84 | self.joinSplits.append(joinSplitData) 85 | 86 | def addSaplingPayment(self, saplingPayment): 87 | self.saplingPayments.append(saplingPayment) 88 | 89 | 90 | class PivxTransactionData(BitcoinTransactionData): 91 | 92 | def __init__(self, txHash, txSize, txTime, txMedianTime, coinbase): 93 | super(PivxTransactionData, self).__init__(txHash, txSize, txTime, txMedianTime, coinbase) 94 | self.zerocoinMints = [] 95 | self.zerocoinSpends = [] 96 | 97 | def getZerocoinMints(self): 98 | return self.zerocoinMints 99 | 100 | def getZerocoinSpends(self): 101 | return self.zerocoinSpends 102 | 103 | def addZerocoinMint(self, mintedValue): 104 | self.zerocoinMints.append(mintedValue) 105 | 106 | def addZerocoinSpend(self, spentValue): 107 | self.zerocoinSpends.append(spentValue) 108 | 109 | def __repr__(self): 110 | base = super(PivxTransactionData, self).__repr__() 111 | mintsRepr = "\n".join(["\nzc mints:"] + [str(i) for i in self.zerocoinMints]) 112 | spendsRepr = "\n".join(["\nzc spends:"] + [str(o) for o in self.zerocoinSpends]) 113 | return base + mintsRepr + spendsRepr 114 | 115 | 116 | class DecredTransactionData(BitcoinTransactionData): 117 | 118 | def __init__(self, txHash, txSize, txTime, txMedianTime, coinbase): 119 | super(DecredTransactionData, self).__init__(txHash, txSize, txTime, txMedianTime, coinbase) 120 | self.vote = False 121 | self.ticket = False 122 | 123 | def __repr__(self): 124 | base = super(DecredTransactionData, self).__repr__() 125 | if self.vote: 126 | base += "\nvote" 127 | if self.ticket: 128 | base += "\nticket" 129 | return base 130 | -------------------------------------------------------------------------------- /coinmetrics/utils/bech32.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Pieter Wuille 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | # https://github.com/sipa/bech32/blob/master/ref/python/segwit_addr.py 22 | 23 | """Reference implementation for Bech32 and segwit addresses.""" 24 | 25 | import binascii 26 | 27 | CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" 28 | 29 | 30 | def bech32_polymod(values): 31 | """Internal function that computes the Bech32 checksum.""" 32 | generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] 33 | chk = 1 34 | for value in values: 35 | top = chk >> 25 36 | chk = (chk & 0x1ffffff) << 5 ^ value 37 | for i in range(5): 38 | chk ^= generator[i] if ((top >> i) & 1) else 0 39 | return chk 40 | 41 | 42 | def bech32_hrp_expand(hrp): 43 | """Expand the HRP into values for checksum computation.""" 44 | return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp] 45 | 46 | 47 | def bech32_verify_checksum(hrp, data): 48 | """Verify a checksum given HRP and converted data characters.""" 49 | return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1 50 | 51 | 52 | def bech32_create_checksum(hrp, data): 53 | """Compute the checksum values given HRP and data.""" 54 | values = bech32_hrp_expand(hrp) + data 55 | polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1 56 | return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] 57 | 58 | 59 | def bech32_encode(hrp, data): 60 | """Compute a Bech32 string given HRP and data values.""" 61 | combined = data + bech32_create_checksum(hrp, data) 62 | return hrp + '1' + ''.join([CHARSET[d] for d in combined]) 63 | 64 | 65 | def bech32_decode(bech): 66 | """Validate a Bech32 string, and determine HRP and data.""" 67 | if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or 68 | (bech.lower() != bech and bech.upper() != bech)): 69 | return (None, None) 70 | bech = bech.lower() 71 | pos = bech.rfind('1') 72 | if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: 73 | return (None, None) 74 | if not all(x in CHARSET for x in bech[pos+1:]): 75 | return (None, None) 76 | hrp = bech[:pos] 77 | data = [CHARSET.find(x) for x in bech[pos+1:]] 78 | if not bech32_verify_checksum(hrp, data): 79 | return (None, None) 80 | return (hrp, data[:-6]) 81 | 82 | 83 | def convertbits(data, frombits, tobits, pad=True): 84 | """General power-of-2 base conversion.""" 85 | acc = 0 86 | bits = 0 87 | ret = [] 88 | maxv = (1 << tobits) - 1 89 | max_acc = (1 << (frombits + tobits - 1)) - 1 90 | for value in data: 91 | if value < 0 or (value >> frombits): 92 | return None 93 | acc = ((acc << frombits) | value) & max_acc 94 | bits += frombits 95 | while bits >= tobits: 96 | bits -= tobits 97 | ret.append((acc >> bits) & maxv) 98 | if pad: 99 | if bits: 100 | ret.append((acc << (tobits - bits)) & maxv) 101 | elif bits >= frombits or ((acc << (tobits - bits)) & maxv): 102 | return None 103 | return ret 104 | 105 | 106 | def decode(hrp, addr): 107 | """Decode a segwit address.""" 108 | hrpgot, data = bech32_decode(addr) 109 | if hrpgot != hrp: 110 | return (None, None) 111 | decoded = convertbits(data[1:], 5, 8, False) 112 | if decoded is None or len(decoded) < 2 or len(decoded) > 40: 113 | return (None, None) 114 | if data[0] > 16: 115 | return (None, None) 116 | if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: 117 | return (None, None) 118 | return (data[0], decoded) 119 | 120 | 121 | def encode(hrp, witver, witprog): 122 | """Encode a segwit address.""" 123 | ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5)) 124 | assert decode(hrp, ret) is not (None, None) 125 | return ret -------------------------------------------------------------------------------- /coinmetrics/utils/pipelines.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import traceback 3 | import threading 4 | import time 5 | from coinmetrics.utils.event import EventEmitter 6 | 7 | 8 | class AtomicCounter(object): 9 | 10 | def __init__(self, value): 11 | self.value = value 12 | self.lock = threading.Lock() 13 | 14 | def inc(self): 15 | with self.lock: 16 | self.value += 1 17 | 18 | def get(self): 19 | return self.value 20 | 21 | 22 | class LinearMultithreadedPipeline(object): 23 | 24 | def __init__(self, workersCount, proc, name): 25 | self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workersCount) 26 | self.proc = proc 27 | self.workersCount = workersCount 28 | self.stopSignal = None 29 | self.name = name 30 | self.exceptions = [] 31 | self.onExecuted = EventEmitter() 32 | 33 | def __str__(self): 34 | return self.name 35 | 36 | def pushTask(self, task, taskIndex): 37 | self.executor.submit(self._executeTask, task, taskIndex) 38 | 39 | def shutdown(self): 40 | self.executor.shutdown() 41 | 42 | def faulted(self): 43 | return len(self.exceptions) > 0 44 | 45 | def _executeTask(self, task, taskIndex): 46 | if not self.stopSignal(): 47 | try: 48 | result = self.proc(task, self.stopSignal) 49 | self.onExecuted.trigger(result, taskIndex) 50 | except: 51 | self.exceptions.append((taskIndex, traceback.format_exc())) 52 | 53 | 54 | class OrderingConnector(object): 55 | 56 | def __init__(self, source, destination): 57 | self.awaitingIndex = 0 58 | self.source = source 59 | self.destination = destination 60 | self.lock = threading.Lock() 61 | self.results = {} 62 | 63 | self.source.onExecuted.subscribe(self._onSourceReceived) 64 | 65 | def _onSourceReceived(self, result, taskIndex): 66 | with self.lock: 67 | self.results[taskIndex] = result 68 | while self.awaitingIndex in self.results: 69 | self.destination.pushTask(self.results[self.awaitingIndex], self.awaitingIndex) 70 | del self.results[self.awaitingIndex] 71 | self.awaitingIndex += 1 72 | 73 | 74 | class ExecutionCounter(object): 75 | 76 | def __init__(self, pipeline, taskCount): 77 | self.pipeline = pipeline 78 | self.count = AtomicCounter(0) 79 | self.taskCount = taskCount 80 | self.pipeline.onExecuted.subscribe(self._onExecuted) 81 | 82 | def allDone(self): 83 | return self.count.get() == self.taskCount 84 | 85 | def getCount(self): 86 | return self.count.get() 87 | 88 | def _onExecuted(self, result, taskIndex): 89 | self.count.inc() 90 | 91 | 92 | class EtaOutput(object): 93 | 94 | def __init__(self, pipeline, eta): 95 | self.eta = eta 96 | self.eta.workStarted() 97 | self.lock = threading.Lock() 98 | pipeline.onExecuted.subscribe(self._onExecuted) 99 | 100 | def _onExecuted(self, result, taskIndex): 101 | with self.lock: 102 | self.eta.workFinished(1) 103 | self.eta.workStarted() 104 | 105 | 106 | class TaskSpawner(object): 107 | 108 | def __init__(self, tasks, pipeline, targetProc, spawnSignal): 109 | self.tasks = tasks 110 | self.pipeline = pipeline 111 | self.total = len(self.tasks) 112 | self.spawned = 0 113 | self.keyboardInterrupt = False 114 | self.targetProc = targetProc 115 | self.lock = threading.Lock() 116 | self.spawnSignal = spawnSignal 117 | self.spawnSignal.subscribe(self._onSpawnSignal) 118 | 119 | def interrupted(self): 120 | return self.keyboardInterrupt 121 | 122 | def run(self, stopSignal): 123 | try: 124 | self._onSpawnSignal() 125 | while not stopSignal(): 126 | time.sleep(0.2) 127 | except KeyboardInterrupt: 128 | self.keyboardInterrupt = True 129 | 130 | def _onSpawnSignal(self, *args): 131 | with self.lock: 132 | target = min(self.total, self.targetProc()) 133 | while self.spawned < target: 134 | self.pipeline.pushTask(self.tasks[self.spawned], self.spawned) 135 | self.spawned += 1 136 | 137 | 138 | def reportPipelineExceptions(log, pipelines, tasks, taskRepr): 139 | for pipeline in pipelines: 140 | if pipeline.faulted(): 141 | message = [] 142 | for taskIndex, exception in pipeline.exceptions: 143 | if taskRepr is None: 144 | message.append("task index %d:\n" % taskIndex) 145 | else: 146 | message.append("task %s:\n" % taskRepr(tasks[taskIndex])) 147 | message.append("%s" % exception) 148 | log.critical("exceptions during execution of pipeline %s:\n%s" % (pipeline, "".join(message))) 149 | 150 | 151 | def runPipelineChain(tasks, pipelines, log, eta=None, taskRepr=None, prefetchCount=1): 152 | def anyPipelineFaulted(): 153 | for pipeline in pipelines: 154 | if pipeline.faulted(): 155 | return True 156 | return False 157 | 158 | def shouldStop(): 159 | return counter.allDone() or taskSpawner.interrupted() or anyPipelineFaulted() 160 | 161 | counter = ExecutionCounter(pipelines[-1], len(tasks)) 162 | 163 | if eta is not None: 164 | EtaOutput(pipelines[-1], eta) 165 | 166 | for pipeline in pipelines: 167 | pipeline.stopSignal = shouldStop 168 | 169 | taskSpawner = TaskSpawner(tasks, pipelines[0], lambda: counter.getCount() + prefetchCount, pipelines[-1].onExecuted) 170 | taskSpawner.run(shouldStop) 171 | 172 | for pipeline in pipelines: 173 | log.info("shutting down pipeline %s" % pipeline) 174 | pipeline.shutdown() 175 | 176 | reportPipelineExceptions(log, pipelines, tasks, taskRepr) 177 | return counter.allDone() and not anyPipelineFaulted(), taskSpawner.interrupted() 178 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/exporter.py: -------------------------------------------------------------------------------- 1 | class BulkExporterBase(object): 2 | 3 | def pushBlock(self, blockData): 4 | self.prologue(blockData) 5 | self.insertBlock(blockData) 6 | self.insertTransactions(blockData) 7 | self.insertOutputs(blockData) 8 | self.insertInputs(blockData) 9 | self.insertCoinbaseScripts(blockData) 10 | self.additionalProcessing(blockData) 11 | self.epilogue(blockData) 12 | 13 | def prologue(self, blockData): 14 | pass 15 | 16 | def epilogue(self, blockData): 17 | pass 18 | 19 | def additionalProcessing(self, blockData): 20 | pass 21 | 22 | 23 | class BitcoinExporter(BulkExporterBase): 24 | 25 | def __init__(self, dbAccess, schema, log): 26 | self.asset = schema.getAsset() 27 | self.schema = schema 28 | self.dbAccess = dbAccess 29 | self.log = log 30 | 31 | def prologue(self, blockData): 32 | inputsCount, outputsCount = 0, 0 33 | for tx in blockData.getTransactions(): 34 | inputsCount += len(tx.getInputs()) 35 | outputsCount += len(tx.getOutputs()) 36 | self.log.info("txs: %d, inputs: %d, outputs: %d" % (len(blockData.getTransactions()), inputsCount, outputsCount)) 37 | 38 | def epilogue(self, blockData): 39 | self.dbAccess.commit() 40 | 41 | def insertBlock(self, blockData): 42 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getBlocksTableName() + "\ 43 | (block_hash, block_height, block_size, block_time, block_median_time, block_difficulty, block_chainwork) \ 44 | VALUES (%s, %s, %s, %s, %s, %s, %s)", (blockData.hashAsNumber, blockData.blockHeight, 45 | blockData.blockSize, blockData.blockTime, blockData.blockMedianTime, blockData.difficulty, blockData.chainworkAsNumber)) 46 | 47 | def insertTransactions(self, blockData): 48 | txs = blockData.getTransactions() 49 | transactionTuples = [] 50 | for index, tx in enumerate(txs): 51 | transactionTuples.append((tx.txHash, tx.txSize, tx.coinbase, blockData.hashAsNumber, tx.txTime, tx.txMedianTime)) 52 | 53 | self.dbAccess.executeValues("INSERT INTO " + self.schema.getTransactionsTableName() + "\ 54 | (tx_hash, tx_size, tx_coinbase, tx_block_hash, tx_time, tx_median_time) VALUES %s", transactionTuples, 512) 55 | 56 | def insertInputs(self, blockData): 57 | batchUpdateData = [] 58 | for tx in blockData.getTransactions(): 59 | for inputTxHash, outputIndex, outputSpendSignature in tx.getInputs(): 60 | batchUpdateData.append((inputTxHash, outputIndex, bytearray.fromhex(outputSpendSignature), 61 | tx.txHash, tx.txTime, tx.txMedianTime)) 62 | 63 | self.dbAccess.executeValues("UPDATE " + self.schema.getOutputsTableName() + "\ 64 | SET output_spending_tx_hash=data.output_spending_tx_hash, output_time_spent=data.output_time_spent, \ 65 | output_median_time_spent=data.output_median_time_spent, output_spend_signature=data.output_spend_signature \ 66 | FROM (VALUES %s) AS data (output_tx_hash, output_index, output_spend_signature, \ 67 | output_spending_tx_hash, output_time_spent, output_median_time_spent) \ 68 | WHERE " + self.schema.getOutputsTableName() + ".output_tx_hash=data.output_tx_hash AND " 69 | + self.schema.getOutputsTableName() + ".output_index=data.output_index", batchUpdateData, 512) 70 | 71 | def insertOutputs(self, blockData): 72 | rows = [] 73 | for tx in blockData.getTransactions(): 74 | for outputIndex, outputType, addresses, scriptHex, value in tx.getOutputs(): 75 | rows.append((tx.txHash, outputIndex, outputType, addresses, 76 | bytearray.fromhex(scriptHex), value, tx.txTime, None, tx.txMedianTime, None)) 77 | self.dbAccess.executeValues("INSERT INTO " + self.schema.getOutputsTableName() + "\ 78 | (output_tx_hash, output_index, output_type, output_addresses, output_script, \ 79 | output_value_satoshi, output_time_created, output_time_spent, output_median_time_created, \ 80 | output_median_time_spent) VALUES %s", rows, 512) 81 | 82 | def insertCoinbaseScripts(self, blockData): 83 | for tx in blockData.getTransactions(): 84 | if tx.getCoinbaseScript() is not None: 85 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getCoinbaseScriptsTableName() + "\ 86 | (coinbase_script_tx_hash, coinbase_script_hex) VALUES (%s, %s)", 87 | (tx.txHash, bytearray.fromhex(tx.getCoinbaseScript()))) 88 | 89 | 90 | class ZcashExporter(BitcoinExporter): 91 | 92 | def additionalProcessing(self, blockData): 93 | for tx in blockData.getTransactions(): 94 | for valueOld, valueNew in tx.getJoinSplits(): 95 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getJoinSplitsTableName() + "\ 96 | (joinsplit_tx_hash, joinsplit_value_old, joinsplit_value_new, joinsplit_time) \ 97 | VALUES (%s, %s, %s, %s)", (tx.txHash, valueOld, valueNew, tx.txTime)) 98 | for inputCount, outputCount, valueBalance in tx.getSaplingPayments(): 99 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getSaplingPaymentTableName() + "\ 100 | (sapling_payment_tx_hash, sapling_payment_input_count, sapling_payment_output_count, sapling_payment_value_balance, sapling_payment_time) \ 101 | VALUES (%s, %s, %s, %s, %s)", (tx.txHash, inputCount, outputCount, valueBalance, tx.txTime)) 102 | 103 | 104 | class PivxExporter(BitcoinExporter): 105 | 106 | def additionalProcessing(self, blockData): 107 | for tx in blockData.getTransactions(): 108 | for mintValue in tx.getZerocoinMints(): 109 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getZerocoinMintsTableName() + "\ 110 | (zerocoin_mint_tx_hash, zerocoin_mint_value, zerocoin_mint_time) \ 111 | VALUES (%s, %s, %s)", (tx.txHash, mintValue, tx.txTime)) 112 | for spendValue in tx.getZerocoinSpends(): 113 | self.dbAccess.queryNoReturnNoCommit("INSERT INTO " + self.schema.getZerocoinSpendsTableName() + "\ 114 | (zerocoin_spend_tx_hash, zerocoin_spend_value, zerocoin_spend_time) \ 115 | VALUES (%s, %s, %s)", (tx.txHash, spendValue, tx.txTime)) 116 | 117 | 118 | class DecredExporter(BitcoinExporter): 119 | 120 | def insertTransactions(self, blockData): 121 | transactionTuples = [] 122 | for tx in blockData.getTransactions(): 123 | transactionTuples.append((tx.txHash, tx.txSize, tx.coinbase, blockData.hashAsNumber, tx.txTime, 124 | tx.vote, tx.ticket)) 125 | self.dbAccess.executeValues( 126 | "INSERT INTO " + self.schema.getTransactionsTableName() + "\ 127 | (tx_hash, tx_size, tx_coinbase, tx_block_hash, tx_time, tx_vote, tx_ticket) VALUES %s", 128 | transactionTuples, 512) -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/__init__.py: -------------------------------------------------------------------------------- 1 | class OmniBlockData(object): 2 | 3 | def __init__(self, blockHeight, blockHash, blockTime): 4 | self.blockHeight = blockHeight 5 | self.blockHash = blockHash 6 | self.blockTime = blockTime 7 | self.simpleSendTransactions = [] 8 | self.sendOwnersTransactions = [] 9 | self.sellForBitcoinTransactions = [] 10 | self.acceptSellForBitcoinTransactions = [] 11 | self.dexPurchaseTransactions = [] 12 | self.createFixedPropertyTransactions = [] 13 | self.createCrowdsalePropertyTransactions = [] 14 | self.closeCrowdsaleTransactions = [] 15 | self.createManagedPropertyTransactions = [] 16 | self.grantTokensTransactions = [] 17 | self.revokeTokensTransactions = [] 18 | self.sendAllTransactions = [] 19 | 20 | def addSimpleSendTransaction(self, tx): 21 | self.simpleSendTransactions.append(tx) 22 | 23 | def addSendOwnersTransaction(self, tx): 24 | self.sendOwnersTransactions.append(tx) 25 | 26 | def addSellForBitcoinTransaction(self, tx): 27 | self.sellForBitcoinTransactions.append(tx) 28 | 29 | def addAcceptSellForBitcoinTransaction(self, tx): 30 | self.acceptSellForBitcoinTransactions.append(tx) 31 | 32 | def addDexPurchaseTransaction(self, tx): 33 | self.dexPurchaseTransactions.append(tx) 34 | 35 | def addCreateFixedPropertyTransaction(self, tx): 36 | self.createFixedPropertyTransactions.append(tx) 37 | 38 | def addCreateCrowdsalePropertyTransaction(self, tx): 39 | self.createCrowdsalePropertyTransactions.append(tx) 40 | 41 | def addCloseCrowdsaleTransaction(self, tx): 42 | self.closeCrowdsaleTransactions.append(tx) 43 | 44 | def addCreateManagedPropertyTransaction(self, tx): 45 | self.createManagedPropertyTransactions.append(tx) 46 | 47 | def addGrantTokensTransaction(self, tx): 48 | self.grantTokensTransactions.append(tx) 49 | 50 | def addRevokeTokensTransaction(self, tx): 51 | self.revokeTokensTransactions.append(tx) 52 | 53 | def addSendAllTransaction(self, tx): 54 | self.sendAllTransactions.append(tx) 55 | 56 | 57 | class OmniTransactionBase(object): 58 | 59 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, fee): 60 | self.block_hash = txBlockHash 61 | self.hash = txHash 62 | self.time = txTime 63 | self.property_id = propertyId 64 | self.sending_address = sendingAddress 65 | self.fee = fee 66 | 67 | def getAttributes(self, prefix): 68 | keys = [] 69 | values = [] 70 | intermediate = [] 71 | for key, value in self.__dict__.items(): 72 | intermediate.append((prefix + "_" + key, value)) 73 | for key, value in sorted(intermediate, key=lambda pair: pair[0]): 74 | keys.append(key) 75 | values.append(value) 76 | return keys, values 77 | 78 | 79 | class OmniSimpleSendTransaction(OmniTransactionBase): 80 | 81 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, receivingAddress, amount, fee): 82 | super(OmniSimpleSendTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 83 | self.receiving_address = receivingAddress 84 | self.amount = amount 85 | 86 | 87 | class OmniSendAllTransaction(OmniTransactionBase): 88 | 89 | def __init__(self, txBlockHash, txHash, txIndex, txTime, propertyId, sendingAddress, receivingAddress, amount, fee): 90 | super(OmniSendAllTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 91 | self.index = txIndex 92 | self.receiving_address = receivingAddress 93 | self.amount = amount 94 | 95 | 96 | class OmniSendOwnersTransaction(OmniTransactionBase): 97 | 98 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee): 99 | super(OmniSendOwnersTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 100 | self.amount = amount 101 | 102 | 103 | class OmniSellForBitcoinTransaction(OmniTransactionBase): 104 | 105 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee, feeRequired, bitcoinDesired, action): 106 | super(OmniSellForBitcoinTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 107 | self.amount = amount 108 | self.fee_required = feeRequired 109 | self.bitcoin_desired = bitcoinDesired 110 | self.action = action 111 | 112 | 113 | class OmniAcceptSellForBitcoinTransaction(OmniTransactionBase): 114 | 115 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee, receivingAddress): 116 | super(OmniAcceptSellForBitcoinTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 117 | self.amount = amount 118 | self.receiving_address = receivingAddress 119 | 120 | 121 | class OmniDexPurchaseTransaction(OmniTransactionBase): 122 | 123 | def __init__(self, txBlockHash, txHash, txIndex, txTime, propertyId, sendingAddress, receivingAddress, amountBought, amountPaid): 124 | self.block_hash = txBlockHash 125 | self.hash = txHash 126 | self.index = txIndex 127 | self.time = txTime 128 | self.property_id = propertyId 129 | self.sending_address = sendingAddress 130 | self.receiving_address = receivingAddress 131 | self.amount_bought = amountBought 132 | self.amount_paid = amountPaid 133 | 134 | 135 | class OmniCreateFixedPropertyTransaction(OmniTransactionBase): 136 | 137 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee, propertyType): 138 | super(OmniCreateFixedPropertyTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 139 | self.amount = amount 140 | self.property_type = propertyType 141 | 142 | 143 | class OmniCreateManagedPropertyTransaction(OmniTransactionBase): 144 | 145 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, fee, propertyType): 146 | super(OmniCreateManagedPropertyTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 147 | self.property_type = propertyType 148 | 149 | 150 | class OmniGrantTokensTransaction(OmniTransactionBase): 151 | 152 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, receivingAddress, amount, fee): 153 | super(OmniGrantTokensTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 154 | self.receiving_address = receivingAddress 155 | self.amount = amount 156 | 157 | 158 | class OmniRevokeTokensTransaction(OmniTransactionBase): 159 | 160 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee): 161 | super(OmniRevokeTokensTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 162 | self.amount = amount 163 | 164 | 165 | class OmniCreateCrowdsalePropertyTransaction(OmniTransactionBase): 166 | 167 | def __init__(self, txBlockHash, txHash, txTime, propertyId, sendingAddress, amount, fee, propertyType, tokensPerUnit, deadline, bonus, issuerPercent): 168 | super(OmniCreateCrowdsalePropertyTransaction, self).__init__(txBlockHash, txHash, txTime, propertyId, sendingAddress, fee) 169 | self.amount = amount 170 | self.property_type = propertyType 171 | self.deadline = deadline 172 | self.tokens_per_unit = tokensPerUnit 173 | self.bonus = bonus 174 | self.issuer_percent = issuerPercent -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/query.py: -------------------------------------------------------------------------------- 1 | class OmniQuery(object): 2 | 3 | def __init__(self, dbAccess, schema): 4 | self.dbAccess = dbAccess 5 | self.schema = schema 6 | 7 | def getBlockHeight(self): 8 | result = self.dbAccess.queryReturnOne("SELECT max(block_height) FROM %s" % self.schema.getBlocksTableName())[0] 9 | return result if result is not None else 0 10 | 11 | 12 | class PropertyQuery(object): 13 | 14 | def __init__(self, dbAccess, schema, propertyId, assetName): 15 | self.dbAccess = dbAccess 16 | self.schema = schema 17 | self.propertyId = propertyId 18 | self.assetName = assetName 19 | 20 | def getAsset(self): 21 | return self.assetName 22 | 23 | def run(self, text): 24 | return self.dbAccess.queryReturnAll(text) 25 | 26 | def getMinBlockTime(self): 27 | result = self.dbAccess.queryReturnOne("SELECT min(block_time) FROM %s" % self.schema.getBlocksTableName())[0] 28 | return result if result is not None else 0 29 | 30 | def getMaxBlockTime(self): 31 | result = self.dbAccess.queryReturnOne("SELECT max(block_time) FROM %s" % self.schema.getBlocksTableName())[0] 32 | return result if result is not None else 0 33 | 34 | def getTxCountBetween(self, minTime, maxTime): 35 | result = self.dbAccess.queryReturnOne("WITH \ 36 | txs AS (\ 37 | SELECT simple_send_tx_hash FROM " + self.schema.getSimpleSendTxTableName() + " \ 38 | WHERE simple_send_tx_property_id=%s AND simple_send_tx_time >= %s AND simple_send_tx_time < %s \ 39 | UNION ALL \ 40 | SELECT send_owners_tx_hash FROM " + self.schema.getSendOwnersTxTableName() + "\ 41 | WHERE send_owners_tx_property_id=%s AND send_owners_tx_time >= %s AND send_owners_tx_time < %s \ 42 | UNION ALL \ 43 | SELECT send_all_tx_hash FROM " + self.schema.getSendAllTxTableName() + "\ 44 | WHERE send_all_tx_property_id=%s AND send_all_tx_time >= %s AND send_all_tx_time < %s GROUP BY send_all_tx_hash \ 45 | ) \ 46 | SELECT COUNT(*) FROM txs", (self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime, 47 | self.propertyId, minTime, maxTime)) 48 | return result[0] if result[0] is not None else 0 49 | 50 | def getOutputVolumeBetween(self, minTime, maxTime): 51 | result = self.dbAccess.queryReturnOne("WITH \ 52 | txs AS (\ 53 | SELECT simple_send_tx_amount AS amount FROM " + self.schema.getSimpleSendTxTableName() + " \ 54 | WHERE simple_send_tx_property_id=%s AND simple_send_tx_time >= %s AND simple_send_tx_time < %s \ 55 | UNION ALL \ 56 | SELECT send_owners_tx_amount AS amount FROM " + self.schema.getSendOwnersTxTableName() + "\ 57 | WHERE send_owners_tx_property_id=%s AND send_owners_tx_time >= %s AND send_owners_tx_time < %s \ 58 | UNION ALL \ 59 | SELECT send_all_tx_amount AS amount FROM " + self.schema.getSendAllTxTableName() + "\ 60 | WHERE send_all_tx_property_id=%s AND send_all_tx_time >= %s AND send_all_tx_time < %s \ 61 | ) \ 62 | SELECT SUM(amount) FROM txs", (self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime, 63 | self.propertyId, minTime, maxTime)) 64 | return result[0] if result[0] is not None else 0 65 | 66 | def getMedianTransactionValueBetween(self, minTime, maxTime): 67 | result = self.dbAccess.queryReturnOne("WITH \ 68 | txs AS (\ 69 | SELECT simple_send_tx_amount AS amount FROM " + self.schema.getSimpleSendTxTableName() + " \ 70 | WHERE simple_send_tx_property_id=%s AND simple_send_tx_time >= %s AND simple_send_tx_time < %s \ 71 | UNION ALL \ 72 | SELECT send_owners_tx_amount AS amount FROM " + self.schema.getSendOwnersTxTableName() + "\ 73 | WHERE send_owners_tx_property_id=%s AND send_owners_tx_time >= %s AND send_owners_tx_time < %s \ 74 | UNION ALL \ 75 | SELECT SUM(send_all_tx_amount) AS amount FROM " + self.schema.getSendAllTxTableName() + "\ 76 | WHERE send_all_tx_property_id=%s AND send_all_tx_time >= %s AND send_all_tx_time < %s GROUP BY send_all_tx_hash \ 77 | ) \ 78 | SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY amount) FROM txs", 79 | (self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime)) 80 | return result[0] 81 | 82 | def getActiveAddressesCountBetween(self, minTime, maxTime): 83 | result = self.dbAccess.queryReturnOne("WITH \ 84 | addresses AS (\ 85 | SELECT simple_send_tx_sending_address AS address FROM " + self.schema.getSimpleSendTxTableName() + " \ 86 | WHERE simple_send_tx_property_id=%s AND simple_send_tx_time >= %s AND simple_send_tx_time < %s \ 87 | UNION ALL \ 88 | SELECT simple_send_tx_receiving_address AS address FROM " + self.schema.getSimpleSendTxTableName() + " \ 89 | WHERE simple_send_tx_property_id=%s AND simple_send_tx_time >= %s AND simple_send_tx_time < %s \ 90 | UNION ALL \ 91 | SELECT send_owners_tx_sending_address AS address FROM " + self.schema.getSendOwnersTxTableName() + "\ 92 | WHERE send_owners_tx_property_id=%s AND send_owners_tx_time >= %s AND send_owners_tx_time < %s \ 93 | UNION ALL \ 94 | SELECT send_all_tx_sending_address AS address FROM " + self.schema.getSendAllTxTableName() + "\ 95 | WHERE send_all_tx_property_id=%s AND send_all_tx_time >= %s AND send_all_tx_time < %s \ 96 | UNION ALL \ 97 | SELECT send_all_tx_receiving_address AS address FROM " + self.schema.getSendAllTxTableName() + "\ 98 | WHERE send_all_tx_property_id=%s AND send_all_tx_time >= %s AND send_all_tx_time < %s \ 99 | ) \ 100 | SELECT COUNT(DISTINCT address) FROM addresses", 101 | (self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime, 102 | self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime)) 103 | return result[0] if result[0] is not None else 0 104 | 105 | 106 | class ManagedPropertyQuery(PropertyQuery): 107 | 108 | def getRewardBetween(self, minTime, maxTime): 109 | result = self.dbAccess.queryReturnOne("WITH \ 110 | txs AS (\ 111 | SELECT grant_tokens_tx_amount AS amount FROM " + self.schema.getGrantTokensTxTableName() + " \ 112 | WHERE grant_tokens_tx_property_id=%s AND grant_tokens_tx_time >= %s AND grant_tokens_tx_time < %s \ 113 | UNION ALL \ 114 | SELECT -revoke_tokens_tx_amount AS amount FROM " + self.schema.getRevokeTokensTxTableName() + "\ 115 | WHERE revoke_tokens_tx_property_id=%s AND revoke_tokens_tx_time >= %s AND revoke_tokens_tx_time < %s \ 116 | ) \ 117 | SELECT SUM(amount) FROM txs", (self.propertyId, minTime, maxTime, self.propertyId, minTime, maxTime)) 118 | return result[0] if result[0] is not None else 0 119 | 120 | 121 | class TetherQuery(ManagedPropertyQuery): 122 | 123 | def __init__(self, dbAccess, schema): 124 | super(TetherQuery, self).__init__(dbAccess, schema, 31, "usdt") 125 | 126 | 127 | class MaidSafeCoinQuery(PropertyQuery): 128 | 129 | def __init__(self, dbAccess, schema): 130 | super(MaidSafeCoinQuery, self).__init__(dbAccess, schema, 3, "maid") 131 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/aggregator.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta, datetime 2 | from coinmetrics.utils.eta import ETA 3 | from coinmetrics.utils.timeutil import alignDateToInterval 4 | 5 | 6 | class DailyStatistic(object): 7 | 8 | def __init__(self, name, dataType, dbAccess, query): 9 | self.name = name 10 | self.dataType = dataType 11 | self.dbAccess = dbAccess 12 | self.query = query 13 | self.asset = query.getAsset() 14 | self.init() 15 | 16 | def getName(self): 17 | return self.name 18 | 19 | def getTableName(self): 20 | return self.tableName 21 | 22 | def init(self): 23 | self.tableName = "statistic_" + self.name + "_" + self.asset 24 | self.dbAccess.queryNoReturnCommit("\ 25 | CREATE TABLE IF NOT EXISTS %s (date TIMESTAMP PRIMARY KEY, value %s)" % (self.tableName, self.dataType)) 26 | 27 | def drop(self): 28 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % self.tableName) 29 | 30 | def runOn(self, date, save=True): 31 | value = self.calculateForDate(date) 32 | if save: 33 | self.save(date, value) 34 | return value 35 | 36 | def getDates(self): 37 | return [row[0] for row in self.dbAccess.queryReturnAll("SELECT date FROM %s" % self.tableName)] 38 | 39 | def save(self, date, value): 40 | self.dbAccess.queryNoReturnCommit(""" 41 | INSERT INTO {0} (date, value) VALUES (%s, %s) 42 | ON CONFLICT ON CONSTRAINT {0}_pkey DO UPDATE SET value=EXCLUDED.value""".format(self.tableName), (date, value)) 43 | 44 | def calculateForDate(self, date): 45 | pass 46 | 47 | 48 | class SimpleStatistic(DailyStatistic): 49 | 50 | def __init__(self, dbAccess, query): 51 | super(SimpleStatistic, self).__init__(self.name, self.dataType, dbAccess, query) 52 | 53 | def calculateForDate(self, date): 54 | return getattr(self.query, self.proc)(date, date + timedelta(days=1)) 55 | 56 | 57 | class DailyTxCountStatistic(SimpleStatistic): 58 | name = "tx_count" 59 | dataType = "INTEGER" 60 | proc = "getTxCountBetween" 61 | 62 | class DailyTxVolumeStatistic(SimpleStatistic): 63 | name = "tx_volume" 64 | dataType = "DECIMAL(32)" 65 | proc = "getOutputVolumeBetween" 66 | 67 | class DailyActiveAddressesStatistic(SimpleStatistic): 68 | name = "active_addresses" 69 | dataType = "INTEGER" 70 | proc = "getActiveAddressesCountBetween" 71 | 72 | class DailyFeesStatistic(SimpleStatistic): 73 | name = "fees" 74 | dataType = "BIGINT" 75 | proc = "getFeesVolumeBetween" 76 | 77 | class DailyRewardStatistic(SimpleStatistic): 78 | name = "reward" 79 | dataType = "BIGINT" 80 | proc = "getRewardBetween" 81 | 82 | class DailyAverageDifficultyStatistic(SimpleStatistic): 83 | name = "average_difficulty" 84 | dataType = "FLOAT" 85 | proc = "getAverageDifficultyBetween" 86 | 87 | class DailyMedianFeeStatistic(SimpleStatistic): 88 | name = "median_fee" 89 | dataType = "BIGINT" 90 | proc = "getMedianFeeBetween" 91 | 92 | class DailyMedianTransactionValueStatistic(SimpleStatistic): 93 | name = "median_tx_value" 94 | dataType = "BIGINT" 95 | proc = "getMedianTransactionValueBetween" 96 | 97 | class DailyPaymentCountStatistic(SimpleStatistic): 98 | name = "payment_count" 99 | dataType = "BIGINT" 100 | proc = "getPaymentCountBetween" 101 | 102 | class DailyBlockSizeStatistic(SimpleStatistic): 103 | name = "block_size" 104 | dataType = "BIGINT" 105 | proc = "getBlockSizeBetween" 106 | 107 | class DailyHeuristicalTxVolumeStatistic(SimpleStatistic): 108 | name = "heuristical_volume" 109 | dataType = "DECIMAL(32)" 110 | proc = "getHeuristicalOutputVolumeBetween" 111 | 112 | class DailyBlockCountStatistic(SimpleStatistic): 113 | name = "block_count" 114 | dataType = "BIGINT" 115 | proc = "getBlockCountBetween" 116 | 117 | class Daily1YCirculatingSupplyStatistic(SimpleStatistic): 118 | name = "1y_circulating_supply" 119 | dataType = "DECIMAL(32)" 120 | proc = "get1YCirculatingSupplyBetween" 121 | 122 | class Daily180DCirculatingSupplyStatistic(SimpleStatistic): 123 | name = "180d_circulating_supply" 124 | dataType = "DECIMAL(32)" 125 | proc = "get180DCirculatingSupplyBetween" 126 | 127 | class Daily30DCirculatingSupplyStatistic(SimpleStatistic): 128 | name = "30d_circulating_supply" 129 | dataType = "DECIMAL(32)" 130 | proc = "get30DCirculatingSupplyBetween" 131 | 132 | class DailyTotalSupplyStatistic(SimpleStatistic): 133 | name = "total_supply" 134 | dataType = "DECIMAL(32)" 135 | proc = "getTotalSupplyBetween" 136 | 137 | class DailyNaive30DCirculatingSupplyStatistic(SimpleStatistic): 138 | name = "30d_naive_circulating_supply" 139 | dataType = "DECIMAL(32)" 140 | proc = "get30DNaiveCirculatingSupplyBetween" 141 | 142 | 143 | class DailyAggregator(object): 144 | 145 | def __init__(self, dbAccess, query, log, minDate=None): 146 | self.dbAccess = dbAccess 147 | self.query = query 148 | self.log = log 149 | self.minDate = minDate 150 | self.metrics = [] 151 | 152 | self.createDailyMetrics() 153 | 154 | def createDailyMetrics(self): 155 | self.addMetric(DailyAverageDifficultyStatistic(self.dbAccess, self.query)) 156 | self.addMetric(DailyTxCountStatistic(self.dbAccess, self.query)) 157 | self.addMetric(DailyTxVolumeStatistic(self.dbAccess, self.query)) 158 | self.addMetric(DailyHeuristicalTxVolumeStatistic(self.dbAccess, self.query)) 159 | self.addMetric(DailyMedianTransactionValueStatistic(self.dbAccess, self.query)) 160 | self.addMetric(DailyActiveAddressesStatistic(self.dbAccess, self.query)) 161 | self.addMetric(DailyFeesStatistic(self.dbAccess, self.query)) 162 | self.addMetric(DailyMedianFeeStatistic(self.dbAccess, self.query)) 163 | self.addMetric(DailyPaymentCountStatistic(self.dbAccess, self.query)) 164 | self.addMetric(DailyRewardStatistic(self.dbAccess, self.query)) 165 | self.addMetric(DailyBlockSizeStatistic(self.dbAccess, self.query)) 166 | self.addMetric(DailyBlockCountStatistic(self.dbAccess, self.query)) 167 | self.addMetric(Daily1YCirculatingSupplyStatistic(self.dbAccess, self.query)) 168 | self.addMetric(Daily180DCirculatingSupplyStatistic(self.dbAccess, self.query)) 169 | self.addMetric(Daily30DCirculatingSupplyStatistic(self.dbAccess, self.query)) 170 | self.addMetric(DailyTotalSupplyStatistic(self.dbAccess, self.query)) 171 | # can be useful for diagnosis 172 | # self.addMetric(DailyNaive30DCirculatingSupplyStatistic(self.dbAccess, self.query)) 173 | 174 | def getMetricNames(self): 175 | return [metric.getName() for metric in self.metrics] 176 | 177 | def addMetric(self, metric): 178 | self.metrics.append(metric) 179 | 180 | def run(self, metricNames, startDate, endDate, shouldSave, forceRecomputation): 181 | minBlockTime, maxBlockTime = self._getBlockchainTimeBounds() 182 | minComputeTime = max(minBlockTime, alignDateToInterval(startDate, timedelta(days=1))) 183 | maxComputeTime = min(maxBlockTime, alignDateToInterval(endDate, timedelta(days=1))) 184 | computeDateSet = set([minComputeTime + timedelta(days=i) for i in range((maxComputeTime - minComputeTime).days + 1)]) 185 | 186 | datesToStatMap = {} 187 | for metric in self.metrics: 188 | if metric.getName() in metricNames: 189 | doneDates = {} if forceRecomputation else metric.getDates() 190 | missingDates = computeDateSet.difference(doneDates) 191 | for missingDate in missingDates: 192 | if missingDate not in datesToStatMap: 193 | datesToStatMap[missingDate] = [] 194 | datesToStatMap[missingDate].append(metric) 195 | 196 | datesAndStats = [(missingDate, statList) for missingDate, statList in datesToStatMap.items()] 197 | datesAndStats = sorted(datesAndStats, key=lambda pair: pair[0]) 198 | if len(datesAndStats) == 0: 199 | self.log.info("no metrics to calculate") 200 | return 201 | 202 | eta = ETA(self.log, len(datesAndStats), 60, 10) 203 | for missingDate, statList in datesAndStats: 204 | eta.workStarted() 205 | self.log.info("date: %s" % missingDate) 206 | for stat in statList: 207 | t = datetime.now() 208 | value = stat.runOn(missingDate, shouldSave) 209 | self.log.info("aggregation result for %s on %s: %s (time spent: %s)" % (stat.getName(), missingDate, str(value), datetime.now() - t)) 210 | eta.workFinished(1) 211 | 212 | def drop(self, metricNames): 213 | for metric in self.metrics: 214 | if metric.getName() in metricNames: 215 | metric.drop() 216 | 217 | def _getBlockchainTimeBounds(self): 218 | minBlockTime = self.query.getMinBlockTime() 219 | maxBlockTime = self.query.getMaxBlockTime() 220 | if minBlockTime == 0 or maxBlockTime == 0: 221 | raise ValueError("no blocks found for %s" % self.query.getAsset()) 222 | 223 | minBlockTime = minBlockTime.replace(hour=0, minute=0, second=0, microsecond=0) 224 | if self.minDate is not None: 225 | minBlockTime = max(minBlockTime, self.minDate) 226 | 227 | maxBlockTime = maxBlockTime.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=1) 228 | return minBlockTime, maxBlockTime 229 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/__init__.py: -------------------------------------------------------------------------------- 1 | import time 2 | import traceback 3 | from coinmetrics.utils import pipelines 4 | from coinmetrics.utils.postgres import PostgresAccess 5 | from coinmetrics.bitsql.schema import * 6 | from coinmetrics.bitsql.query import * 7 | from coinmetrics.bitsql.exporter import * 8 | from coinmetrics.bitsql.node import * 9 | from coinmetrics.bitsql.aggregator import * 10 | from coinmetrics.bitsql.omni.node import OmniNode 11 | from coinmetrics.bitsql.omni.schema import OmniSchema 12 | from coinmetrics.bitsql.omni.query import OmniQuery, TetherQuery, MaidSafeCoinQuery 13 | from coinmetrics.bitsql.omni.exporter import OmniExporter 14 | from coinmetrics.bitsql.omni.aggregator import OmniManagedPropertyAggregator, OmniCrowdsalePropertyAggregator 15 | from coinmetrics.utils.execution import executeInParallelSameProc 16 | 17 | 18 | def postgresFactory(*dbParams): 19 | return PostgresAccess(*dbParams) 20 | 21 | 22 | def dbObjectsFactory(asset, db, log): 23 | def bitcoinClone(name): 24 | return [ 25 | lambda db: BitcoinSchema(name, db), 26 | lambda db, schema: BitcoinQuery(db, schema), 27 | lambda db, schema: BitcoinExporter(db, schema, log), 28 | lambda db, query: DailyAggregator(db, query, log) 29 | ] 30 | 31 | registry = { 32 | "btc": bitcoinClone("btc"), 33 | "bch": [ 34 | lambda db: BitcoinSchema("bch", db), 35 | lambda db, schema: BitcoinQuery(db, schema), 36 | lambda db, schema: BitcoinExporter(db, schema, log), 37 | lambda db, query: DailyAggregator(db, query, log, datetime(year=2017, month=8, day=3)) 38 | ], 39 | "btg": [ 40 | lambda db: BitcoinSchema("btg", db), 41 | lambda db, schema: BitcoinQuery(db, schema), 42 | lambda db, schema: BitcoinExporter(db, schema, log), 43 | lambda db, query: DailyAggregator(db, query, log, datetime(year=2017, month=11, day=16)) 44 | ], 45 | "bsv": [ 46 | lambda db: BitcoinSchema("bsv", db), 47 | lambda db, schema: BitcoinQuery(db, schema), 48 | lambda db, schema: BitcoinExporter(db, schema, log), 49 | lambda db, query: DailyAggregator(db, query, log, datetime(year=2018, month=11, day=16)) 50 | ], 51 | "ltc": bitcoinClone("ltc"), 52 | "vtc": bitcoinClone("vtc"), 53 | "doge": bitcoinClone("doge"), 54 | "dash": bitcoinClone("dash"), 55 | "dgb": bitcoinClone("dgb"), 56 | "xvg": bitcoinClone("xvg"), 57 | "dcr": [ 58 | lambda db: DecredSchema("dcr", db), 59 | lambda db, schema: DecredQuery(db, schema), 60 | lambda db, schema: DecredExporter(db, schema, log), 61 | lambda db, query: DailyAggregator(db, query, log) 62 | ], 63 | "zec": [ 64 | lambda db: ZcashSchema("zec", db), 65 | lambda db, schema: ZcashQuery(db, schema), 66 | lambda db, schema: ZcashExporter(db, schema, log), 67 | lambda db, query: DailyAggregator(db, query, log) 68 | ], 69 | "btcp": [ 70 | lambda db: ZcashSchema("btcp", db), 71 | lambda db, schema: ZcashQuery(db, schema), 72 | lambda db, schema: ZcashExporter(db, schema, log), 73 | lambda db, query: DailyAggregator(db, query, log) 74 | ], 75 | "pivx": [ 76 | lambda db: PivxSchema("pivx", db), 77 | lambda db, schema: PivxQuery(db, schema), 78 | lambda db, schema: PivxExporter(db, schema, log), 79 | lambda db, query: DailyAggregator(db, query, log) 80 | ], 81 | "omnilayer": [ 82 | lambda db: OmniSchema(db), 83 | lambda db, schema: OmniQuery(db, schema), 84 | lambda db, schema: OmniExporter(db, schema, log), 85 | lambda db, query: None 86 | ], 87 | "usdt": [ 88 | lambda db: OmniSchema(db), 89 | lambda db, schema: TetherQuery(db, schema), 90 | lambda db, schema: None, 91 | lambda db, query: OmniManagedPropertyAggregator(db, query, log) 92 | ], 93 | "maid": [ 94 | lambda db: OmniSchema(db), 95 | lambda db, schema: MaidSafeCoinQuery(db, schema), 96 | lambda db, schema: None, 97 | lambda db, query: OmniCrowdsalePropertyAggregator(db, query, log), 98 | ] 99 | } 100 | 101 | if asset not in registry: 102 | raise Exception("Unknown asset: %s" % asset) 103 | else: 104 | schemaFactory, queryFactory, exporterFactory, aggregatorFactory = registry[asset] 105 | schema = schemaFactory(db) 106 | query = queryFactory(db, schema) 107 | exporter = exporterFactory(db, schema) 108 | aggregator = aggregatorFactory(db, query) 109 | return schema, query, exporter, aggregator 110 | 111 | 112 | def nodeFactory(asset, *args): 113 | registry = { 114 | "btc": lambda: BitcoinNode(*args), 115 | "bch": lambda: BitcoinNode(*args), 116 | "btg": lambda: BitcoinGoldNode(*args), 117 | "bsv": lambda: BitcoinSvNode(*args), 118 | "ltc": lambda: BitcoinNodeBase(*args), 119 | "vtc": lambda: BitcoinNodeBase(*args), 120 | "doge": lambda: DogecoinNode(*args), 121 | "dash": lambda: DashNode(*args), 122 | "dgb": lambda: BitcoinNodeBase(*args), 123 | "xvg": lambda: VergeNode(*args), 124 | "zec": lambda: ZcashNode(*args), 125 | "btcp": lambda: BitcoinPrivateNode(*args), 126 | "pivx": lambda: PivxNode(*args), 127 | "dcr": lambda: DecredNode(*args), 128 | "omnilayer": lambda: OmniNode(*args), 129 | } 130 | 131 | if asset not in registry: 132 | raise Exception("Unknown asset: %s" % asset) 133 | else: 134 | return registry[asset]() 135 | 136 | 137 | def runExport(asset, nodeList, dbParams, log, loop=False, lag=60 * 60 * 2, rpcThreads=8): 138 | def getNodeBlockCount(index, nodeParams): 139 | node = nodeFactory(asset, *nodeParams) 140 | try: 141 | blockCount = node.getBlockCount() 142 | return (index, blockCount, node) 143 | except Exception as e: 144 | log.warning("failed to get block count from node {0}: {1}".format(nodeParams, e)) 145 | raise e 146 | 147 | def pickNode(nodeList, asset): 148 | args = [(index, nodeParams) for index, nodeParams in enumerate(nodeList)] 149 | result = [value for value, exception in executeInParallelSameProc(getNodeBlockCount, args) if exception is None] 150 | 151 | if len(result) == 0: 152 | raise Exception("failed to get block count from any node") 153 | else: 154 | pick, bestHeight, bestIndex = None, 0, len(nodeList) 155 | for index, blockCount, node in result: 156 | if blockCount > bestHeight: 157 | bestHeight = blockCount 158 | bestIndex = index 159 | pick = node 160 | elif blockCount == bestHeight: 161 | if index < bestIndex: 162 | bestIndex = index 163 | pick = node 164 | log.info("picked node: {0}".format(pick)) 165 | return pick, bestHeight 166 | 167 | def proc(db): 168 | node, nodeHeight = pickNode(nodeList, asset) 169 | _, query, exporter, _ = dbObjectsFactory(asset, db, log) 170 | 171 | blockTime = BLOCK_TIMES[asset] 172 | blockLag = lag // blockTime 173 | blocksPerWeek = 7 * 24 * 3600 // blockTime 174 | 175 | dbHeight = query.getBlockHeight() 176 | if dbHeight is None: 177 | dbHeight = -1 178 | nodeHeight -= blockLag 179 | 180 | if nodeHeight <= dbHeight: 181 | log.info("node height (%d) is smaller than or equal to db height (%d), nothing to do" % (nodeHeight, dbHeight)) 182 | return False, False 183 | else: 184 | log.info("node height: %d, db height: %d, blocks to sync: %d" % (nodeHeight, dbHeight, nodeHeight - dbHeight)) 185 | fromHeight = dbHeight + 1 186 | toHeight = nodeHeight 187 | 188 | def load(height, stopSignal): 189 | return node.getBlock(height) 190 | 191 | def store(blockData, stopSignal): 192 | exporter.pushBlock(blockData) 193 | log.info("saved block at height: %d (%s)" % (blockData.blockHeight, blockData.blockTime)) 194 | 195 | heights = [i + fromHeight for i in range(toHeight - fromHeight + 1)] 196 | eta = ETA(log, toHeight - fromHeight + 1, blocksPerWeek, 10) 197 | loadPipeline = pipelines.LinearMultithreadedPipeline(rpcThreads, load, "node") 198 | storePipeline = pipelines.LinearMultithreadedPipeline(1, store, "db") 199 | pipelines.OrderingConnector(loadPipeline, storePipeline) 200 | result = pipelines.runPipelineChain(heights, [loadPipeline, storePipeline], log, eta, lambda task: "height " + str(task), 64) 201 | return result 202 | 203 | if not loop: 204 | proc(postgresFactory(*dbParams)) 205 | else: 206 | while True: 207 | keyboardInterrupt = False 208 | 209 | try: 210 | db = postgresFactory(*dbParams) 211 | result, keyboardInterrupt = proc(db) 212 | except Exception as e: 213 | log.critical(traceback.format_exc()) 214 | finally: 215 | db.close() 216 | 217 | if keyboardInterrupt: 218 | break 219 | 220 | try: 221 | time.sleep(30.0) 222 | except KeyboardInterrupt: 223 | break 224 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/schema.py: -------------------------------------------------------------------------------- 1 | from coinmetrics.bitsql.constants import * 2 | 3 | 4 | class BitcoinSchema(object): 5 | 6 | def __init__(self, asset, dbAccess): 7 | self.asset = asset 8 | self.dbAccess = dbAccess 9 | self.init() 10 | 11 | def getAsset(self): 12 | return self.asset 13 | 14 | def getBlocksTableName(self): 15 | return self.blocksTableName 16 | 17 | def getTransactionsTableName(self): 18 | return self.transactionsTableName 19 | 20 | def getOutputsTableName(self): 21 | return self.outputsTableName 22 | 23 | def getCoinbaseScriptsTableName(self): 24 | return self.coinbaseScriptsTableName 25 | 26 | def init(self): 27 | self.blocksTableName = self.asset + "_blocks" 28 | self.transactionsTableName = self.asset + "_transactions" 29 | self.outputsTableName = self.asset + "_outputs" 30 | self.coinbaseScriptsTableName = self.asset + "_coinbase_scripts" 31 | 32 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 33 | block_hash DECIMAL(%s) PRIMARY KEY, \ 34 | block_height INTEGER, \ 35 | block_size INTEGER, \ 36 | block_time TIMESTAMP, \ 37 | block_median_time TIMESTAMP, \ 38 | block_difficulty DOUBLE PRECISION, \ 39 | block_chainwork DECIMAL(%s) \ 40 | )" % (self.blocksTableName, HASH_PRECISION, CHAINWORK_PRECISION)) 41 | 42 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 43 | tx_hash DECIMAL(%s) PRIMARY KEY, \ 44 | tx_block_hash DECIMAL(%s), \ 45 | tx_size INTEGER, \ 46 | tx_time TIMESTAMP, \ 47 | tx_median_time TIMESTAMP, \ 48 | tx_coinbase BOOLEAN \ 49 | )" % (self.transactionsTableName, HASH_PRECISION, HASH_PRECISION)) 50 | 51 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 52 | output_tx_hash DECIMAL(%s), \ 53 | output_index INTEGER, \ 54 | output_type SMALLINT, \ 55 | output_addresses VARCHAR(%s)[], \ 56 | output_script BYTEA, \ 57 | output_spend_signature BYTEA, \ 58 | output_value_satoshi DECIMAL(%s), \ 59 | output_spending_tx_hash DECIMAL(%s), \ 60 | output_time_created TIMESTAMP, \ 61 | output_time_spent TIMESTAMP, \ 62 | output_median_time_created TIMESTAMP, \ 63 | output_median_time_spent TIMESTAMP, \ 64 | PRIMARY KEY(output_tx_hash, output_index)\ 65 | )" % (self.outputsTableName, HASH_PRECISION, MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION, HASH_PRECISION)) 66 | 67 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 68 | coinbase_script_tx_hash DECIMAL(%s) PRIMARY KEY, \ 69 | coinbase_script_hex BYTEA \ 70 | )" % (self.coinbaseScriptsTableName, HASH_PRECISION)) 71 | 72 | def drop(self): 73 | self.dropIndexes() 74 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.coinbaseScriptsTableName,)) 75 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.outputsTableName,)) 76 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.transactionsTableName,)) 77 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.blocksTableName,)) 78 | 79 | def addIndexes(self): 80 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_block_time_index ON %s_blocks(block_time)" % (self.asset, self.asset)) 81 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_block_height_index ON %s_blocks(block_height)" % (self.asset, self.asset)) 82 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_tx_time_index ON %s_transactions(tx_time)" % (self.asset, self.asset)) 83 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_output_time_spent_index ON %s_outputs(output_time_spent)" % (self.asset, self.asset)) 84 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_output_time_created_index ON %s_outputs(output_time_created)" % (self.asset, self.asset)) 85 | 86 | def dropIndexes(self): 87 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_output_time_created_index" % (self.asset,)) 88 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_output_time_spent_index" % (self.asset,)) 89 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_tx_time_index" % (self.asset,)) 90 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_block_height_index" % (self.asset,)) 91 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_block_time_index" % (self.asset,)) 92 | 93 | def vacuum(self): 94 | isolationLevel = self.dbAccess.connection.isolation_level 95 | self.dbAccess.connection.set_isolation_level(0) 96 | self.dbAccess.queryNoReturnNoCommit("VACUUM ANALYZE %s" % self.getOutputsTableName()) 97 | self.dbAccess.connection.set_isolation_level(isolationLevel) 98 | 99 | 100 | class ZcashSchema(BitcoinSchema): 101 | 102 | def getJoinSplitsTableName(self): 103 | return self.joinSplitsTableName 104 | 105 | def getSaplingPaymentTableName(self): 106 | return self.saplingPaymentTableName 107 | 108 | def init(self): 109 | super(ZcashSchema, self).init() 110 | self.joinSplitsTableName = self.asset + "_joinsplits" 111 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 112 | id SERIAL PRIMARY KEY, \ 113 | joinsplit_tx_hash DECIMAL(%s), \ 114 | joinsplit_value_old DECIMAL(%s), \ 115 | joinsplit_value_new DECIMAL(%s), \ 116 | joinsplit_time TIMESTAMP \ 117 | )" % (self.joinSplitsTableName, HASH_PRECISION, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 118 | 119 | self.saplingPaymentTableName = self.asset + "_sapling_payments" 120 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 121 | sapling_payment_tx_hash DECIMAL(%s), \ 122 | sapling_payment_value_balance DECIMAL(%s), \ 123 | sapling_payment_input_count INTEGER, \ 124 | sapling_payment_output_count INTEGER, \ 125 | sapling_payment_time TIMESTAMP, \ 126 | PRIMARY KEY(sapling_payment_tx_hash) \ 127 | )" % (self.saplingPaymentTableName, HASH_PRECISION, OUTPUT_VALUE_PRECISION)) 128 | 129 | def drop(self): 130 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.saplingPaymentTableName,)) 131 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.joinSplitsTableName,)) 132 | super(ZcashSchema, self).drop() 133 | 134 | def addIndexes(self): 135 | super(ZcashSchema, self).addIndexes() 136 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_joinsplit_time_index ON %s_joinsplits(joinsplit_time)" % (self.asset, self.asset)) 137 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_sapling_payment_time_index ON %s_sapling_payments(sapling_payment_time)" % (self.asset, self.asset)) 138 | 139 | def dropIndexes(self): 140 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_sapling_payment_time_index" % (self.asset,)) 141 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_joinsplit_time_index" % (self.asset,)) 142 | super(ZcashSchema, self).dropIndexes() 143 | 144 | 145 | class PivxSchema(BitcoinSchema): 146 | 147 | def getZerocoinMintsTableName(self): 148 | return self.zerocoinMintsTableName 149 | 150 | def getZerocoinSpendsTableName(self): 151 | return self.zerocoinSpendsTableName 152 | 153 | def init(self): 154 | super(PivxSchema, self).init() 155 | self.zerocoinMintsTableName = self.asset + "_zerocoin_mints" 156 | self.zerocoinSpendsTableName = self.asset + "_zerocoin_spends" 157 | 158 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 159 | id SERIAL PRIMARY KEY, \ 160 | zerocoin_mint_tx_hash DECIMAL(%s), \ 161 | zerocoin_mint_value DECIMAL(%s), \ 162 | zerocoin_mint_time TIMESTAMP \ 163 | )" % (self.zerocoinMintsTableName, HASH_PRECISION, OUTPUT_VALUE_PRECISION)) 164 | 165 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 166 | id SERIAL PRIMARY KEY, \ 167 | zerocoin_spend_tx_hash DECIMAL(%s), \ 168 | zerocoin_spend_value DECIMAL(%s), \ 169 | zerocoin_spend_time TIMESTAMP \ 170 | )" % (self.zerocoinSpendsTableName, HASH_PRECISION, OUTPUT_VALUE_PRECISION)) 171 | 172 | def drop(self): 173 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.zerocoinSpendsTableName,)) 174 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.zerocoinMintsTableName,)) 175 | super(PivxSchema, self).drop() 176 | 177 | def addIndexes(self): 178 | super(PivxSchema, self).addIndexes() 179 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_zerocoin_mint_time_index ON %s_zerocoin_mints(zerocoin_mint_time)" % (self.asset, self.asset)) 180 | self.dbAccess.queryNoReturnCommit("CREATE INDEX %s_zerocoin_spend_time_index ON %s_zerocoin_spends(zerocoin_spend_time)" % (self.asset, self.asset)) 181 | 182 | def dropIndexes(self): 183 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_zerocoin_spend_time_index" % (self.asset,)) 184 | self.dbAccess.queryNoReturnCommit("DROP INDEX IF EXISTS %s_zerocoin_mint_time_index" % (self.asset,)) 185 | super(PivxSchema, self).dropIndexes() 186 | 187 | 188 | class DecredSchema(BitcoinSchema): 189 | 190 | def init(self): 191 | super(DecredSchema, self).init() 192 | self.dbAccess.queryNoReturnCommit("ALTER TABLE %s ADD COLUMN IF NOT EXISTS tx_vote BOOLEAN" % self.transactionsTableName) 193 | self.dbAccess.queryNoReturnCommit("ALTER TABLE %s ADD COLUMN IF NOT EXISTS tx_ticket BOOLEAN" % self.transactionsTableName) 194 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/node.py: -------------------------------------------------------------------------------- 1 | from coinmetrics.utils.jsonrpc import JsonRpcCaller 2 | from coinmetrics.bitsql.omni import * 3 | from coinmetrics.bitsql.constants import HASH_PRECISION, MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION 4 | from coinmetrics.bitsql.node import outputValueToSatoshis 5 | from datetime import datetime 6 | 7 | SELL_FOR_BITCOIN_ACTIONS = { 8 | "new": 1, 9 | "update": 2, 10 | "cancel": 3, 11 | } 12 | 13 | PROPERTY_TYPES = { 14 | "indivisible": 1, 15 | "divisible": 2, 16 | } 17 | 18 | IGNORE_TX_TYPES = {25, 26, 28, 70, 71, 185, 65534} 19 | 20 | 21 | def omniOutputValueSatoshi(amount): 22 | if amount.find(".") == -1: 23 | amount = int(amount) * 100000000 24 | assert(amount < 10**OUTPUT_VALUE_PRECISION) 25 | return amount 26 | else: 27 | return outputValueToSatoshis(amount) 28 | 29 | 30 | class OmniNode(object): 31 | 32 | def __init__(self, host, port, user, password): 33 | self.omniAccess = JsonRpcCaller(host, port, user, password) 34 | self.txProcessors = { 35 | 0: self.processSimpleSend, 36 | 3: self.processSendOwners, 37 | 4: self.processSendAll, 38 | 20: self.processSellForBitcoin, 39 | 22: self.processAcceptSellForBitcoin, 40 | 50: self.processCreateFixedProperty, 41 | 51: self.processCreateCrowdsaleProperty, 42 | 53: self.processCloseCrowdsale, 43 | 54: self.processCreateManagedProperty, 44 | 55: self.processGrantTokens, 45 | 56: self.processRevokeTokens, 46 | } 47 | 48 | def getBlockCount(self): 49 | return self.omniAccess.call("getblockcount") 50 | 51 | def getBlock(self, height): 52 | blockHash = self.omniAccess.call("getblockhash", [height]) 53 | blockDict = self.omniAccess.call("getblock", [blockHash]) 54 | hashAsNumber = int(blockDict["hash"], base=16) 55 | assert(hashAsNumber < 10**HASH_PRECISION) 56 | block = OmniBlockData(height, hashAsNumber, datetime.utcfromtimestamp(blockDict["time"])) 57 | 58 | transactionHashes = self.omniAccess.call("omni_listblocktransactions", [height]) 59 | # no bulk request due to Omni node freezing 60 | for txHash in transactionHashes: 61 | txInfo = self.omniAccess.call("omni_gettransaction", [txHash]) 62 | if "valid" not in txInfo: 63 | assert(txInfo["type"] == "DEx Purchase") 64 | self.processDexPurchase(txInfo, block) 65 | elif txInfo["valid"]: 66 | txType = txInfo["type_int"] 67 | if txType in self.txProcessors: 68 | self.txProcessors[txType](txInfo, block) 69 | elif txType not in IGNORE_TX_TYPES: 70 | print(txInfo) 71 | print("unknown tx type: {0}".format(txType)) 72 | assert(False) 73 | 74 | return block 75 | 76 | def processSimpleSend(self, txInfo, block): 77 | assert(len(txInfo["referenceaddress"]) < MAX_ADDRESS_LENGTH) 78 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 79 | txData = OmniSimpleSendTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 80 | txInfo["referenceaddress"], omniOutputValueSatoshi(txInfo["amount"]), fee) 81 | block.addSimpleSendTransaction(txData) 82 | 83 | def processSendAll(self, txInfo, block): 84 | assert(len(txInfo["referenceaddress"]) < MAX_ADDRESS_LENGTH) 85 | assert(len(txInfo["sendingaddress"]) < MAX_ADDRESS_LENGTH) 86 | hashAsNumber = int(txInfo["txid"], base=16) 87 | assert(hashAsNumber < 10**HASH_PRECISION) 88 | txTime = datetime.utcfromtimestamp(txInfo["blocktime"]) 89 | for index, send in enumerate(txInfo["subsends"]): 90 | txData = OmniSendAllTransaction(block.blockHash, hashAsNumber, index, txTime, send["propertyid"], 91 | txInfo["sendingaddress"], txInfo["referenceaddress"], omniOutputValueSatoshi(send["amount"]), 92 | omniOutputValueSatoshi(txInfo["fee"])) 93 | block.addSendAllTransaction(txData) 94 | 95 | def processSendOwners(self, txInfo, block): 96 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 97 | txData = OmniSendOwnersTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 98 | omniOutputValueSatoshi(txInfo["amount"]), fee) 99 | block.addSendOwnersTransaction(txData) 100 | 101 | def processSellForBitcoin(self, txInfo, block): 102 | if not txInfo["action"] in SELL_FOR_BITCOIN_ACTIONS: 103 | print(txInfo) 104 | print("unknown action of sell-for-bitcoin tx: {0}".format(txInfo["action"])) 105 | assert(False) 106 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 107 | txData = OmniSellForBitcoinTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 108 | omniOutputValueSatoshi(txInfo["amount"]), fee, omniOutputValueSatoshi(txInfo["feerequired"]), 109 | omniOutputValueSatoshi(txInfo["bitcoindesired"]), SELL_FOR_BITCOIN_ACTIONS[txInfo["action"]]) 110 | block.addSellForBitcoinTransaction(txData) 111 | 112 | def processAcceptSellForBitcoin(self, txInfo, block): 113 | assert(len(txInfo["referenceaddress"]) < MAX_ADDRESS_LENGTH) 114 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 115 | txData = OmniAcceptSellForBitcoinTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 116 | omniOutputValueSatoshi(txInfo["amount"]), fee, txInfo["referenceaddress"]) 117 | block.addAcceptSellForBitcoinTransaction(txData) 118 | 119 | def processCreateFixedProperty(self, txInfo, block): 120 | if not txInfo["propertytype"] in PROPERTY_TYPES: 121 | print(txInfo) 122 | print("unknown property type: {0}".format(txInfo["propertytype"])) 123 | assert(False) 124 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 125 | txData = OmniCreateFixedPropertyTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 126 | omniOutputValueSatoshi(txInfo["amount"]), fee, PROPERTY_TYPES[txInfo["propertytype"]]) 127 | block.addCreateFixedPropertyTransaction(txData) 128 | 129 | def processCreateManagedProperty(self, txInfo, block): 130 | if not txInfo["propertytype"] in PROPERTY_TYPES: 131 | print(txInfo) 132 | print("unknown property type: {0}".format(txInfo["propertytype"])) 133 | assert(False) 134 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 135 | txData = OmniCreateManagedPropertyTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 136 | fee, PROPERTY_TYPES[txInfo["propertytype"]]) 137 | block.addCreateManagedPropertyTransaction(txData) 138 | 139 | def processGrantTokens(self, txInfo, block): 140 | assert(len(txInfo["referenceaddress"]) < MAX_ADDRESS_LENGTH) 141 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 142 | txData = OmniGrantTokensTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 143 | txInfo["referenceaddress"], omniOutputValueSatoshi(txInfo["amount"]), fee) 144 | block.addGrantTokensTransaction(txData) 145 | 146 | def processRevokeTokens(self, txInfo, block): 147 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 148 | txData = OmniRevokeTokensTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, 149 | omniOutputValueSatoshi(txInfo["amount"]), fee) 150 | block.addRevokeTokensTransaction(txData) 151 | 152 | def processCreateCrowdsaleProperty(self, txInfo, block): 153 | if not txInfo["propertytype"] in PROPERTY_TYPES: 154 | print(txInfo) 155 | print("unknown property type: {0}".format(txInfo["propertytype"])) 156 | assert(False) 157 | try: 158 | deadline = datetime.utcfromtimestamp(txInfo["deadline"]) 159 | except ValueError: 160 | deadline = datetime(2100, 1, 1) 161 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 162 | txData = OmniCreateCrowdsalePropertyTransaction(block.blockHash, txHash, txTime, propertyId, sendingAddress, omniOutputValueSatoshi(txInfo["amount"]), 163 | fee, PROPERTY_TYPES[txInfo["propertytype"]], omniOutputValueSatoshi(txInfo["tokensperunit"]), 164 | deadline, txInfo["earlybonus"], txInfo["percenttoissuer"]) 165 | block.addCreateCrowdsalePropertyTransaction(txData) 166 | 167 | def processCloseCrowdsale(self, txInfo, block): 168 | txHash, txTime, sendingAddress, propertyId, fee = self.getCommonTxAttributes(txInfo) 169 | txData = OmniTransactionBase(block.blockHash, txHash, txTime, propertyId, sendingAddress, fee) 170 | block.addCloseCrowdsaleTransaction(txData) 171 | 172 | def processDexPurchase(self, txInfo, block): 173 | assert(len(txInfo["sendingaddress"]) < MAX_ADDRESS_LENGTH) 174 | hashAsNumber = int(txInfo["txid"], base=16) 175 | assert(hashAsNumber < 10**HASH_PRECISION) 176 | txTime = datetime.utcfromtimestamp(txInfo["blocktime"]) 177 | for index, purchase in enumerate(txInfo["purchases"]): 178 | if purchase["valid"]: 179 | assert(len(purchase["referenceaddress"]) < MAX_ADDRESS_LENGTH) 180 | txData = OmniDexPurchaseTransaction(block.blockHash, hashAsNumber, index, txTime, purchase["propertyid"], 181 | txInfo["sendingaddress"], purchase["referenceaddress"], omniOutputValueSatoshi(purchase["amountbought"]), 182 | omniOutputValueSatoshi(purchase["amountpaid"])) 183 | block.addDexPurchaseTransaction(txData) 184 | 185 | def getCommonTxAttributes(self, txInfo): 186 | assert(len(txInfo["sendingaddress"]) < MAX_ADDRESS_LENGTH) 187 | hashAsNumber = int(txInfo["txid"], base=16) 188 | assert(hashAsNumber < 10**HASH_PRECISION) 189 | txTime = datetime.utcfromtimestamp(txInfo["blocktime"]) 190 | return hashAsNumber, txTime, txInfo["sendingaddress"], txInfo["propertyid"], omniOutputValueSatoshi(txInfo["fee"]) 191 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/omni/schema.py: -------------------------------------------------------------------------------- 1 | from coinmetrics.bitsql.constants import HASH_PRECISION, MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION 2 | 3 | 4 | class OmniSchema(object): 5 | 6 | def __init__(self, dbAccess): 7 | self.dbAccess = dbAccess 8 | self.init() 9 | 10 | def getBlocksTableName(self): 11 | return self.blocksTableName 12 | 13 | def getSimpleSendTxTableName(self): 14 | return self.simpleSendTxTableName 15 | 16 | def getSimpleSendTxPrefix(self): 17 | return "simple_send_tx" 18 | 19 | def getSellForBitcoinTxTableName(self): 20 | return self.sellForBitcoinTxTableName 21 | 22 | def getSellForBitcoinTxPrefix(self): 23 | return "sell_for_bitcoin_tx" 24 | 25 | def getAcceptSellForBitcoinTxTableName(self): 26 | return self.acceptSellForBitcoinTxTableName 27 | 28 | def getAcceptSellForBitcoinTxPrefix(self): 29 | return "accept_sell_for_bitcoin_tx" 30 | 31 | def getDexPurchaseTxTableName(self): 32 | return self.dexPurchaseTxTableName 33 | 34 | def getDexPurchaseTxPrefix(self): 35 | return "dex_purchase_tx" 36 | 37 | def getCreateFixedPropertyTxTableName(self): 38 | return self.createFixedPropertyTxTableName 39 | 40 | def getCreateFixedPropertyTxPrefix(self): 41 | return "create_fixed_property_tx" 42 | 43 | def getCreateCrowdsalePropertyTxTableName(self): 44 | return self.createCrowdsalePropertyTxTableName 45 | 46 | def getCreateCrowdsalePropertyTxPrefix(self): 47 | return "create_crowdsale_property_tx" 48 | 49 | def getCloseCrowdsaleTxTableName(self): 50 | return self.closeCrowdsaleTxTableName 51 | 52 | def getCloseCrowdsaleTxPrefix(self): 53 | return "close_crowdsale_tx" 54 | 55 | def getSendOwnersTxTableName(self): 56 | return self.sendOwnersTxTableName 57 | 58 | def getSendOwnersTxPrefix(self): 59 | return "send_owners_tx" 60 | 61 | def getCreateManagedPropertyTxTableName(self): 62 | return self.createManagedPropertyTxTableName 63 | 64 | def getCreateManagedPropertyTxPrefix(self): 65 | return "create_managed_property_tx" 66 | 67 | def getGrantTokensTxTableName(self): 68 | return self.grantTokensTxTableName 69 | 70 | def getGrantTokensTxPrefix(self): 71 | return "grant_tokens_tx" 72 | 73 | def getRevokeTokensTxTableName(self): 74 | return self.revokeTokensTxTableName 75 | 76 | def getRevokeTokensTxPrefix(self): 77 | return "revoke_tokens_tx" 78 | 79 | def getSendAllTxTableName(self): 80 | return self.sendAllTxTableName 81 | 82 | def getSendAllTxPrefix(self): 83 | return "send_all_tx" 84 | 85 | def init(self): 86 | self.blocksTableName = "omni_blocks" 87 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 88 | block_hash DECIMAL(%s) PRIMARY KEY, \ 89 | block_time TIMESTAMP, \ 90 | block_height INTEGER \ 91 | )" % (self.blocksTableName, HASH_PRECISION)) 92 | 93 | self.simpleSendTxTableName = "omni_simple_send_transactions" 94 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 95 | simple_send_tx_hash DECIMAL(%s) PRIMARY KEY, \ 96 | simple_send_tx_block_hash DECIMAL(%s), \ 97 | simple_send_tx_time TIMESTAMP, \ 98 | simple_send_tx_sending_address VARCHAR(%s), \ 99 | simple_send_tx_receiving_address VARCHAR(%s), \ 100 | simple_send_tx_property_id BIGINT, \ 101 | simple_send_tx_amount DECIMAL(%s), \ 102 | simple_send_tx_fee DECIMAL(%s) \ 103 | )" % (self.simpleSendTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 104 | MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 105 | 106 | self.sendAllTxTableName = "omni_send_all_transactions" 107 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 108 | send_all_tx_hash DECIMAL(%s), \ 109 | send_all_tx_index INTEGER, \ 110 | send_all_tx_block_hash DECIMAL(%s), \ 111 | send_all_tx_time TIMESTAMP, \ 112 | send_all_tx_sending_address VARCHAR(%s), \ 113 | send_all_tx_receiving_address VARCHAR(%s), \ 114 | send_all_tx_property_id BIGINT, \ 115 | send_all_tx_amount DECIMAL(%s), \ 116 | send_all_tx_fee DECIMAL(%s), \ 117 | PRIMARY KEY(send_all_tx_hash, send_all_tx_index) \ 118 | )" % (self.sendAllTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 119 | MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 120 | 121 | self.sendOwnersTxTableName = "omni_send_owners_transactions" 122 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 123 | send_owners_tx_hash DECIMAL(%s) PRIMARY KEY, \ 124 | send_owners_tx_block_hash DECIMAL(%s), \ 125 | send_owners_tx_time TIMESTAMP, \ 126 | send_owners_tx_sending_address VARCHAR(%s), \ 127 | send_owners_tx_property_id BIGINT, \ 128 | send_owners_tx_amount DECIMAL(%s), \ 129 | send_owners_tx_fee DECIMAL(%s) \ 130 | )" % (self.sendOwnersTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 131 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 132 | 133 | self.sellForBitcoinTxTableName = "omni_sell_for_bitcoin_transactions" 134 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 135 | sell_for_bitcoin_tx_hash DECIMAL(%s) PRIMARY KEY, \ 136 | sell_for_bitcoin_tx_block_hash DECIMAL(%s), \ 137 | sell_for_bitcoin_tx_time TIMESTAMP, \ 138 | sell_for_bitcoin_tx_sending_address VARCHAR(%s), \ 139 | sell_for_bitcoin_tx_property_id BIGINT, \ 140 | sell_for_bitcoin_tx_amount DECIMAL(%s), \ 141 | sell_for_bitcoin_tx_fee DECIMAL(%s), \ 142 | sell_for_bitcoin_tx_fee_required DECIMAL(%s), \ 143 | sell_for_bitcoin_tx_bitcoin_desired DECIMAL(%s), \ 144 | sell_for_bitcoin_tx_action SMALLINT \ 145 | )" % (self.sellForBitcoinTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 146 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 147 | 148 | self.acceptSellForBitcoinTxTableName = "omni_accept_sell_for_bitcoin_transactions" 149 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 150 | accept_sell_for_bitcoin_tx_hash DECIMAL(%s) PRIMARY KEY, \ 151 | accept_sell_for_bitcoin_tx_block_hash DECIMAL(%s), \ 152 | accept_sell_for_bitcoin_tx_time TIMESTAMP, \ 153 | accept_sell_for_bitcoin_tx_sending_address VARCHAR(%s), \ 154 | accept_sell_for_bitcoin_tx_property_id BIGINT, \ 155 | accept_sell_for_bitcoin_tx_amount DECIMAL(%s), \ 156 | accept_sell_for_bitcoin_tx_fee DECIMAL(%s), \ 157 | accept_sell_for_bitcoin_tx_receiving_address VARCHAR(%s) \ 158 | )" % (self.acceptSellForBitcoinTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 159 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION, MAX_ADDRESS_LENGTH)) 160 | 161 | self.dexPurchaseTxTableName = "omni_dex_purchase_transactions" 162 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 163 | dex_purchase_tx_hash DECIMAL(%s), \ 164 | dex_purchase_tx_index INTEGER, \ 165 | dex_purchase_tx_block_hash DECIMAL(%s), \ 166 | dex_purchase_tx_time TIMESTAMP, \ 167 | dex_purchase_tx_sending_address VARCHAR(%s), \ 168 | dex_purchase_tx_property_id BIGINT, \ 169 | dex_purchase_tx_amount_bought DECIMAL(%s), \ 170 | dex_purchase_tx_amount_paid DECIMAL(%s), \ 171 | dex_purchase_tx_receiving_address VARCHAR(%s), \ 172 | PRIMARY KEY(dex_purchase_tx_hash, dex_purchase_tx_index) \ 173 | )" % (self.dexPurchaseTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 174 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION, MAX_ADDRESS_LENGTH)) 175 | 176 | self.createFixedPropertyTxTableName = "omni_create_fixed_property_transactions" 177 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 178 | create_fixed_property_tx_hash DECIMAL(%s) PRIMARY KEY, \ 179 | create_fixed_property_tx_block_hash DECIMAL(%s), \ 180 | create_fixed_property_tx_time TIMESTAMP, \ 181 | create_fixed_property_tx_sending_address VARCHAR(%s), \ 182 | create_fixed_property_tx_property_id BIGINT, \ 183 | create_fixed_property_tx_property_type SMALLINT, \ 184 | create_fixed_property_tx_amount DECIMAL(%s), \ 185 | create_fixed_property_tx_fee DECIMAL(%s) \ 186 | )" % (self.createFixedPropertyTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 187 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 188 | 189 | self.createManagedPropertyTxTableName = "omni_create_managed_property_transactions" 190 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 191 | create_managed_property_tx_hash DECIMAL(%s) PRIMARY KEY, \ 192 | create_managed_property_tx_block_hash DECIMAL(%s), \ 193 | create_managed_property_tx_time TIMESTAMP, \ 194 | create_managed_property_tx_sending_address VARCHAR(%s), \ 195 | create_managed_property_tx_property_id BIGINT, \ 196 | create_managed_property_tx_property_type SMALLINT, \ 197 | create_managed_property_tx_fee DECIMAL(%s) \ 198 | )" % (self.createManagedPropertyTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 199 | OUTPUT_VALUE_PRECISION)) 200 | 201 | self.grantTokensTxTableName = "omni_grant_tokens_transactions" 202 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 203 | grant_tokens_tx_hash DECIMAL(%s) PRIMARY KEY, \ 204 | grant_tokens_tx_block_hash DECIMAL(%s), \ 205 | grant_tokens_tx_time TIMESTAMP, \ 206 | grant_tokens_tx_sending_address VARCHAR(%s), \ 207 | grant_tokens_tx_receiving_address VARCHAR(%s), \ 208 | grant_tokens_tx_property_id BIGINT, \ 209 | grant_tokens_tx_amount DECIMAL(%s), \ 210 | grant_tokens_tx_fee DECIMAL(%s) \ 211 | )" % (self.grantTokensTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 212 | MAX_ADDRESS_LENGTH, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 213 | 214 | self.revokeTokensTxTableName = "omni_revoke_tokens_transactions" 215 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 216 | revoke_tokens_tx_hash DECIMAL(%s) PRIMARY KEY, \ 217 | revoke_tokens_tx_block_hash DECIMAL(%s), \ 218 | revoke_tokens_tx_time TIMESTAMP, \ 219 | revoke_tokens_tx_sending_address VARCHAR(%s), \ 220 | revoke_tokens_tx_property_id BIGINT, \ 221 | revoke_tokens_tx_amount DECIMAL(%s), \ 222 | revoke_tokens_tx_fee DECIMAL(%s) \ 223 | )" % (self.revokeTokensTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 224 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 225 | 226 | self.createCrowdsalePropertyTxTableName = "omni_create_crowdsale_property_transactions" 227 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 228 | create_crowdsale_property_tx_hash DECIMAL(%s) PRIMARY KEY, \ 229 | create_crowdsale_property_tx_block_hash DECIMAL(%s), \ 230 | create_crowdsale_property_tx_time TIMESTAMP, \ 231 | create_crowdsale_property_tx_sending_address VARCHAR(%s), \ 232 | create_crowdsale_property_tx_property_id BIGINT, \ 233 | create_crowdsale_property_tx_property_type SMALLINT, \ 234 | create_crowdsale_property_tx_amount DECIMAL(%s), \ 235 | create_crowdsale_property_tx_deadline TIMESTAMP, \ 236 | create_crowdsale_property_tx_issuer_percent SMALLINT, \ 237 | create_crowdsale_property_tx_bonus SMALLINT, \ 238 | create_crowdsale_property_tx_fee DECIMAL(%s), \ 239 | create_crowdsale_property_tx_tokens_per_unit DECIMAL(%s) \ 240 | )" % (self.createCrowdsalePropertyTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 241 | OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION, OUTPUT_VALUE_PRECISION)) 242 | 243 | self.closeCrowdsaleTxTableName = "omni_close_crowdsale_transactions" 244 | self.dbAccess.queryNoReturnCommit("CREATE TABLE IF NOT EXISTS %s (\ 245 | close_crowdsale_tx_hash DECIMAL(%s) PRIMARY KEY, \ 246 | close_crowdsale_tx_block_hash DECIMAL(%s), \ 247 | close_crowdsale_tx_time TIMESTAMP, \ 248 | close_crowdsale_tx_sending_address VARCHAR(%s), \ 249 | close_crowdsale_tx_property_id BIGINT, \ 250 | close_crowdsale_tx_fee DECIMAL(%s) \ 251 | )" % (self.closeCrowdsaleTxTableName, HASH_PRECISION, HASH_PRECISION, MAX_ADDRESS_LENGTH, 252 | OUTPUT_VALUE_PRECISION)) 253 | 254 | borderBlock = self.dbAccess.queryReturnAll("SELECT * FROM " + self.blocksTableName + " WHERE block_height=%s", (252316,)) 255 | if len(borderBlock) == 0: 256 | self.dbAccess.queryNoReturnCommit("INSERT INTO " + self.blocksTableName + " (block_hash, block_height) VALUES (%s, %s)", 257 | (int("0000000000000000bfccb13b63c06cd5ee7075d2f5cc08b1b2f8e98365f5538e", 16), 252316)) 258 | 259 | def drop(self): 260 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.closeCrowdsaleTxTableName,)) 261 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.createCrowdsalePropertyTxTableName,)) 262 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.revokeTokensTxTableName,)) 263 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.grantTokensTxTableName,)) 264 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.createManagedPropertyTxTableName,)) 265 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.createFixedPropertyTxTableName,)) 266 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.dexPurchaseTxTableName,)) 267 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.acceptSellForBitcoinTxTableName,)) 268 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.sellForBitcoinTxTableName,)) 269 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.sendOwnersTxTableName,)) 270 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.sendAllTxTableName,)) 271 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.simpleSendTxTableName,)) 272 | self.dbAccess.queryNoReturnCommit("DROP TABLE IF EXISTS %s" % (self.blocksTableName,)) 273 | 274 | def addIndexes(self): 275 | self.dbAccess.queryNoReturnCommit("CREATE INDEX omni_simple_send_transactions_time_index ON %s(simple_send_tx_time)" % self.simpleSendTxTableName) 276 | self.dbAccess.queryNoReturnCommit("CREATE INDEX omni_send_owners_transactions_time_index ON %s(send_owners_tx_time)" % self.sendOwnersTxTableName) 277 | self.dbAccess.queryNoReturnCommit("CREATE INDEX omni_grant_tokens_transactions_time_index ON %s(grant_tokens_tx_time)" % self.grantTokensTxTableName) 278 | self.dbAccess.queryNoReturnCommit("CREATE INDEX omni_revoke_tokens_transactions_time_index ON %s(revoke_tokens_tx_time)" % self.revokeTokensTxTableName) 279 | self.dbAccess.queryNoReturnCommit("CREATE INDEX omni_send_all_transactions_time_index ON %s(send_all_tx_time)" % self.sendAllTxTableName) 280 | 281 | def dropIndexes(self): 282 | self.dbAccess.queryNoReturnCommit("DROP INDEX omni_send_all_transactions_time_index") 283 | self.dbAccess.queryNoReturnCommit("DROP INDEX omni_revoke_tokens_transactions_time_index") 284 | self.dbAccess.queryNoReturnCommit("DROP INDEX omni_grant_tokens_transactions_time_index") 285 | self.dbAccess.queryNoReturnCommit("DROP INDEX omni_send_owners_transactions_time_index") 286 | self.dbAccess.queryNoReturnCommit("DROP INDEX omni_simple_send_transactions_time_index") 287 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/node.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | from coinmetrics.utils import bech32 3 | from coinmetrics.utils.jsonrpc import JsonRpcCaller 4 | from coinmetrics.bitsql.constants import * 5 | from coinmetrics.bitsql.data import * 6 | from datetime import datetime 7 | 8 | 9 | def outputValueToSatoshis(outputValue): 10 | if type(outputValue) != int: 11 | assert(float(outputValue) >= 0.0) 12 | valuePieces = outputValue.split(".") 13 | assert(len(valuePieces) == 2) 14 | fractionDigitCount = len(valuePieces[1]) 15 | digits = int("".join(valuePieces)) 16 | value = digits * 10**(8 - fractionDigitCount) 17 | naiveValue = float(outputValue) * 100000000.0 18 | assert(abs(value - naiveValue) <= 1024) 19 | else: 20 | value = outputValue * 100000000 21 | assert(value < 10**OUTPUT_VALUE_PRECISION) 22 | assert(value >= 0) 23 | return value 24 | 25 | 26 | class BitcoinNodeBase(object): 27 | 28 | def __init__(self, host, port, user, password): 29 | self.bitcoinAccess = JsonRpcCaller(host, port, user, password) 30 | 31 | def __repr__(self): 32 | return self.bitcoinAccess.getAddress() 33 | 34 | def getBlockCount(self): 35 | return self.bitcoinAccess.call("getblockcount") 36 | 37 | def supportsGetBlock2(self): 38 | return True 39 | 40 | def getBlock(self, height): 41 | blockHash = self.bitcoinAccess.call("getblockhash", [height]) 42 | if self.supportsGetBlock2(): 43 | blockDict = self.bitcoinAccess.call("getblock", [blockHash, 2]) 44 | else: 45 | blockDict = self.bitcoinAccess.call("getblock", [blockHash]) 46 | block = self.initBlock(blockDict) 47 | 48 | txDicts = self.getBlockTransactions(blockDict, block) 49 | txIndex = 0 50 | for txDict in txDicts: 51 | transaction = self.processTransaction(txDict, txIndex, block) 52 | block.addTransaction(transaction) 53 | txIndex += 1 54 | 55 | return block 56 | 57 | def getBlockTransactions(self, blockDict, blockData): 58 | if self.supportsGetBlock2(): 59 | txDicts = blockDict["tx"] 60 | result = [] 61 | for tx in txDicts: 62 | if not self.excludeTransaction(tx["txid"], blockData): 63 | result.append(tx) 64 | return result 65 | else: 66 | hashes = [] 67 | for txHash in blockDict["tx"]: 68 | if not self.excludeTransaction(txHash, blockData): 69 | hashes.append(txHash) 70 | if len(hashes) > 0: 71 | return self.bitcoinAccess.bulkCall(("getrawtransaction", [txHash, 1]) for txHash in hashes) 72 | else: 73 | return [] 74 | 75 | def initBlock(self, blockDict): 76 | hashAsNumber = int(blockDict["hash"], base=16) 77 | assert(hashAsNumber < 10**HASH_PRECISION) 78 | chainWorkAsNumber = self.getBlockChainWork(blockDict) 79 | assert(chainWorkAsNumber < 10**CHAINWORK_PRECISION) 80 | blockTime = datetime.utcfromtimestamp(blockDict["time"]) 81 | medianTime = self.getBlockMedianTime(blockDict) 82 | blockSize = int(blockDict["size"]) 83 | difficulty = float(blockDict["difficulty"]) 84 | height = int(blockDict["height"]) 85 | return BitcoinBlockData(height, hashAsNumber, chainWorkAsNumber, blockTime, medianTime, blockSize, difficulty) 86 | 87 | def getBlockChainWork(self, blockDict): 88 | return int(blockDict["chainwork"], base=16) 89 | 90 | def getBlockMedianTime(self, blockDict): 91 | return datetime.utcfromtimestamp(blockDict["mediantime"]) 92 | 93 | def processTransaction(self, txDict, txIndex, blockData): 94 | transaction = self.initTransaction(txDict, txIndex, blockData) 95 | self.processInputs(transaction, txDict, txIndex, blockData) 96 | self.processOutputs(transaction, txDict, txIndex) 97 | if self.transactionIsCoinbase(txDict, txIndex, blockData) and self.shouldSaveCoinbaseScript(txDict, txIndex, blockData): 98 | assert len(txDict["vin"]) == 1 99 | transaction.setCoinbaseScript(txDict["vin"][0]["coinbase"]) 100 | return transaction 101 | 102 | def initTransaction(self, txDict, txIndex, blockData): 103 | txSize = self.getTransactionSize(txDict) 104 | txHashAsNumber = int(txDict["txid"], base=16) 105 | assert(txHashAsNumber < 10**HASH_PRECISION) 106 | coinbase = self.transactionIsCoinbase(txDict, txIndex, blockData) 107 | return BitcoinTransactionData(txHashAsNumber, txSize, blockData.blockTime, blockData.blockMedianTime, coinbase) 108 | 109 | def getTransactionSize(self, txDict): 110 | return int(txDict["size"]) 111 | 112 | def transactionIsCoinbase(self, txDict, txIndex, blockData): 113 | for inputDict in txDict["vin"]: 114 | if "coinbase" in inputDict: 115 | return True 116 | return False 117 | 118 | def excludeTransaction(self, txHash, blockData): 119 | return False 120 | 121 | def shouldSaveCoinbaseScript(self, txDict, txIndex, blockData): 122 | return True 123 | 124 | def processInputs(self, transaction, txDict, txIndex, blockData): 125 | if self.shouldProcessTxInputs(transaction, txDict, txIndex): 126 | index = 0 127 | for inputDict in txDict["vin"]: 128 | self.processInput(transaction, inputDict, index) 129 | index += 1 130 | 131 | def shouldProcessTxInputs(self, transaction, txDict, txIndex): 132 | return not transaction.coinbase 133 | 134 | def processInput(self, transaction, inputDict, inputIndex): 135 | inputTxHashAsNumber = int(inputDict["txid"], base=16) 136 | outputIndex = int(inputDict["vout"]) 137 | scriptSig = inputDict["scriptSig"]["hex"] 138 | transaction.addInput((inputTxHashAsNumber, outputIndex, scriptSig)) 139 | 140 | def processOutputs(self, transaction, txDict, txIndex): 141 | outputIndex = 0 142 | for outputDict in txDict["vout"]: 143 | self.processOutput(transaction, outputDict, outputIndex) 144 | outputIndex += 1 145 | 146 | def processOutput(self, transaction, outputDict, outputIndex): 147 | if not outputDict["scriptPubKey"]["type"] in OUTPUT_TYPES: 148 | print(outputDict["scriptPubKey"]["type"]) 149 | assert(outputDict["scriptPubKey"]["type"] in OUTPUT_TYPES) 150 | outputType = OUTPUT_TYPES[outputDict["scriptPubKey"]["type"]] 151 | 152 | if "addresses" in outputDict["scriptPubKey"]: 153 | addresses = outputDict["scriptPubKey"]["addresses"] 154 | else: 155 | addresses = [] 156 | 157 | if outputType in [OUTPUT_TYPES["nonstandard"], OUTPUT_TYPES["nulldata"], OUTPUT_TYPES["multisig"]]: 158 | pass 159 | elif outputType == OUTPUT_TYPES["pubkey"]: 160 | if "addresses" in outputDict["scriptPubKey"]: 161 | assert(len(addresses) == 1) 162 | elif outputType in [OUTPUT_TYPES["witness_v0_keyhash"], OUTPUT_TYPES["witness_v0_scripthash"]]: 163 | self.processSegwitOutput(outputDict, addresses) 164 | else: 165 | assert len(addresses) == 1 166 | 167 | for address in addresses: 168 | assert(len(address) <= MAX_ADDRESS_LENGTH) 169 | assert(len(address) > 0) 170 | 171 | scriptHex = outputDict["scriptPubKey"]["hex"] 172 | value = outputValueToSatoshis(outputDict["value"]) 173 | transaction.addOutput((outputIndex, outputType, addresses, scriptHex, value)) 174 | 175 | def processSegwitOutput(self, outputDict, addresses): 176 | assert len(addresses) == 1 177 | 178 | 179 | class BitcoinNode(BitcoinNodeBase): 180 | 181 | def excludeTransaction(self, txHash, blockData): 182 | txHash = int(txHash, base=16) 183 | if (blockData.blockHeight == 91842 and txHash == 96714513404922958314624647138985365973445136781445592526454084119790809023897) or (blockData.blockHeight == 91880 and txHash == 103012905635419619419213554242971767587813086721766557307841598175607561106536): 184 | return True 185 | else: 186 | return False 187 | 188 | 189 | class BitcoinGoldNode(BitcoinNode): 190 | 191 | def processSegwitOutput(self, outputDict, addresses): 192 | assert len(addresses) <= 1 193 | if len(addresses) == 0: 194 | outputType = OUTPUT_TYPES[outputDict["scriptPubKey"]["type"]] 195 | prefix = "wkh_" if outputType == OUTPUT_TYPES["witness_v0_keyhash"] else "wsh_" 196 | addresses.append(bech32.encode(prefix, 0, binascii.unhexlify(outputDict["scriptPubKey"]["hex"][4:]))) 197 | 198 | 199 | class BitcoinSvNode(BitcoinNode): 200 | 201 | def supportsGetBlock2(self): 202 | return False 203 | 204 | 205 | # joinsplits: 3306, 3308 206 | # sapling payments: b17707e9daff8c39bba43c3d0986c231d9bb7f8c62dd961d61f79def6fd8b85d 5582ac303b6ecbd2af0c94a6cb6259a237818defceaccfdf961b5eefa79eb5eb 207 | class ZcashNode(BitcoinNodeBase): 208 | 209 | def getTransactionSize(self, txDict): 210 | # Zcash node doesn't report tx size 211 | return 0 212 | 213 | def getBlockMedianTime(self, blockDict): 214 | # ZCash node doesn't report median block time 215 | return datetime(1970, 1, 1) 216 | 217 | def initTransaction(self, txDict, txIndex, blockData): 218 | data = super(ZcashNode, self).initTransaction(txDict, txIndex, blockData) 219 | return ZcashTransactionData(data.txHash, data.txSize, data.txTime, data.txMedianTime, data.coinbase) 220 | 221 | def processTransaction(self, txDict, txIndex, blockData): 222 | transaction = super(ZcashNode, self).processTransaction(txDict, txIndex, blockData) 223 | for joinSplit in txDict["vjoinsplit"]: 224 | valueOld = outputValueToSatoshis(joinSplit["vpub_old"]) 225 | valueNew = outputValueToSatoshis(joinSplit["vpub_new"]) 226 | transaction.addJoinSplit((valueOld, valueNew)) 227 | 228 | if ("vShieldedSpend" in txDict) or ("vShieldedOutput" in txDict): 229 | shieldedInputCount = len(txDict["vShieldedSpend"]) 230 | shieldedOutputCount = len(txDict["vShieldedOutput"]) 231 | if shieldedInputCount > 0 or shieldedOutputCount > 0: 232 | valueBalanceString = txDict["valueBalance"] 233 | if valueBalanceString[0] == "-": 234 | valueBalance = -outputValueToSatoshis(txDict["valueBalance"][1:]) 235 | else: 236 | valueBalance = outputValueToSatoshis(txDict["valueBalance"]) 237 | transaction.addSaplingPayment((shieldedInputCount, shieldedOutputCount, valueBalance)) 238 | return transaction 239 | 240 | 241 | class BitcoinPrivateNode(ZcashNode): 242 | 243 | def supportsGetBlock2(self): 244 | return False 245 | 246 | def processSegwitOutput(self, outputDict, addresses): 247 | assert len(addresses) <= 1 248 | if len(addresses) == 0: 249 | outputType = OUTPUT_TYPES[outputDict["scriptPubKey"]["type"]] 250 | prefix = "wkh_" if outputType == OUTPUT_TYPES["witness_v0_keyhash"] else "wsh_" 251 | addresses.append(bech32.encode(prefix, 0, binascii.unhexlify(outputDict["scriptPubKey"]["hex"][4:]))) 252 | 253 | 254 | class DogecoinNode(BitcoinNodeBase): 255 | 256 | def getTransactionSize(self, txDict): 257 | # Dogecoin node doesn't report tx size 258 | return 0 259 | 260 | def supportsGetBlock2(self): 261 | return False 262 | 263 | def getBlockMedianTime(self, blockDict): 264 | # dogecoin node doesn't report median block time 265 | return datetime(1970, 1, 1) 266 | 267 | 268 | # zerocoin mints: 880002 269 | # zerocoin spends: 880029 270 | # zpiv staking: 1156138 271 | class PivxNode(BitcoinNode): 272 | 273 | def transactionIsCoinbase(self, txDict, txIndex, blockData): 274 | isCoinbase = super(PivxNode, self).transactionIsCoinbase(txDict, txIndex, blockData) 275 | return isCoinbase or (blockData.blockHeight > 259200 and txIndex == 1) 276 | 277 | def getTransactionSize(self, txDict): 278 | # PIVX node doesn't report tx size 279 | return 0 280 | 281 | def supportsGetBlock2(self): 282 | return False 283 | 284 | def getBlockMedianTime(self, blockDict): 285 | # PIVX node doesn't report median block time 286 | return datetime(1970, 1, 1) 287 | 288 | def initTransaction(self, txDict, txIndex, blockData): 289 | data = super(PivxNode, self).initTransaction(txDict, txIndex, blockData) 290 | return PivxTransactionData(data.txHash, data.txSize, data.txTime, data.txMedianTime, data.coinbase) 291 | 292 | def processInput(self, transaction, inputDict, index): 293 | inputTxHashAsNumber = int(inputDict["txid"], base=16) 294 | if inputTxHashAsNumber == 0: 295 | amount = self.bitcoinAccess.call("getspentzerocoinamount", [transaction.getHashString(), index]) 296 | transaction.addZerocoinSpend(outputValueToSatoshis(amount)) 297 | else: 298 | return super(PivxNode, self).processInput(transaction, inputDict, index) 299 | 300 | def shouldProcessTxInputs(self, transaction, txDict, txIndex): 301 | return not (transaction.coinbase and txIndex == 0) 302 | 303 | def processOutput(self, transaction, outputDict, outputIndex): 304 | if outputDict["scriptPubKey"]["type"] == "zerocoinmint": 305 | transaction.addZerocoinMint(outputValueToSatoshis(outputDict["value"])) 306 | else: 307 | return super(PivxNode, self).processOutput(transaction, outputDict, outputIndex) 308 | 309 | def shouldSaveCoinbaseScript(self, txDict, txIndex, blockData): 310 | return txIndex == 0 311 | 312 | 313 | class VergeNode(BitcoinNodeBase): 314 | 315 | def getBlockChainWork(self, blockDict): 316 | return 0 317 | 318 | def getTransactionSize(self, txDict): 319 | return 0 320 | 321 | def supportsGetBlock2(self): 322 | return False 323 | 324 | def getBlockMedianTime(self, blockDict): 325 | return datetime(1970, 1, 1) 326 | 327 | 328 | class DecredNode(BitcoinNodeBase): 329 | 330 | def __init__(self, host, port, user, password): 331 | self.bitcoinAccess = JsonRpcCaller(host, port, user, password, tls=True, tlsVerify=False) 332 | self.duplicateTransactions = { 333 | ("752db9a8fa003bb7fbacad57627001973b6b95500cb0aab0dfe406483467ac10", 83822), 334 | ("8521fb31190eacd9aaf4b27862ef88e55c6a6de8b66f241733800ddaa0b27e1b", 83912), 335 | ("0042f3ec6660fdfb566bec6147d28e277d04c9b117d9f656f433d14b6c00167b", 83912), 336 | ("1593791a34585557554ec85e49e39afda330dd745e7a82fc0efb0003bfae71a9", 83912), 337 | ("d075e4a96ffb0cae0be81ad9c9e3a77ad491d1586a61a533a48460891317fb73", 83912), 338 | ("cdea5df0ca44027336ec85c2bd3da792237fba1c582599494cda6780204ae128", 83912), 339 | ("63b1a7dcffe67fcdd962f86f467a122ac48fa31667a71c6e7f4c138feabdb43f", 87859), 340 | ("bf7a1a036ab4a79b6ef6062fb7e56e509660055e2a99bb346867b132d4fb5da8", 88437), 341 | ("70f9f77fa969609c3928e024caaa872a99e07a4a6c2d05732f30b34df3935333", 88724), 342 | ("e5b88eaaacedd86de1a670fd47ab705dbdad767df09bf0902fcbceab8987a28f", 90215), 343 | } 344 | self.missingInfoTransactions = { 345 | "f3b0e919ccac9b78b7fb15cb3e5e1f93f7dc0ed05080de3e8d67577f38b2c03d", 346 | "50893c9acf2776a8e7cee7674431142cd41292370387a0b8e79bd8d577a5c55f", 347 | "3d3c6581d9092db691e1624ed67a9647edab0027843f3583776a25d2dd3223e4", 348 | "daa402d95ac86763e02d8e6eaac227fc66ae9a13de27d4782758f37c178b72d8", 349 | "67f78257c355a86d8ebe2b0a5c053e73a26c9cdb43838085c21c1d715fda3487", 350 | "5b4e88991b1a217ac2763be68da209b113c5e7d613d14cd9298a4e64d8596589", 351 | "62dcaa62b47ddb6b5b1a5458b40ab2bad4160f668e2343a43130420cfd8cf3ec", 352 | "2d045cf863a3f813fbe513e5a3d5b24ee447c01d43d8f6d840d2804a053b6d20", 353 | "8a8a5903aa7bd8ea735e0219e216a0738858c9ae3ad4c436ef20168b9d9c1922", 354 | "b419240eab69242421a9183a0f5b3d6a4129c17cd0556b318dbe112df6cd866c", 355 | "f75dca3407378e73ef14d089dfb958a1e69578575242faffd9a02b641a547818", 356 | "e6171883344e3c9dbbfc544e451becd408233318c071949979d67f94d122e17f", 357 | "1310c7b01ce435598e30e74c4407431c27b5bdc03283a5e831f26d3a79497b4c", 358 | "a68d110d662d04aa3f4100894edccaddbd26252529a7d399c196f6667567e6dc", 359 | "69cff27b13002aa81e41598b844f45c32b1d4cb1b23ff82b53456ded49140a73", 360 | "48de4e70c6f556a7c33f18d7085f937855c4d3b285c9b8b16f348cb44d1d4e82", 361 | } 362 | 363 | def getBlockChainWork(self, blockDict): 364 | return 0 365 | 366 | def getTransactionSize(self, txDict): 367 | return 0 368 | 369 | def supportsGetBlock2(self): 370 | return False 371 | 372 | def getBlockMedianTime(self, blockDict): 373 | return datetime(1970, 1, 1) 374 | 375 | def excludeTransaction(self, txHash, blockData): 376 | return (txHash, blockData.blockHeight) in self.duplicateTransactions or txHash in self.missingInfoTransactions 377 | 378 | def getBlockTransactions(self, blockDict, blockData): 379 | baseTxs = super(DecredNode, self).getBlockTransactions(blockDict, blockData) 380 | if "stx" in blockDict: 381 | stakeTxs = self.bitcoinAccess.bulkCall(("getrawtransaction", [txHash, 1]) for txHash in blockDict["stx"]) 382 | return baseTxs + stakeTxs 383 | else: 384 | return baseTxs 385 | 386 | def initTransaction(self, txDict, txIndex, blockData): 387 | data = super(DecredNode, self).initTransaction(txDict, txIndex, blockData) 388 | return DecredTransactionData(data.txHash, data.txSize, data.txTime, data.txMedianTime, data.coinbase) 389 | 390 | def processTransaction(self, txDict, txIndex, blockData): 391 | result = super(DecredNode, self).processTransaction(txDict, txIndex, blockData) 392 | assert(not (result.vote and result.ticket)) 393 | return result 394 | 395 | def processInput(self, transaction, inputDict, index): 396 | if "stakebase" in inputDict: 397 | transaction.vote = True 398 | if "txid" in inputDict: 399 | return super(DecredNode, self).processInput(transaction, inputDict, index) 400 | else: 401 | return super(DecredNode, self).processInput(transaction, inputDict, index) 402 | 403 | def processOutput(self, transaction, outputDict, outputIndex): 404 | outputType = outputDict["scriptPubKey"]["type"] 405 | if outputType == "sstxcommitment" or outputType == "stakerevoke" or outputType == "stakesubmission": 406 | transaction.ticket = True 407 | return super(DecredNode, self).processOutput(transaction, outputDict, outputIndex) 408 | 409 | 410 | class DashNode(BitcoinNodeBase): 411 | 412 | def supportsGetBlock2(self): 413 | return False 414 | -------------------------------------------------------------------------------- /coinmetrics/bitsql/query.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from dateutil.relativedelta import relativedelta 3 | 4 | 5 | class BitcoinQuery(object): 6 | 7 | def __init__(self, dbAccess, schema): 8 | self.dbAccess = dbAccess 9 | self.schema = schema 10 | self.asset = schema.getAsset() 11 | self.blocksTable = schema.getBlocksTableName() 12 | self.txTable = schema.getTransactionsTableName() 13 | self.outputsTable = schema.getOutputsTableName() 14 | 15 | def getSchema(self): 16 | return self.schema 17 | 18 | def getAsset(self): 19 | return self.schema.getAsset() 20 | 21 | def getBlockHeight(self): 22 | result = self.dbAccess.queryReturnOne("SELECT max(block_height) FROM %s" % self.schema.getBlocksTableName())[0] 23 | return result 24 | 25 | def getBlockTime(self, height): 26 | result = self.dbAccess.queryReturnOne("SELECT block_time FROM %s WHERE block_height=%d" % (self.schema.getBlocksTableName(), height)) 27 | return result[0] 28 | 29 | def getMinBlockTime(self): 30 | result = self.dbAccess.queryReturnOne("SELECT min(block_time) FROM %s" % self.schema.getBlocksTableName()) 31 | return result[0] if result is not None else 0 32 | 33 | def getMaxBlockTime(self): 34 | result = self.dbAccess.queryReturnOne("SELECT max(block_time) FROM %s" % self.schema.getBlocksTableName()) 35 | return result[0] if result is not None else 0 36 | 37 | def getBlockHeightsBefore(self, maxTime): 38 | result = self.dbAccess.queryReturnAll("SELECT block_height FROM " + self.blocksTable + " WHERE block_time <= %s", (maxTime,)) 39 | return [row[0] for row in result] 40 | 41 | def getTransactionsBetween(self, minDate, maxDate): 42 | return self.dbAccess.queryReturnAll("\ 43 | SELECT \ 44 | * \ 45 | FROM " + self.txTable + " WHERE \ 46 | (tx_time >= %s AND tx_time < %s)", (minDate, maxDate)) 47 | 48 | def getInputsBetween(self, minDate, maxDate): 49 | return self.dbAccess.queryReturnAll("\ 50 | SELECT \ 51 | * \ 52 | FROM " + self.outputsTable + " WHERE \ 53 | (output_time_spent >= %s AND output_time_spent < %s)", (minDate, maxDate)) 54 | 55 | def getInputTxHashAndAddressesBetween(self, minDate, maxDate): 56 | return self.dbAccess.queryReturnAll("\ 57 | SELECT \ 58 | output_spending_tx_hash, output_addresses \ 59 | FROM " + self.outputsTable + " WHERE \ 60 | (output_time_spent >= %s AND output_time_spent < %s)", (minDate, maxDate)) 61 | 62 | def getAverageDifficultyBetween(self, minDate, maxDate): 63 | result = self.dbAccess.queryReturnOne("\ 64 | SELECT \ 65 | AVG(block_difficulty) \ 66 | FROM " + self.blocksTable + " WHERE \ 67 | block_time >= %s AND block_time < %s", (minDate, maxDate)) 68 | return result[0] 69 | 70 | def getBlockSizeBetween(self, minDate, maxDate): 71 | result = self.dbAccess.queryReturnOne("\ 72 | SELECT \ 73 | SUM(block_size) \ 74 | FROM " + self.blocksTable + " WHERE \ 75 | block_time >= %s AND block_time < %s", (minDate, maxDate)) 76 | return result[0] if result[0] is not None else 0 77 | 78 | def getBlockCountBetween(self, minDate, maxDate): 79 | result = self.dbAccess.queryReturnOne("\ 80 | SELECT \ 81 | COUNT(*) \ 82 | FROM " + self.blocksTable + " WHERE \ 83 | block_time >= %s AND block_time < %s", (minDate, maxDate)) 84 | return result[0] if result[0] is not None else 0 85 | 86 | def getTxCountBetween(self, minDate, maxDate): 87 | result = self.dbAccess.queryReturnOne("\ 88 | SELECT \ 89 | COUNT(*) \ 90 | FROM " + self.txTable + " WHERE \ 91 | tx_coinbase=false AND (tx_time >= %s AND tx_time < %s)", (minDate, maxDate)) 92 | return result[0] 93 | 94 | def getOutputVolumeBetween(self, minDate, maxDate): 95 | result = self.dbAccess.queryReturnOne("WITH \ 96 | o AS (\ 97 | SELECT \ 98 | output_tx_hash, \ 99 | output_index, \ 100 | output_value_satoshi, \ 101 | output_addresses \ 102 | FROM " + self.outputsTable + " WHERE \ 103 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 104 | i AS (\ 105 | SELECT \ 106 | output_spending_tx_hash, \ 107 | output_index, \ 108 | output_addresses \ 109 | FROM " + self.outputsTable + " WHERE \ 110 | (output_time_spent >= %s AND output_time_spent < %s)), \ 111 | t AS (\ 112 | SELECT \ 113 | tx_hash \ 114 | FROM " + self.txTable + " WHERE \ 115 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false) \ 116 | SELECT \ 117 | sum(o.output_value_satoshi) \ 118 | FROM o JOIN t ON \ 119 | o.output_tx_hash = t.tx_hash \ 120 | LEFT JOIN (\ 121 | SELECT \ 122 | o.output_tx_hash as change_output_tx_hash, \ 123 | o.output_index as change_output_index \ 124 | FROM o JOIN i ON \ 125 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 126 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 127 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 128 | WHERE change.change_output_tx_hash is NULL", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 129 | return result[0] if result[0] is not None else 0 130 | 131 | def getHeuristicalOutputVolumeBetween(self, minDate, maxDate): 132 | result = self.dbAccess.queryReturnOne("WITH \ 133 | o AS (\ 134 | SELECT \ 135 | output_tx_hash, \ 136 | output_index, \ 137 | output_value_satoshi, \ 138 | output_addresses \ 139 | FROM " + self.outputsTable + " WHERE \ 140 | (output_time_created >= %s AND output_time_created < %s) \ 141 | AND output_type>1 \ 142 | AND ((output_time_spent is NULL) OR \ 143 | (EXTRACT(EPOCH FROM (output_time_spent - output_time_created)) > 2400))), \ 144 | i AS (\ 145 | SELECT \ 146 | output_spending_tx_hash, \ 147 | output_index, \ 148 | output_addresses \ 149 | FROM " + self.outputsTable + " WHERE \ 150 | (output_time_spent >= %s AND output_time_spent < %s)), \ 151 | t AS (\ 152 | SELECT \ 153 | tx_hash \ 154 | FROM " + self.txTable + " WHERE \ 155 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false) \ 156 | SELECT \ 157 | sum(o.output_value_satoshi) \ 158 | FROM o JOIN t ON \ 159 | o.output_tx_hash = t.tx_hash \ 160 | LEFT JOIN (\ 161 | SELECT \ 162 | o.output_tx_hash as change_output_tx_hash, \ 163 | o.output_index as change_output_index \ 164 | FROM o JOIN i ON \ 165 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 166 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 167 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 168 | WHERE change.change_output_tx_hash is NULL", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 169 | return result[0] if result[0] is not None else 0 170 | 171 | def getActiveAddressesCountBetween(self, minDate, maxDate): 172 | result = self.dbAccess.queryReturnOne("\ 173 | SELECT \ 174 | count(distinct address) \ 175 | FROM ( \ 176 | SELECT \ 177 | unnest(output_addresses) AS address \ 178 | FROM " + self.outputsTable + " WHERE \ 179 | (output_time_created >= %s AND output_time_created < %s) \ 180 | UNION ALL \ 181 | SELECT \ 182 | unnest(output_addresses) AS address \ 183 | FROM " + self.outputsTable + " WHERE \ 184 | (output_time_spent >= %s AND output_time_spent < %s)) active_addresses", (minDate, maxDate, minDate, maxDate)) 185 | return result[0] 186 | 187 | def getFeesVolumeBetween(self, minDate, maxDate): 188 | result = self.dbAccess.queryReturnOne("WITH \ 189 | o AS (\ 190 | SELECT \ 191 | output_tx_hash, \ 192 | output_value_satoshi \ 193 | FROM " + self.outputsTable + " WHERE \ 194 | (output_time_created >= %s AND output_time_created < %s)), \ 195 | i AS (\ 196 | SELECT \ 197 | output_spending_tx_hash, \ 198 | output_value_satoshi \ 199 | FROM " + self.outputsTable + " WHERE \ 200 | (output_time_spent >= %s AND output_time_spent < %s)), \ 201 | t AS (\ 202 | SELECT \ 203 | tx_hash \ 204 | FROM " + self.txTable + " WHERE \ 205 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 206 | volume_o AS (SELECT sum(o.output_value_satoshi) v FROM o JOIN t ON t.tx_hash=o.output_tx_hash), \ 207 | volume_i AS (SELECT sum(i.output_value_satoshi) v FROM i JOIN t ON t.tx_hash=i.output_spending_tx_hash) \ 208 | SELECT \ 209 | coalesce(volume_i.v, 0) - coalesce(volume_o.v, 0) \ 210 | FROM \ 211 | volume_o CROSS JOIN volume_i", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 212 | return result[0] 213 | 214 | def getMedianFeeBetween(self, minDate, maxDate): 215 | result = self.dbAccess.queryReturnOne("WITH \ 216 | o AS (\ 217 | SELECT \ 218 | output_tx_hash, \ 219 | output_value_satoshi \ 220 | FROM " + self.outputsTable + " WHERE \ 221 | (output_time_created >= %s AND output_time_created < %s)), \ 222 | i AS (\ 223 | SELECT \ 224 | output_spending_tx_hash, \ 225 | output_value_satoshi \ 226 | FROM " + self.outputsTable + " WHERE \ 227 | (output_time_spent >= %s AND output_time_spent < %s)), \ 228 | t AS (\ 229 | SELECT \ 230 | tx_hash \ 231 | FROM " + self.txTable + " WHERE \ 232 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 233 | so AS (\ 234 | SELECT \ 235 | coalesce(sum(o.output_value_satoshi), 0) as sum_outputs, \ 236 | t.tx_hash \ 237 | FROM t JOIN o ON \ 238 | t.tx_hash=o.output_tx_hash \ 239 | GROUP BY t.tx_hash), \ 240 | si AS (\ 241 | SELECT \ 242 | coalesce(sum(i.output_value_satoshi), 0) as sum_inputs, \ 243 | t.tx_hash \ 244 | FROM t JOIN i ON \ 245 | t.tx_hash=i.output_spending_tx_hash \ 246 | GROUP BY t.tx_hash), \ 247 | fees AS (\ 248 | SELECT \ 249 | coalesce(si.sum_inputs, 0) - coalesce(so.sum_outputs, 0) as fee, \ 250 | si.tx_hash as hash \ 251 | FROM si FULL OUTER JOIN so ON \ 252 | si.tx_hash=so.tx_hash) \ 253 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY fee) FROM fees", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 254 | return result[0] 255 | 256 | def getMedianTransactionValueBetween(self, minDate, maxDate): 257 | result = self.dbAccess.queryReturnOne("WITH \ 258 | o AS (\ 259 | SELECT \ 260 | output_tx_hash, \ 261 | output_index, \ 262 | output_value_satoshi, \ 263 | output_addresses \ 264 | FROM " + self.outputsTable + " WHERE \ 265 | (output_time_created >= %s AND output_time_created < %s)), \ 266 | i AS (\ 267 | SELECT \ 268 | output_spending_tx_hash, \ 269 | output_index, \ 270 | output_addresses \ 271 | FROM " + self.outputsTable + " WHERE \ 272 | (output_time_spent >= %s AND output_time_spent < %s)), \ 273 | t AS (\ 274 | SELECT \ 275 | tx_hash \ 276 | FROM " + self.txTable + " WHERE \ 277 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 278 | so AS (\ 279 | SELECT \ 280 | sum(o.output_value_satoshi) as sum_outputs \ 281 | FROM t JOIN o ON \ 282 | t.tx_hash=o.output_tx_hash \ 283 | LEFT JOIN (\ 284 | SELECT \ 285 | o.output_tx_hash as change_output_tx_hash, \ 286 | o.output_index as change_output_index \ 287 | FROM o JOIN i ON \ 288 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 289 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 290 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 291 | WHERE change.change_output_tx_hash is NULL \ 292 | GROUP BY t.tx_hash) \ 293 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY sum_outputs) FROM so", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 294 | return result[0] 295 | 296 | def getPaymentCountBetween(self, minDate, maxDate): 297 | result = self.dbAccess.queryReturnOne("WITH \ 298 | o AS (\ 299 | SELECT \ 300 | output_tx_hash, \ 301 | output_value_satoshi \ 302 | FROM " + self.outputsTable + " WHERE \ 303 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 304 | t AS (\ 305 | SELECT \ 306 | tx_hash \ 307 | FROM " + self.txTable + " WHERE \ 308 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 309 | so AS (\ 310 | SELECT \ 311 | GREATEST(count(*) - 1, 0) as payments, \ 312 | t.tx_hash \ 313 | FROM t JOIN o ON \ 314 | t.tx_hash=o.output_tx_hash \ 315 | GROUP BY t.tx_hash) \ 316 | SELECT sum(payments) FROM so", (minDate, maxDate, minDate, maxDate)) 317 | return result[0] if result[0] is not None else 0 318 | 319 | def getRewardBetween(self, minDate, maxDate): 320 | result = self.dbAccess.queryReturnOne("WITH \ 321 | t AS (\ 322 | SELECT \ 323 | tx_hash \ 324 | FROM " + self.txTable + " WHERE \ 325 | tx_coinbase=true AND (tx_time >= %s AND tx_time < %s)), \ 326 | o AS (\ 327 | SELECT \ 328 | output_value_satoshi, \ 329 | output_tx_hash \ 330 | FROM " + self.outputsTable + " WHERE \ 331 | (output_time_created >= %s AND output_time_created < %s)) \ 332 | SELECT sum(o.output_value_satoshi) FROM t JOIN o ON t.tx_hash=o.output_tx_hash", (minDate, maxDate, minDate, maxDate)) 333 | return result[0] if result[0] is not None else 0 334 | 335 | def getTotalSupplyBetween(self, minDate, maxDate): 336 | previousValue = self.dbAccess.queryReturnOne(""" 337 | SELECT 338 | value 339 | FROM 340 | statistic_total_supply_{asset} 341 | WHERE 342 | date = %s 343 | """.format( 344 | asset=self.asset 345 | ), (minDate - timedelta(days=1),)) 346 | previousValue = previousValue[0] if previousValue is not None else 0 347 | 348 | todayReward = self.getRewardBetween(minDate, maxDate) 349 | todayFees = self.getFeesVolumeBetween(minDate, maxDate) 350 | return previousValue + todayReward - todayFees 351 | 352 | def get30DNaiveCirculatingSupplyBetween(self, _, maxDate): 353 | delta = timedelta(days=30) 354 | 355 | result = self.dbAccess.queryReturnOne(""" 356 | SELECT 357 | SUM(output_value_satoshi) 358 | FROM 359 | {outputs} 360 | JOIN 361 | {transactions} 362 | ON 363 | output_tx_hash = tx_hash 364 | WHERE 365 | tx_coinbase IS FALSE 366 | AND 367 | ((output_time_spent is NULL) OR (output_time_spent >= %s)) 368 | AND 369 | ((output_time_created >= %s) AND (output_time_created < %s)) 370 | """.format( 371 | outputs=self.outputsTable, transactions=self.txTable, 372 | ), (maxDate, maxDate - delta, maxDate,)) 373 | return result[0] 374 | 375 | # We use an iterative approach to compute this stat. 376 | # Today's circulating supply is equal to: 377 | # yesterday's circulating supply 378 | # + non-reward unspent outputs created today 379 | # - minus spent outputs created that are no more than ${maxDate - delta} old 380 | # - minus outputs unspent to this day but created exacly minDate - delta days ago 381 | def _getCirculatingSupplyBetween(self, minDate, maxDate, delta, deltaString): 382 | previousValue = self.dbAccess.queryReturnOne(""" 383 | SELECT 384 | value 385 | FROM 386 | statistic_{deltaString}_circulating_supply_{asset} 387 | WHERE 388 | date = %s 389 | """.format( 390 | deltaString=deltaString, 391 | asset=self.asset 392 | ), (minDate - timedelta(days=1),)) 393 | previousValue = previousValue[0] if previousValue is not None else 0 394 | 395 | spentValue = self.dbAccess.queryReturnOne(""" 396 | SELECT 397 | COALESCE(SUM(output_value_satoshi), 0) 398 | FROM 399 | {outputs} 400 | JOIN 401 | {transactions} 402 | ON 403 | output_tx_hash = tx_hash 404 | WHERE 405 | tx_coinbase IS FALSE 406 | AND 407 | (output_time_created >= %s AND output_time_created < %s) 408 | AND 409 | (output_time_spent >= %s AND output_time_spent < %s) 410 | """.format( 411 | outputs=self.outputsTable, transactions=self.txTable, 412 | ), (minDate - delta, minDate, minDate, maxDate)) 413 | spentValue = spentValue[0] if spentValue is not None else 0 414 | 415 | maturedValue = self.dbAccess.queryReturnOne(""" 416 | SELECT 417 | COALESCE(SUM(output_value_satoshi), 0) 418 | FROM 419 | {outputs} 420 | JOIN 421 | {transactions} 422 | ON 423 | output_tx_hash = tx_hash 424 | WHERE 425 | tx_coinbase IS FALSE 426 | AND 427 | (output_time_created >= %s AND output_time_created < %s) 428 | AND 429 | (output_time_spent IS NULL OR output_time_spent >= %s) 430 | """.format( 431 | outputs=self.outputsTable, transactions=self.txTable, 432 | ), (minDate - delta, maxDate - delta, maxDate)) 433 | maturedValue = maturedValue[0] if maturedValue is not None else 0 434 | 435 | createdValue = self.dbAccess.queryReturnOne(""" 436 | SELECT 437 | COALESCE(SUM(output_value_satoshi), 0) 438 | FROM 439 | {outputs} 440 | JOIN 441 | {transactions} 442 | ON 443 | output_tx_hash = tx_hash 444 | WHERE 445 | tx_coinbase IS FALSE 446 | AND 447 | (output_time_created >= %s AND output_time_created < %s) 448 | AND 449 | ((output_time_spent is NULL) OR (output_time_spent >= %s)) 450 | """.format( 451 | outputs=self.outputsTable, transactions=self.txTable, 452 | ), (minDate, maxDate, maxDate)) 453 | createdValue = createdValue[0] if createdValue is not None else 0 454 | 455 | return previousValue - spentValue - maturedValue + createdValue 456 | 457 | def get1YCirculatingSupplyBetween(self, minDate, maxDate): 458 | return self._getCirculatingSupplyBetween(minDate, maxDate, timedelta(days=365), "1y") 459 | 460 | def get180DCirculatingSupplyBetween(self, minDate, maxDate): 461 | return self._getCirculatingSupplyBetween(minDate, maxDate, timedelta(days=180), "180d") 462 | 463 | def get30DCirculatingSupplyBetween(self, minDate, maxDate): 464 | return self._getCirculatingSupplyBetween(minDate, maxDate, timedelta(days=30), "30d") 465 | 466 | def run(self, text): 467 | return self.dbAccess.queryReturnAll(text) 468 | 469 | 470 | class ZcashQuery(BitcoinQuery): 471 | 472 | def __init__(self, dbAccess, schema): 473 | super(ZcashQuery, self).__init__(dbAccess, schema) 474 | self.joinSplitsTable = self.schema.getJoinSplitsTableName() 475 | self.saplingPaymentTable = self.schema.getSaplingPaymentTableName() 476 | 477 | def getFeesVolumeBetween(self, minDate, maxDate): 478 | baseFees = super(ZcashQuery, self).getFeesVolumeBetween(minDate, maxDate) 479 | joinSplitDiffs = self.getJoinSplitsDiffValueBetween(minDate, maxDate) 480 | saplingValueBalance = self.getSaplingValueBalanceBetween(minDate, maxDate) 481 | return baseFees + joinSplitDiffs + saplingValueBalance 482 | 483 | def getOutputVolumeBetween(self, minDate, maxDate): 484 | baseResult = super(ZcashQuery, self).getOutputVolumeBetween(minDate, maxDate) 485 | joinSplitResult = self.getJoinSplitsNegativeDiffValueBetween(minDate, maxDate) 486 | saplingValueBalance = self.getSaplingNegativeValueBalanceBetween(minDate, maxDate) 487 | return baseResult + joinSplitResult + saplingValueBalance 488 | 489 | def getHeuristicalOutputVolumeBetween(self, minDate, maxDate): 490 | baseResult = super(ZcashQuery, self).getHeuristicalOutputVolumeBetween(minDate, maxDate) 491 | joinSplitResult = self.getJoinSplitsNegativeDiffValueBetween(minDate, maxDate) 492 | saplingValueBalance = self.getSaplingNegativeValueBalanceBetween(minDate, maxDate) 493 | return baseResult + joinSplitResult + saplingValueBalance 494 | 495 | def getJoinSplitsDiffValueBetween(self, minDate, maxDate): 496 | result = self.dbAccess.queryReturnOne("\ 497 | SELECT \ 498 | sum(joinsplit_value_new) - sum(joinsplit_value_old) AS value \ 499 | FROM " + self.joinSplitsTable + " WHERE \ 500 | (joinsplit_time >= %s AND joinsplit_time < %s)", (minDate, maxDate)) 501 | return result[0] if result[0] is not None else 0 502 | 503 | def getJoinSplitsNegativeDiffValueBetween(self, minDate, maxDate): 504 | result = self.dbAccess.queryReturnOne("\ 505 | SELECT \ 506 | sum(-least( joinsplit_value_new - joinsplit_value_old, 0)) AS value \ 507 | FROM " + self.joinSplitsTable + " WHERE \ 508 | (joinsplit_time >= %s AND joinsplit_time < %s)", (minDate, maxDate)) 509 | return result[0] if result[0] is not None else 0 510 | 511 | def getSaplingValueBalanceBetween(self, minDate, maxDate): 512 | result = self.dbAccess.queryReturnOne("\ 513 | SELECT \ 514 | sum(sapling_payment_value_balance) AS value \ 515 | FROM " + self.saplingPaymentTable + " WHERE \ 516 | (sapling_payment_time >= %s AND sapling_payment_time < %s)", (minDate, maxDate)) 517 | return result[0] if result[0] is not None else 0 518 | 519 | def getSaplingNegativeValueBalanceBetween(self, minDate, maxDate): 520 | result = self.dbAccess.queryReturnOne("\ 521 | SELECT \ 522 | sum(-least(sapling_payment_value_balance, 0)) AS value \ 523 | FROM " + self.saplingPaymentTable + " WHERE \ 524 | (sapling_payment_time >= %s AND sapling_payment_time < %s)", (minDate, maxDate)) 525 | return result[0] if result[0] is not None else 0 526 | 527 | def getMedianFeeBetween(self, minDate, maxDate): 528 | result = self.dbAccess.queryReturnOne("WITH \ 529 | joinsplit AS (\ 530 | SELECT \ 531 | joinsplit_value_new - joinsplit_value_old as value, \ 532 | joinsplit_tx_hash \ 533 | FROM " + self.joinSplitsTable + " WHERE \ 534 | (joinsplit_time >= %s AND joinsplit_time < %s)), \ 535 | sapling_payment AS (\ 536 | SELECT \ 537 | sapling_payment_value_balance as value, \ 538 | sapling_payment_tx_hash \ 539 | FROM " + self.saplingPaymentTable + " WHERE \ 540 | (sapling_payment_time >= %s AND sapling_payment_time < %s)), \ 541 | o AS (\ 542 | SELECT \ 543 | output_tx_hash, \ 544 | output_value_satoshi \ 545 | FROM " + self.outputsTable + " WHERE \ 546 | (output_time_created >= %s AND output_time_created < %s)), \ 547 | i AS (\ 548 | SELECT \ 549 | output_spending_tx_hash, \ 550 | output_value_satoshi \ 551 | FROM " + self.outputsTable + " WHERE \ 552 | (output_time_spent >= %s AND output_time_spent < %s)), \ 553 | t AS (\ 554 | SELECT \ 555 | tx_hash \ 556 | FROM " + self.txTable + " WHERE \ 557 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 558 | so AS (\ 559 | SELECT \ 560 | -coalesce(sum(o.output_value_satoshi), 0) as sum, \ 561 | t.tx_hash \ 562 | FROM t JOIN o ON \ 563 | t.tx_hash=o.output_tx_hash \ 564 | GROUP BY t.tx_hash \ 565 | UNION ALL \ 566 | SELECT \ 567 | coalesce(sum(i.output_value_satoshi), 0) as sum, \ 568 | t.tx_hash \ 569 | FROM t JOIN i ON \ 570 | t.tx_hash=i.output_spending_tx_hash \ 571 | GROUP BY t.tx_hash \ 572 | UNION ALL \ 573 | SELECT \ 574 | coalesce(sum(joinsplit.value), 0) AS sum, \ 575 | t.tx_hash \ 576 | FROM t JOIN joinsplit ON \ 577 | t.tx_hash=joinsplit.joinsplit_tx_hash \ 578 | GROUP BY t.tx_hash \ 579 | UNION ALL \ 580 | SELECT \ 581 | coalesce(sum(sapling_payment.value), 0) AS sum, \ 582 | t.tx_hash \ 583 | FROM t JOIN sapling_payment ON \ 584 | t.tx_hash=sapling_payment.sapling_payment_tx_hash \ 585 | GROUP BY t.tx_hash), \ 586 | fees AS (\ 587 | SELECT \ 588 | coalesce(sum(so.sum), 0) as fee \ 589 | FROM so \ 590 | GROUP BY so.tx_hash) \ 591 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY fee) FROM fees", (minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate)) 592 | return result[0] 593 | 594 | def getMedianTransactionValueBetween(self, minDate, maxDate): 595 | result = self.dbAccess.queryReturnOne("WITH \ 596 | joinsplit AS (\ 597 | SELECT \ 598 | -least(joinsplit_value_new - joinsplit_value_old, 0) as value, \ 599 | joinsplit_tx_hash \ 600 | FROM " + self.joinSplitsTable + " WHERE \ 601 | (joinsplit_time >= %s AND joinsplit_time < %s)), \ 602 | sapling_payment AS (\ 603 | SELECT \ 604 | -least(sapling_payment_value_balance, 0) as value, \ 605 | sapling_payment_tx_hash \ 606 | FROM " + self.saplingPaymentTable + " WHERE \ 607 | (sapling_payment_time >= %s AND sapling_payment_time < %s)), \ 608 | o AS (\ 609 | SELECT \ 610 | output_tx_hash, \ 611 | output_index, \ 612 | output_value_satoshi, \ 613 | output_addresses \ 614 | FROM " + self.outputsTable + " WHERE \ 615 | (output_time_created >= %s AND output_time_created < %s)), \ 616 | i AS (\ 617 | SELECT \ 618 | output_spending_tx_hash, \ 619 | output_index, \ 620 | output_addresses \ 621 | FROM " + self.outputsTable + " WHERE \ 622 | (output_time_spent >= %s AND output_time_spent < %s)), \ 623 | t AS (\ 624 | SELECT \ 625 | tx_hash \ 626 | FROM " + self.txTable + " WHERE \ 627 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 628 | so AS (\ 629 | SELECT \ 630 | coalesce(sum(o.output_value_satoshi), 0) as partial_sum, \ 631 | t.tx_hash as tx_hash \ 632 | FROM t JOIN o ON \ 633 | t.tx_hash=o.output_tx_hash \ 634 | WHERE (o.output_tx_hash, o.output_index) NOT IN (\ 635 | SELECT \ 636 | o.output_tx_hash as output_tx_hash, \ 637 | o.output_index as output_index \ 638 | FROM o JOIN i ON \ 639 | (o.output_tx_hash = i.output_spending_tx_hash) \ 640 | AND \ 641 | ((o.output_addresses && i.output_addresses) = true)) \ 642 | GROUP BY t.tx_hash), \ 643 | sj AS (\ 644 | SELECT \ 645 | coalesce(sum(joinsplit.value), 0) as partial_sum, \ 646 | t.tx_hash as tx_hash \ 647 | FROM t JOIN joinsplit ON \ 648 | t.tx_hash=joinsplit.joinsplit_tx_hash \ 649 | GROUP BY t.tx_hash), \ 650 | ssp AS (\ 651 | SELECT \ 652 | coalesce(sum(sapling_payment.value), 0) as partial_sum, \ 653 | t.tx_hash as tx_hash \ 654 | FROM t JOIN sapling_payment ON \ 655 | t.tx_hash=sapling_payment.sapling_payment_tx_hash \ 656 | GROUP BY t.tx_hash), \ 657 | sall AS (\ 658 | SELECT * FROM sj UNION ALL \ 659 | SELECT * FROM so UNION ALL \ 660 | SELECT * FROM ssp \ 661 | ), \ 662 | total AS (SELECT sum(partial_sum) AS sum_total, tx_hash FROM sall GROUP BY tx_hash) \ 663 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY sum_total) FROM total", (minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate)) 664 | return result[0] 665 | 666 | def getPaymentCountBetween(self, minDate, maxDate): 667 | result = self.dbAccess.queryReturnOne("WITH \ 668 | o AS (\ 669 | SELECT \ 670 | output_tx_hash, \ 671 | output_value_satoshi \ 672 | FROM " + self.outputsTable + " WHERE \ 673 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 674 | sapling_payment AS (\ 675 | SELECT \ 676 | sapling_payment_output_count as output_count, \ 677 | sapling_payment_tx_hash \ 678 | FROM " + self.saplingPaymentTable + " WHERE \ 679 | (sapling_payment_time >= %s AND sapling_payment_time < %s)), \ 680 | t AS (\ 681 | SELECT \ 682 | tx_hash \ 683 | FROM " + self.txTable + " WHERE \ 684 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 685 | all_outputs AS (\ 686 | SELECT \ 687 | count(*) as output_count, \ 688 | t.tx_hash \ 689 | FROM t JOIN o ON \ 690 | t.tx_hash=o.output_tx_hash \ 691 | GROUP BY t.tx_hash \ 692 | UNION ALL \ 693 | SELECT \ 694 | sum(sapling_payment.output_count) as output_count, \ 695 | t.tx_hash \ 696 | FROM t JOIN sapling_payment ON \ 697 | t.tx_hash=sapling_payment.sapling_payment_tx_hash \ 698 | GROUP BY t.tx_hash), \ 699 | total AS (SELECT greatest(sum(output_count) - 1, 0) as payments FROM all_outputs GROUP BY tx_hash) \ 700 | SELECT sum(payments) FROM total", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 701 | return result[0] if result[0] is not None else 0 702 | 703 | 704 | class PivxQuery(BitcoinQuery): 705 | 706 | def __init__(self, dbAccess, schema): 707 | super(PivxQuery, self).__init__(dbAccess, schema) 708 | self.zerocoinMintsTableName = self.schema.getZerocoinMintsTableName() 709 | self.zerocoinSpendsTableName = self.schema.getZerocoinSpendsTableName() 710 | 711 | def getRewardBetween(self, minDate, maxDate): 712 | baseResult = super(PivxQuery, self).getRewardBetween(minDate, maxDate) 713 | coinbaseInputs = self.getCoinbaseInputVolumeBetween(minDate, maxDate) 714 | coinbaseZerocoinMints = self.getZerocoinMintsVolumeBetween(minDate, maxDate, True) 715 | coinbaseZerocoinSpends = self.getZerocoinSpendsVolumeBetween(minDate, maxDate, True) 716 | return baseResult - coinbaseInputs + coinbaseZerocoinMints - coinbaseZerocoinSpends 717 | 718 | def getOutputVolumeBetween(self, minDate, maxDate): 719 | baseResult = super(PivxQuery, self).getOutputVolumeBetween(minDate, maxDate) 720 | zerocoinMints = self.getZerocoinMintsVolumeBetween(minDate, maxDate, False) 721 | return baseResult + zerocoinMints 722 | 723 | def getHeuristicalOutputVolumeBetween(self, minDate, maxDate): 724 | baseResult = super(PivxQuery, self).getHeuristicalOutputVolumeBetween(minDate, maxDate) 725 | zerocoinMints = self.getZerocoinMintsVolumeBetween(minDate, maxDate, False) 726 | return baseResult + zerocoinMints 727 | 728 | def getFeesVolumeBetween(self, minDate, maxDate): 729 | baseFees = super(PivxQuery, self).getFeesVolumeBetween(minDate, maxDate) 730 | zerocoinMints = self.getZerocoinMintsVolumeBetween(minDate, maxDate, False) 731 | zerocoinSpends = self.getZerocoinSpendsVolumeBetween(minDate, maxDate, False) 732 | return baseFees - zerocoinMints + zerocoinSpends 733 | 734 | def getZerocoinMintsVolumeBetween(self, minDate, maxDate, coinbase): 735 | result = self.dbAccess.queryReturnOne("WITH \ 736 | t AS (\ 737 | SELECT \ 738 | tx_hash \ 739 | FROM " + self.txTable + " WHERE \ 740 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=%s), \ 741 | z AS (\ 742 | SELECT \ 743 | zerocoin_mint_tx_hash, \ 744 | zerocoin_mint_value \ 745 | FROM " + self.zerocoinMintsTableName + " WHERE \ 746 | (zerocoin_mint_time >= %s AND zerocoin_mint_time < %s)) \ 747 | SELECT \ 748 | sum(zerocoin_mint_value) \ 749 | FROM z JOIN t ON \ 750 | z.zerocoin_mint_tx_hash=t.tx_hash", (minDate, maxDate, coinbase, minDate, maxDate)) 751 | return result[0] if result[0] is not None else 0 752 | 753 | def getZerocoinSpendsVolumeBetween(self, minDate, maxDate, coinbase): 754 | result = self.dbAccess.queryReturnOne("WITH \ 755 | t AS (\ 756 | SELECT \ 757 | tx_hash \ 758 | FROM " + self.txTable + " WHERE \ 759 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=%s), \ 760 | z AS (\ 761 | SELECT \ 762 | zerocoin_spend_tx_hash, \ 763 | zerocoin_spend_value \ 764 | FROM " + self.zerocoinSpendsTableName + " WHERE \ 765 | (zerocoin_spend_time >= %s AND zerocoin_spend_time < %s)) \ 766 | SELECT \ 767 | sum(zerocoin_spend_value) \ 768 | FROM z JOIN t ON \ 769 | z.zerocoin_spend_tx_hash=t.tx_hash", (minDate, maxDate, coinbase, minDate, maxDate)) 770 | return result[0] if result[0] is not None else 0 771 | 772 | def getMedianFeeBetween(self, minDate, maxDate): 773 | result = self.dbAccess.queryReturnOne("WITH \ 774 | zspend AS (\ 775 | SELECT \ 776 | zerocoin_spend_tx_hash, \ 777 | zerocoin_spend_value \ 778 | FROM " + self.zerocoinSpendsTableName + " WHERE \ 779 | (zerocoin_spend_time >= %s AND zerocoin_spend_time < %s)), \ 780 | zmint AS (\ 781 | SELECT \ 782 | zerocoin_mint_tx_hash, \ 783 | zerocoin_mint_value \ 784 | FROM " + self.zerocoinMintsTableName + " WHERE \ 785 | (zerocoin_mint_time >= %s AND zerocoin_mint_time < %s)), \ 786 | o AS (\ 787 | SELECT \ 788 | output_tx_hash, \ 789 | output_value_satoshi \ 790 | FROM " + self.outputsTable + " WHERE \ 791 | (output_time_created >= %s AND output_time_created < %s)), \ 792 | i AS (\ 793 | SELECT \ 794 | output_spending_tx_hash, \ 795 | output_value_satoshi \ 796 | FROM " + self.outputsTable + " WHERE \ 797 | (output_time_spent >= %s AND output_time_spent < %s)), \ 798 | t AS (\ 799 | SELECT \ 800 | tx_hash \ 801 | FROM " + self.txTable + " WHERE \ 802 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 803 | so AS (\ 804 | SELECT \ 805 | -coalesce(sum(o.output_value_satoshi), 0) as sum, \ 806 | t.tx_hash \ 807 | FROM t JOIN o ON t.tx_hash=o.output_tx_hash GROUP BY t.tx_hash \ 808 | UNION ALL \ 809 | SELECT \ 810 | coalesce(sum(i.output_value_satoshi), 0) as sum, \ 811 | t.tx_hash \ 812 | FROM t JOIN i ON t.tx_hash=i.output_spending_tx_hash GROUP BY t.tx_hash \ 813 | UNION ALL \ 814 | SELECT \ 815 | coalesce(sum(zspend.zerocoin_spend_value), 0) as sum, \ 816 | t.tx_hash \ 817 | FROM t join zspend ON t.tx_hash=zspend.zerocoin_spend_tx_hash GROUP BY t.tx_hash \ 818 | UNION ALL \ 819 | SELECT \ 820 | -coalesce(sum(zmint.zerocoin_mint_value), 0) as sum, \ 821 | t.tx_hash \ 822 | FROM t join zmint ON t.tx_hash=zmint.zerocoin_mint_tx_hash GROUP BY t.tx_hash), \ 823 | fees AS (\ 824 | SELECT \ 825 | coalesce(sum(so.sum), 0) as fee \ 826 | FROM so \ 827 | GROUP BY so.tx_hash) \ 828 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY fee) FROM fees", (minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate)) 829 | return result[0] 830 | 831 | def getMedianTransactionValueBetween(self, minDate, maxDate): 832 | result = self.dbAccess.queryReturnOne("WITH \ 833 | z AS (\ 834 | SELECT \ 835 | zerocoin_mint_tx_hash, \ 836 | zerocoin_mint_value \ 837 | FROM " + self.zerocoinMintsTableName + " WHERE \ 838 | (zerocoin_mint_time >= %s AND zerocoin_mint_time < %s)), \ 839 | o AS (\ 840 | SELECT \ 841 | output_tx_hash, \ 842 | output_index, \ 843 | output_value_satoshi, \ 844 | output_addresses \ 845 | FROM " + self.outputsTable + " WHERE \ 846 | (output_time_created >= %s AND output_time_created < %s)), \ 847 | i AS (\ 848 | SELECT \ 849 | output_spending_tx_hash, \ 850 | output_index, \ 851 | output_value_satoshi, \ 852 | output_addresses \ 853 | FROM " + self.outputsTable + " WHERE \ 854 | (output_time_spent >= %s AND output_time_spent < %s)), \ 855 | t AS (\ 856 | SELECT \ 857 | tx_hash \ 858 | FROM " + self.txTable + " WHERE \ 859 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 860 | so AS (\ 861 | SELECT \ 862 | coalesce(sum(o.output_value_satoshi), 0) as sum_outputs, \ 863 | t.tx_hash \ 864 | FROM t JOIN o ON \ 865 | t.tx_hash=o.output_tx_hash \ 866 | WHERE \ 867 | (o.output_tx_hash, o.output_index) NOT IN \ 868 | (SELECT \ 869 | o.output_tx_hash as output_tx_hash, \ 870 | o.output_index as output_index \ 871 | FROM o JOIN i ON \ 872 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 873 | ((o.output_addresses && i.output_addresses) = true)) \ 874 | GROUP BY t.tx_hash), \ 875 | sz AS (\ 876 | SELECT \ 877 | sum(z.zerocoin_mint_value) as sum_zerocoin, \ 878 | t.tx_hash \ 879 | FROM t JOIN z ON \ 880 | t.tx_hash=z.zerocoin_mint_tx_hash \ 881 | GROUP BY t.tx_hash), \ 882 | total AS (\ 883 | SELECT \ 884 | (coalesce(so.sum_outputs, 0) + coalesce(sz.sum_zerocoin, 0)) as sum_total \ 885 | FROM so FULL OUTER JOIN sz ON \ 886 | so.tx_hash=sz.tx_hash) \ 887 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY sum_total) FROM total", (minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate)) 888 | return result[0] 889 | 890 | def getPaymentCountBetween(self, minDate, maxDate): 891 | result = self.dbAccess.queryReturnOne("WITH \ 892 | z AS (\ 893 | SELECT \ 894 | zerocoin_mint_tx_hash \ 895 | FROM " + self.zerocoinMintsTableName + " WHERE \ 896 | (zerocoin_mint_time >= %s AND zerocoin_mint_time < %s)), \ 897 | o AS (\ 898 | SELECT \ 899 | output_tx_hash \ 900 | FROM " + self.outputsTable + " WHERE \ 901 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 902 | t AS (\ 903 | SELECT \ 904 | tx_hash \ 905 | FROM " + self.txTable + " WHERE \ 906 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false), \ 907 | so AS (\ 908 | SELECT \ 909 | count(*) as payments, \ 910 | t.tx_hash \ 911 | FROM t JOIN o ON \ 912 | t.tx_hash=o.output_tx_hash \ 913 | GROUP BY t.tx_hash), \ 914 | sz AS (\ 915 | SELECT \ 916 | count(*) as payments, \ 917 | t.tx_hash \ 918 | FROM t JOIN z ON \ 919 | t.tx_hash=z.zerocoin_mint_tx_hash \ 920 | GROUP BY t.tx_hash), \ 921 | total AS (\ 922 | SELECT \ 923 | greatest(coalesce(so.payments, 0) + coalesce(sz.payments, 0) - 1, 0) AS payments \ 924 | FROM so FULL OUTER JOIN sz ON \ 925 | so.tx_hash=sz.tx_hash) \ 926 | SELECT sum(payments) FROM total", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 927 | return result[0] 928 | 929 | def getCoinbaseInputVolumeBetween(self, minDate, maxDate): 930 | result = self.dbAccess.queryReturnOne("WITH \ 931 | t AS (\ 932 | SELECT \ 933 | tx_hash \ 934 | FROM " + self.txTable + " WHERE \ 935 | tx_coinbase=true AND (tx_time >= %s AND tx_time < %s)), \ 936 | i AS (\ 937 | SELECT \ 938 | output_spending_tx_hash, \ 939 | output_value_satoshi \ 940 | FROM " + self.outputsTable + " WHERE \ 941 | (output_time_spent >= %s AND output_time_spent < %s)) \ 942 | SELECT \ 943 | sum(output_value_satoshi) \ 944 | FROM i JOIN t ON \ 945 | t.tx_hash=i.output_spending_tx_hash", (minDate, maxDate, minDate, maxDate)) 946 | return result[0] if result[0] is not None else 0 947 | 948 | 949 | class DecredQuery(BitcoinQuery): 950 | 951 | def getTxCountBetween(self, minDate, maxDate): 952 | result = self.dbAccess.queryReturnOne("\ 953 | SELECT \ 954 | COUNT(*) \ 955 | FROM " + self.txTable + " WHERE \ 956 | tx_coinbase=false AND tx_vote=false AND (tx_time >= %s AND tx_time < %s)", (minDate, maxDate)) 957 | return result[0] 958 | 959 | def getFeesVolumeBetween(self, minDate, maxDate): 960 | result = self.dbAccess.queryReturnOne("WITH \ 961 | o AS (\ 962 | SELECT \ 963 | output_tx_hash, \ 964 | output_value_satoshi \ 965 | FROM " + self.outputsTable + " WHERE \ 966 | (output_time_created >= %s AND output_time_created < %s)), \ 967 | i AS (\ 968 | SELECT \ 969 | output_spending_tx_hash, \ 970 | output_value_satoshi \ 971 | FROM " + self.outputsTable + " WHERE \ 972 | (output_time_spent >= %s AND output_time_spent < %s)), \ 973 | t AS (\ 974 | SELECT \ 975 | tx_hash \ 976 | FROM " + self.txTable + " WHERE \ 977 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_vote=false), \ 978 | volume_o AS (SELECT sum(o.output_value_satoshi) v FROM o JOIN t ON t.tx_hash=o.output_tx_hash), \ 979 | volume_i AS (SELECT sum(i.output_value_satoshi) v FROM i JOIN t ON t.tx_hash=i.output_spending_tx_hash) \ 980 | SELECT \ 981 | coalesce(volume_i.v, 0) - coalesce(volume_o.v, 0) \ 982 | FROM \ 983 | volume_o CROSS JOIN volume_i", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 984 | return result[0] 985 | 986 | def getOutputVolumeBetween(self, minDate, maxDate): 987 | result = self.dbAccess.queryReturnOne("WITH \ 988 | o AS (\ 989 | SELECT \ 990 | output_tx_hash, \ 991 | output_index, \ 992 | output_value_satoshi, \ 993 | output_addresses \ 994 | FROM " + self.outputsTable + " WHERE \ 995 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 996 | i AS (\ 997 | SELECT \ 998 | output_spending_tx_hash, \ 999 | output_index, \ 1000 | output_addresses \ 1001 | FROM " + self.outputsTable + " WHERE \ 1002 | (output_time_spent >= %s AND output_time_spent < %s)), \ 1003 | t AS (\ 1004 | SELECT \ 1005 | tx_hash \ 1006 | FROM " + self.txTable + " WHERE \ 1007 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_vote=false AND tx_ticket=false) \ 1008 | SELECT \ 1009 | sum(o.output_value_satoshi) \ 1010 | FROM o JOIN t ON \ 1011 | o.output_tx_hash = t.tx_hash \ 1012 | LEFT JOIN (\ 1013 | SELECT \ 1014 | o.output_tx_hash as change_output_tx_hash, \ 1015 | o.output_index as change_output_index \ 1016 | FROM o JOIN i ON \ 1017 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 1018 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 1019 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 1020 | WHERE change.change_output_tx_hash is NULL", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 1021 | return result[0] if result[0] is not None else 0 1022 | 1023 | def getHeuristicalOutputVolumeBetween(self, minDate, maxDate): 1024 | result = self.dbAccess.queryReturnOne("WITH \ 1025 | o AS (\ 1026 | SELECT \ 1027 | output_tx_hash, \ 1028 | output_index, \ 1029 | output_value_satoshi, \ 1030 | output_addresses \ 1031 | FROM " + self.outputsTable + " WHERE \ 1032 | (output_time_created >= %s AND output_time_created < %s) \ 1033 | AND output_type>1 \ 1034 | AND ((output_time_spent is NULL) OR \ 1035 | (EXTRACT(EPOCH FROM (output_time_spent - output_time_created)) > 2400))), \ 1036 | i AS (\ 1037 | SELECT \ 1038 | output_spending_tx_hash, \ 1039 | output_index, \ 1040 | output_addresses \ 1041 | FROM " + self.outputsTable + " WHERE \ 1042 | (output_time_spent >= %s AND output_time_spent < %s)), \ 1043 | t AS (\ 1044 | SELECT \ 1045 | tx_hash \ 1046 | FROM " + self.txTable + " WHERE \ 1047 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_vote=false AND tx_ticket=false) \ 1048 | SELECT \ 1049 | sum(o.output_value_satoshi) \ 1050 | FROM o JOIN t ON \ 1051 | o.output_tx_hash = t.tx_hash \ 1052 | LEFT JOIN (\ 1053 | SELECT \ 1054 | o.output_tx_hash as change_output_tx_hash, \ 1055 | o.output_index as change_output_index \ 1056 | FROM o JOIN i ON \ 1057 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 1058 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 1059 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 1060 | WHERE change.change_output_tx_hash is NULL", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 1061 | return result[0] if result[0] is not None else 0 1062 | 1063 | def getMedianFeeBetween(self, minDate, maxDate): 1064 | result = self.dbAccess.queryReturnOne("WITH \ 1065 | o AS (\ 1066 | SELECT \ 1067 | output_tx_hash, \ 1068 | output_value_satoshi \ 1069 | FROM " + self.outputsTable + " WHERE \ 1070 | (output_time_created >= %s AND output_time_created < %s)), \ 1071 | i AS (\ 1072 | SELECT \ 1073 | output_spending_tx_hash, \ 1074 | output_value_satoshi \ 1075 | FROM " + self.outputsTable + " WHERE \ 1076 | (output_time_spent >= %s AND output_time_spent < %s)), \ 1077 | t AS (\ 1078 | SELECT \ 1079 | tx_hash \ 1080 | FROM " + self.txTable + " WHERE \ 1081 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_vote=false), \ 1082 | so AS (\ 1083 | SELECT \ 1084 | coalesce(sum(o.output_value_satoshi), 0) as sum_outputs, \ 1085 | t.tx_hash \ 1086 | FROM t JOIN o ON \ 1087 | t.tx_hash=o.output_tx_hash \ 1088 | GROUP BY t.tx_hash), \ 1089 | si AS (\ 1090 | SELECT \ 1091 | coalesce(sum(i.output_value_satoshi), 0) as sum_inputs, \ 1092 | t.tx_hash \ 1093 | FROM t JOIN i ON \ 1094 | t.tx_hash=i.output_spending_tx_hash \ 1095 | GROUP BY t.tx_hash), \ 1096 | fees AS (\ 1097 | SELECT \ 1098 | coalesce(si.sum_inputs, 0) - coalesce(so.sum_outputs, 0) as fee, \ 1099 | si.tx_hash as hash \ 1100 | FROM si FULL OUTER JOIN so ON \ 1101 | si.tx_hash=so.tx_hash) \ 1102 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY fee) FROM fees", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 1103 | return result[0] 1104 | 1105 | def getMedianTransactionValueBetween(self, minDate, maxDate): 1106 | result = self.dbAccess.queryReturnOne("WITH \ 1107 | o AS (\ 1108 | SELECT \ 1109 | output_tx_hash, \ 1110 | output_index, \ 1111 | output_value_satoshi, \ 1112 | output_addresses \ 1113 | FROM " + self.outputsTable + " WHERE \ 1114 | (output_time_created >= %s AND output_time_created < %s)), \ 1115 | i AS (\ 1116 | SELECT \ 1117 | output_spending_tx_hash, \ 1118 | output_index, \ 1119 | output_addresses \ 1120 | FROM " + self.outputsTable + " WHERE \ 1121 | (output_time_spent >= %s AND output_time_spent < %s)), \ 1122 | t AS (\ 1123 | SELECT \ 1124 | tx_hash \ 1125 | FROM " + self.txTable + " WHERE \ 1126 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_vote=false AND tx_ticket=false), \ 1127 | so AS (\ 1128 | SELECT \ 1129 | sum(o.output_value_satoshi) as sum_outputs \ 1130 | FROM t JOIN o ON \ 1131 | t.tx_hash=o.output_tx_hash \ 1132 | LEFT JOIN (\ 1133 | SELECT \ 1134 | o.output_tx_hash as change_output_tx_hash, \ 1135 | o.output_index as change_output_index \ 1136 | FROM o JOIN i ON \ 1137 | (o.output_tx_hash = i.output_spending_tx_hash) AND \ 1138 | ((o.output_addresses && i.output_addresses) = true)) change ON \ 1139 | o.output_tx_hash=change.change_output_tx_hash AND o.output_index=change.change_output_index \ 1140 | WHERE change.change_output_tx_hash is NULL \ 1141 | GROUP BY t.tx_hash) \ 1142 | SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY sum_outputs) FROM so", (minDate, maxDate, minDate, maxDate, minDate, maxDate)) 1143 | return result[0] 1144 | 1145 | def getPaymentCountBetween(self, minDate, maxDate): 1146 | result = self.dbAccess.queryReturnOne("WITH \ 1147 | o AS (\ 1148 | SELECT \ 1149 | output_tx_hash, \ 1150 | output_value_satoshi \ 1151 | FROM " + self.outputsTable + " WHERE \ 1152 | (output_time_created >= %s AND output_time_created < %s) AND output_type>1), \ 1153 | t AS (\ 1154 | SELECT \ 1155 | tx_hash \ 1156 | FROM " + self.txTable + " WHERE \ 1157 | (tx_time >= %s AND tx_time < %s) AND tx_coinbase=false AND tx_ticket=false AND tx_vote=false), \ 1158 | so AS (\ 1159 | SELECT \ 1160 | GREATEST(count(*) - 1, 0) as payments, \ 1161 | t.tx_hash \ 1162 | FROM t JOIN o ON \ 1163 | t.tx_hash=o.output_tx_hash \ 1164 | GROUP BY t.tx_hash) \ 1165 | SELECT sum(payments) FROM so", (minDate, maxDate, minDate, maxDate)) 1166 | return result[0] if result[0] is not None else 0 1167 | 1168 | def getRewardBetween(self, minDate, maxDate): 1169 | result = self.dbAccess.queryReturnOne("WITH \ 1170 | t AS (\ 1171 | SELECT \ 1172 | tx_hash \ 1173 | FROM " + self.txTable + " WHERE \ 1174 | tx_coinbase=true AND (tx_time >= %s AND tx_time < %s)), \ 1175 | o AS (\ 1176 | SELECT \ 1177 | output_value_satoshi, \ 1178 | output_tx_hash \ 1179 | FROM " + self.outputsTable + " WHERE \ 1180 | (output_time_created >= %s AND output_time_created < %s)), \ 1181 | pow AS (\ 1182 | SELECT \ 1183 | sum(o.output_value_satoshi) AS value \ 1184 | FROM t JOIN o ON \ 1185 | t.tx_hash=o.output_tx_hash), \ 1186 | st AS (\ 1187 | SELECT \ 1188 | tx_hash \ 1189 | FROM " + self.txTable + " WHERE \ 1190 | tx_vote=true AND (tx_time >= %s AND tx_time < %s)), \ 1191 | so AS (\ 1192 | SELECT \ 1193 | output_value_satoshi, \ 1194 | output_tx_hash \ 1195 | FROM " + self.outputsTable + " WHERE \ 1196 | (output_time_created >= %s AND output_time_created < %s)), \ 1197 | si AS (\ 1198 | SELECT \ 1199 | output_value_satoshi, \ 1200 | output_spending_tx_hash \ 1201 | FROM " + self.outputsTable + " WHERE \ 1202 | (output_time_spent >= %s AND output_time_spent < %s)), \ 1203 | sso AS (\ 1204 | SELECT \ 1205 | sum(so.output_value_satoshi) AS value \ 1206 | FROM st JOIN so ON \ 1207 | st.tx_hash=so.output_tx_hash), \ 1208 | ssi AS (\ 1209 | SELECT \ 1210 | sum(si.output_value_satoshi) AS value \ 1211 | FROM st JOIN si ON \ 1212 | st.tx_hash=si.output_spending_tx_hash) \ 1213 | SELECT \ 1214 | (coalesce(pow.value, 0) + coalesce(sso.value, 0) - coalesce(ssi.value, 0)) \ 1215 | FROM pow CROSS JOIN sso CROSS JOIN ssi", (minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate, minDate, maxDate)) 1216 | return result[0] 1217 | --------------------------------------------------------------------------------